drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <linux/mlx5/fs.h>
  35 #include <net/vxlan.h>
  36 #include <net/geneve.h>
  37 #include <linux/bpf.h>
  38 #include <linux/debugfs.h>
  39 #include <linux/if_bridge.h>
  40 #include <linux/filter.h>
  41 #include <net/page_pool.h>
  42 #include <net/pkt_sched.h>
  43 #include <net/xdp_sock_drv.h>
  44 #include "eswitch.h"
  45 #include "en.h"
  46 #include "en/txrx.h"
  47 #include "en_tc.h"
  48 #include "en_rep.h"
  49 #include "en_accel/ipsec.h"
  50 #include "en_accel/macsec.h"
  51 #include "en_accel/en_accel.h"
  52 #include "en_accel/ktls.h"
  53 #include "lib/vxlan.h"
  54 #include "lib/clock.h"
  55 #include "en/port.h"
  56 #include "en/xdp.h"
  57 #include "lib/eq.h"
  58 #include "en/monitor_stats.h"
  59 #include "en/health.h"
  60 #include "en/params.h"
  61 #include "en/xsk/pool.h"
  62 #include "en/xsk/setup.h"
  63 #include "en/xsk/rx.h"
  64 #include "en/xsk/tx.h"
  65 #include "en/hv_vhca_stats.h"
  66 #include "en/devlink.h"
  67 #include "lib/mlx5.h"
  68 #include "en/ptp.h"
  69 #include "en/htb.h"
  70 #include "qos.h"
  71 #include "en/trap.h"
  72
  73 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
  74                                             enum mlx5e_mpwrq_umr_mode umr_mode)
  75 {
  76         u16 umr_wqebbs, max_wqebbs;
  77         bool striding_rq_umr;
  78
  79         striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  80                           MLX5_CAP_ETH(mdev, reg_umr_sq);
  81         if (!striding_rq_umr)
  82                 return false;
  83
  84         umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
  85         max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
  86         /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is
  87          * calculated from mlx5e_get_max_sq_aligned_wqebbs.
  88          */
  89         if (WARN_ON(umr_wqebbs > max_wqebbs))
  90                 return false;
  91
  92         return true;
  93 }
  94
  95 void mlx5e_update_carrier(struct mlx5e_priv *priv)
  96 {
  97         struct mlx5_core_dev *mdev = priv->mdev;
  98         u8 port_state;
  99         bool up;
 100
 101         port_state = mlx5_query_vport_state(mdev,
 102                                             MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
 103                                             0);
 104
 105         up = port_state == VPORT_STATE_UP;
 106         if (up == netif_carrier_ok(priv->netdev))
 107                 netif_carrier_event(priv->netdev);
 108         if (up) {
 109                 netdev_info(priv->netdev, "Link up\n");
 110                 netif_carrier_on(priv->netdev);
 111         } else {
 112                 netdev_info(priv->netdev, "Link down\n");
 113                 netif_carrier_off(priv->netdev);
 114         }
 115 }
 116
 117 static void mlx5e_update_carrier_work(struct work_struct *work)
 118 {
 119         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 120                                                update_carrier_work);
 121
 122         mutex_lock(&priv->state_lock);
 123         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 124                 if (priv->profile->update_carrier)
 125                         priv->profile->update_carrier(priv);
 126         mutex_unlock(&priv->state_lock);
 127 }
 128
 129 static void mlx5e_update_stats_work(struct work_struct *work)
 130 {
 131         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 132                                                update_stats_work);
 133
 134         mutex_lock(&priv->state_lock);
 135         priv->profile->update_stats(priv);
 136         mutex_unlock(&priv->state_lock);
 137 }
 138
 139 void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
 140 {
 141         if (!priv->profile->update_stats)
 142                 return;
 143
 144         if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
 145                 return;
 146
 147         queue_work(priv->wq, &priv->update_stats_work);
 148 }
 149
 150 static int async_event(struct notifier_block *nb, unsigned long event, void *data)
 151 {
 152         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
 153         struct mlx5_eqe   *eqe = data;
 154
 155         if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
 156                 return NOTIFY_DONE;
 157
 158         switch (eqe->sub_type) {
 159         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
 160         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
 161                 queue_work(priv->wq, &priv->update_carrier_work);
 162                 break;
 163         default:
 164                 return NOTIFY_DONE;
 165         }
 166
 167         return NOTIFY_OK;
 168 }
 169
 170 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 171 {
 172         priv->events_nb.notifier_call = async_event;
 173         mlx5_notifier_register(priv->mdev, &priv->events_nb);
 174 }
 175
 176 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 177 {
 178         mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
 179 }
 180
 181 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
 182 {
 183         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
 184         struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data;
 185         int err;
 186
 187         switch (event) {
 188         case MLX5_DRIVER_EVENT_TYPE_TRAP:
 189                 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap);
 190                 if (err) {
 191                         trap_event_ctx->err = err;
 192                         return NOTIFY_BAD;
 193                 }
 194                 break;
 195         default:
 196                 return NOTIFY_DONE;
 197         }
 198         return NOTIFY_OK;
 199 }
 200
 201 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
 202 {
 203         priv->blocking_events_nb.notifier_call = blocking_event;
 204         mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
 205 }
 206
 207 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
 208 {
 209         mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
 210 }
 211
 212 static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode)
 213 {
 214         u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
 215         u32 sz;
 216
 217         sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT);
 218
 219         return sz / MLX5_OCTWORD;
 220 }
 221
 222 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
 223                                        struct mlx5e_icosq *sq,
 224                                        struct mlx5e_umr_wqe *wqe)
 225 {
 226         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 227         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 228         u16 octowords;
 229         u8 ds_cnt;
 230
 231         ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift,
 232                                                      rq->mpwqe.umr_mode),
 233                               MLX5_SEND_WQE_DS);
 234
 235         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 236                                       ds_cnt);
 237         cseg->umr_mkey  = rq->mpwqe.umr_mkey_be;
 238
 239         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
 240         octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode);
 241         ucseg->xlt_octowords = cpu_to_be16(octowords);
 242         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 243 }
 244
 245 static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node)
 246 {
 247         rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo),
 248                                          GFP_KERNEL, node);
 249         if (!rq->mpwqe.shampo)
 250                 return -ENOMEM;
 251         return 0;
 252 }
 253
 254 static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq)
 255 {
 256         kvfree(rq->mpwqe.shampo);
 257 }
 258
 259 static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
 260 {
 261         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 262
 263         shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL,
 264                                             node);
 265         shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq,
 266                                                 sizeof(*shampo->info)),
 267                                      GFP_KERNEL, node);
 268         shampo->pages = kvzalloc_node(array_size(shampo->hd_per_wq,
 269                                                  sizeof(*shampo->pages)),
 270                                      GFP_KERNEL, node);
 271         if (!shampo->bitmap || !shampo->info || !shampo->pages)
 272                 goto err_nomem;
 273
 274         return 0;
 275
 276 err_nomem:
 277         kvfree(shampo->info);
 278         kvfree(shampo->bitmap);
 279         kvfree(shampo->pages);
 280
 281         return -ENOMEM;
 282 }
 283
 284 static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
 285 {
 286         kvfree(rq->mpwqe.shampo->bitmap);
 287         kvfree(rq->mpwqe.shampo->info);
 288         kvfree(rq->mpwqe.shampo->pages);
 289 }
 290
 291 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 292 {
 293         int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 294         size_t alloc_size;
 295
 296         alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info,
 297                                                    alloc_units.frag_pages,
 298                                                    rq->mpwqe.pages_per_wqe));
 299
 300         rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
 301         if (!rq->mpwqe.info)
 302                 return -ENOMEM;
 303
 304         /* For deferred page release (release right before alloc), make sure
 305          * that on first round release is not called.
 306          */
 307         for (int i = 0; i < wq_sz; i++) {
 308                 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i);
 309
 310                 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
 311         }
 312
 313         mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
 314
 315         return 0;
 316 }
 317
 318
 319 static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)
 320 {
 321         switch (umr_mode) {
 322         case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
 323                 return MLX5_MKC_ACCESS_MODE_MTT;
 324         case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
 325                 return MLX5_MKC_ACCESS_MODE_KSM;
 326         case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
 327                 return MLX5_MKC_ACCESS_MODE_KLMS;
 328         case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
 329                 return MLX5_MKC_ACCESS_MODE_KSM;
 330         }
 331         WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
 332         return 0;
 333 }
 334
 335 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
 336                                  u32 npages, u8 page_shift, u32 *umr_mkey,
 337                                  dma_addr_t filler_addr,
 338                                  enum mlx5e_mpwrq_umr_mode umr_mode,
 339                                  u32 xsk_chunk_size)
 340 {
 341         struct mlx5_mtt *mtt;
 342         struct mlx5_ksm *ksm;
 343         struct mlx5_klm *klm;
 344         u32 octwords;
 345         int inlen;
 346         void *mkc;
 347         u32 *in;
 348         int err;
 349         int i;
 350
 351         if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED ||
 352              umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) &&
 353             !MLX5_CAP_GEN(mdev, fixed_buffer_size)) {
 354                 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n");
 355                 return -EINVAL;
 356         }
 357
 358         octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode);
 359
 360         inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in),
 361                                     MLX5_OCTWORD, octwords);
 362         if (inlen < 0)
 363                 return inlen;
 364
 365         in = kvzalloc(inlen, GFP_KERNEL);
 366         if (!in)
 367                 return -ENOMEM;
 368
 369         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 370
 371         MLX5_SET(mkc, mkc, free, 1);
 372         MLX5_SET(mkc, mkc, umr_en, 1);
 373         MLX5_SET(mkc, mkc, lw, 1);
 374         MLX5_SET(mkc, mkc, lr, 1);
 375         MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode));
 376         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 377         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 378         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 379         MLX5_SET64(mkc, mkc, len, npages << page_shift);
 380         MLX5_SET(mkc, mkc, translations_octword_size, octwords);
 381         if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)
 382                 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2);
 383         else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED)
 384                 MLX5_SET(mkc, mkc, log_page_size, page_shift);
 385         MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords);
 386
 387         /* Initialize the mkey with all MTTs pointing to a default
 388          * page (filler_addr). When the channels are activated, UMR
 389          * WQEs will redirect the RX WQEs to the actual memory from
 390          * the RQ's pool, while the gaps (wqe_overflow) remain mapped
 391          * to the default page.
 392          */
 393         switch (umr_mode) {
 394         case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
 395                 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 396                 for (i = 0; i < npages; i++) {
 397                         klm[i << 1] = (struct mlx5_klm) {
 398                                 .va = cpu_to_be64(filler_addr),
 399                                 .bcount = cpu_to_be32(xsk_chunk_size),
 400                                 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
 401                         };
 402                         klm[(i << 1) + 1] = (struct mlx5_klm) {
 403                                 .va = cpu_to_be64(filler_addr),
 404                                 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size),
 405                                 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
 406                         };
 407                 }
 408                 break;
 409         case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
 410                 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 411                 for (i = 0; i < npages; i++)
 412                         ksm[i] = (struct mlx5_ksm) {
 413                                 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
 414                                 .va = cpu_to_be64(filler_addr),
 415                         };
 416                 break;
 417         case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
 418                 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 419                 for (i = 0; i < npages; i++)
 420                         mtt[i] = (struct mlx5_mtt) {
 421                                 .ptag = cpu_to_be64(filler_addr),
 422                         };
 423                 break;
 424         case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
 425                 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 426                 for (i = 0; i < npages * 4; i++) {
 427                         ksm[i] = (struct mlx5_ksm) {
 428                                 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
 429                                 .va = cpu_to_be64(filler_addr),
 430                         };
 431                 }
 432                 break;
 433         }
 434
 435         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 436
 437         kvfree(in);
 438         return err;
 439 }
 440
 441 static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
 442                                      u64 nentries,
 443                                      u32 *umr_mkey)
 444 {
 445         int inlen;
 446         void *mkc;
 447         u32 *in;
 448         int err;
 449
 450         inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 451
 452         in = kvzalloc(inlen, GFP_KERNEL);
 453         if (!in)
 454                 return -ENOMEM;
 455
 456         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 457
 458         MLX5_SET(mkc, mkc, free, 1);
 459         MLX5_SET(mkc, mkc, umr_en, 1);
 460         MLX5_SET(mkc, mkc, lw, 1);
 461         MLX5_SET(mkc, mkc, lr, 1);
 462         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
 463         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 464         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 465         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 466         MLX5_SET(mkc, mkc, translations_octword_size, nentries);
 467         MLX5_SET(mkc, mkc, length64, 1);
 468         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 469
 470         kvfree(in);
 471         return err;
 472 }
 473
 474 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
 475 {
 476         u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0;
 477         u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 478         u32 num_entries, max_num_entries;
 479         u32 umr_mkey;
 480         int err;
 481
 482         max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode);
 483
 484         /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */
 485         if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe,
 486                                             &num_entries) ||
 487                          num_entries > max_num_entries))
 488                 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n",
 489                               __func__, wq_size, rq->mpwqe.mtts_per_wqe,
 490                               max_num_entries);
 491
 492         err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift,
 493                                     &umr_mkey, rq->wqe_overflow.addr,
 494                                     rq->mpwqe.umr_mode, xsk_chunk_size);
 495         rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey);
 496         return err;
 497 }
 498
 499 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
 500                                        struct mlx5e_rq *rq)
 501 {
 502         u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size));
 503
 504         if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) {
 505                 mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n",
 506                               max_klm_size, rq->mpwqe.shampo->hd_per_wq);
 507                 return -EINVAL;
 508         }
 509         return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq,
 510                                          &rq->mpwqe.shampo->mkey);
 511 }
 512
 513 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 514 {
 515         struct mlx5e_wqe_frag_info next_frag = {};
 516         struct mlx5e_wqe_frag_info *prev = NULL;
 517         int i;
 518
 519         WARN_ON(rq->xsk_pool);
 520
 521         next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0];
 522
 523         /* Skip first release due to deferred release. */
 524         next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 525
 526         for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 527                 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 528                 struct mlx5e_wqe_frag_info *frag =
 529                         &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
 530                 int f;
 531
 532                 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 533                         if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
 534                                 /* Pages are assigned at runtime. */
 535                                 next_frag.frag_page++;
 536                                 next_frag.offset = 0;
 537                                 if (prev)
 538                                         prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
 539                         }
 540                         *frag = next_frag;
 541
 542                         /* prepare next */
 543                         next_frag.offset += frag_info[f].frag_stride;
 544                         prev = frag;
 545                 }
 546         }
 547
 548         if (prev)
 549                 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
 550 }
 551
 552 static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq)
 553 {
 554         int i;
 555
 556         /* Assumptions used by XSK batched allocator. */
 557         WARN_ON(rq->wqe.info.num_frags != 1);
 558         WARN_ON(rq->wqe.info.log_num_frags != 0);
 559         WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
 560
 561         /* Considering the above assumptions a fragment maps to a single
 562          * xsk_buff.
 563          */
 564         for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 565                 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i];
 566
 567                 /* Skip first release due to deferred release as WQES are
 568                  * not allocated yet.
 569                  */
 570                 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 571         }
 572 }
 573
 574 static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node)
 575 {
 576         int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 577         int len = wq_sz << rq->wqe.info.log_num_frags;
 578         struct mlx5e_wqe_frag_info *frags;
 579         union mlx5e_alloc_units *aus;
 580         int aus_sz;
 581
 582         if (rq->xsk_pool)
 583                 aus_sz = sizeof(*aus->xsk_buffs);
 584         else
 585                 aus_sz = sizeof(*aus->frag_pages);
 586
 587         aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node);
 588         if (!aus)
 589                 return -ENOMEM;
 590
 591         frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node);
 592         if (!frags) {
 593                 kvfree(aus);
 594                 return -ENOMEM;
 595         }
 596
 597         rq->wqe.alloc_units = aus;
 598         rq->wqe.frags = frags;
 599
 600         if (rq->xsk_pool)
 601                 mlx5e_init_xsk_buffs(rq);
 602         else
 603                 mlx5e_init_frags_partition(rq);
 604
 605         return 0;
 606 }
 607
 608 static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq)
 609 {
 610         kvfree(rq->wqe.frags);
 611         kvfree(rq->wqe.alloc_units);
 612 }
 613
 614 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
 615 {
 616         struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
 617
 618         mlx5e_reporter_rq_cqe_err(rq);
 619 }
 620
 621 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 622 {
 623         rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
 624         if (!rq->wqe_overflow.page)
 625                 return -ENOMEM;
 626
 627         rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
 628                                              PAGE_SIZE, rq->buff.map_dir);
 629         if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
 630                 __free_page(rq->wqe_overflow.page);
 631                 return -ENOMEM;
 632         }
 633         return 0;
 634 }
 635
 636 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 637 {
 638          dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
 639                         rq->buff.map_dir);
 640          __free_page(rq->wqe_overflow.page);
 641 }
 642
 643 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
 644                              struct mlx5e_rq *rq)
 645 {
 646         struct mlx5_core_dev *mdev = c->mdev;
 647         int err;
 648
 649         rq->wq_type      = params->rq_wq_type;
 650         rq->pdev         = c->pdev;
 651         rq->netdev       = c->netdev;
 652         rq->priv         = c->priv;
 653         rq->tstamp       = c->tstamp;
 654         rq->clock        = &mdev->clock;
 655         rq->icosq        = &c->icosq;
 656         rq->ix           = c->ix;
 657         rq->channel      = c;
 658         rq->mdev         = mdev;
 659         rq->hw_mtu =
 660                 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en;
 661         rq->xdpsq        = &c->rq_xdpsq;
 662         rq->stats        = &c->priv->channel_stats[c->ix]->rq;
 663         rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
 664         err = mlx5e_rq_set_handlers(rq, params, NULL);
 665         if (err)
 666                 return err;
 667
 668         return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id);
 669 }
 670
 671 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
 672                                 struct mlx5e_params *params,
 673                                 struct mlx5e_rq_param *rqp,
 674                                 struct mlx5e_rq *rq,
 675                                 u32 *pool_size,
 676                                 int node)
 677 {
 678         void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
 679         int wq_size;
 680         int err;
 681
 682         if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 683                 return 0;
 684         err = mlx5e_rq_shampo_hd_alloc(rq, node);
 685         if (err)
 686                 goto out;
 687         rq->mpwqe.shampo->hd_per_wq =
 688                 mlx5e_shampo_hd_per_wq(mdev, params, rqp);
 689         err = mlx5e_create_rq_hd_umr_mkey(mdev, rq);
 690         if (err)
 691                 goto err_shampo_hd;
 692         err = mlx5e_rq_shampo_hd_info_alloc(rq, node);
 693         if (err)
 694                 goto err_shampo_info;
 695         rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node);
 696         if (!rq->hw_gro_data) {
 697                 err = -ENOMEM;
 698                 goto err_hw_gro_data;
 699         }
 700         rq->mpwqe.shampo->key =
 701                 cpu_to_be32(rq->mpwqe.shampo->mkey);
 702         rq->mpwqe.shampo->hd_per_wqe =
 703                 mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
 704         wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
 705         *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
 706                      MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
 707         return 0;
 708
 709 err_hw_gro_data:
 710         mlx5e_rq_shampo_hd_info_free(rq);
 711 err_shampo_info:
 712         mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey);
 713 err_shampo_hd:
 714         mlx5e_rq_shampo_hd_free(rq);
 715 out:
 716         return err;
 717 }
 718
 719 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
 720 {
 721         if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 722                 return;
 723
 724         kvfree(rq->hw_gro_data);
 725         mlx5e_rq_shampo_hd_info_free(rq);
 726         mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey);
 727         mlx5e_rq_shampo_hd_free(rq);
 728 }
 729
 730 static __be32 mlx5e_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev)
 731 {
 732         u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
 733         u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
 734         int res;
 735
 736         if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey))
 737                 return MLX5_TERMINATE_SCATTER_LIST_LKEY;
 738
 739         MLX5_SET(query_special_contexts_in, in, opcode,
 740                  MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
 741         res = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
 742         if (res)
 743                 return MLX5_TERMINATE_SCATTER_LIST_LKEY;
 744
 745         res = MLX5_GET(query_special_contexts_out, out,
 746                        terminate_scatter_list_mkey);
 747         return cpu_to_be32(res);
 748 }
 749
 750 static int mlx5e_alloc_rq(struct mlx5e_params *params,
 751                           struct mlx5e_xsk_param *xsk,
 752                           struct mlx5e_rq_param *rqp,
 753                           int node, struct mlx5e_rq *rq)
 754 {
 755         struct mlx5_core_dev *mdev = rq->mdev;
 756         void *rqc = rqp->rqc;
 757         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 758         u32 pool_size;
 759         int wq_sz;
 760         int err;
 761         int i;
 762
 763         rqp->wq.db_numa_node = node;
 764         INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
 765
 766         if (params->xdp_prog)
 767                 bpf_prog_inc(params->xdp_prog);
 768         RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
 769
 770         rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 771         rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
 772         pool_size = 1 << params->log_rq_mtu_frames;
 773
 774         rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
 775
 776         switch (rq->wq_type) {
 777         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 778                 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
 779                                         &rq->wq_ctrl);
 780                 if (err)
 781                         goto err_rq_xdp_prog;
 782
 783                 err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
 784                 if (err)
 785                         goto err_rq_wq_destroy;
 786
 787                 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
 788
 789                 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 790
 791                 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
 792                 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
 793                 rq->mpwqe.pages_per_wqe =
 794                         mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
 795                                                   rq->mpwqe.umr_mode);
 796                 rq->mpwqe.umr_wqebbs =
 797                         mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift,
 798                                                rq->mpwqe.umr_mode);
 799                 rq->mpwqe.mtts_per_wqe =
 800                         mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift,
 801                                                  rq->mpwqe.umr_mode);
 802
 803                 pool_size = rq->mpwqe.pages_per_wqe <<
 804                         mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
 805
 806                 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
 807                         pool_size *= 2; /* additional page per packet for the linear part */
 808
 809                 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 810                 rq->mpwqe.num_strides =
 811                         BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
 812                 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
 813
 814                 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
 815
 816                 err = mlx5e_create_rq_umr_mkey(mdev, rq);
 817                 if (err)
 818                         goto err_rq_drop_page;
 819
 820                 err = mlx5e_rq_alloc_mpwqe_info(rq, node);
 821                 if (err)
 822                         goto err_rq_mkey;
 823
 824                 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node);
 825                 if (err)
 826                         goto err_free_mpwqe_info;
 827
 828                 break;
 829         default: /* MLX5_WQ_TYPE_CYCLIC */
 830                 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
 831                                          &rq->wq_ctrl);
 832                 if (err)
 833                         goto err_rq_xdp_prog;
 834
 835                 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
 836
 837                 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 838
 839                 rq->wqe.info = rqp->frags_info;
 840                 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 841
 842                 err = mlx5e_init_wqe_alloc_info(rq, node);
 843                 if (err)
 844                         goto err_rq_wq_destroy;
 845         }
 846
 847         if (xsk) {
 848                 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 849                                                  MEM_TYPE_XSK_BUFF_POOL, NULL);
 850                 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
 851         } else {
 852                 /* Create a page_pool and register it with rxq */
 853                 struct page_pool_params pp_params = { 0 };
 854
 855                 pp_params.order     = 0;
 856                 pp_params.flags     = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | PP_FLAG_PAGE_FRAG;
 857                 pp_params.pool_size = pool_size;
 858                 pp_params.nid       = node;
 859                 pp_params.dev       = rq->pdev;
 860                 pp_params.napi      = rq->cq.napi;
 861                 pp_params.dma_dir   = rq->buff.map_dir;
 862                 pp_params.max_len   = PAGE_SIZE;
 863
 864                 /* page_pool can be used even when there is no rq->xdp_prog,
 865                  * given page_pool does not handle DMA mapping there is no
 866                  * required state to clear. And page_pool gracefully handle
 867                  * elevated refcnt.
 868                  */
 869                 rq->page_pool = page_pool_create(&pp_params);
 870                 if (IS_ERR(rq->page_pool)) {
 871                         err = PTR_ERR(rq->page_pool);
 872                         rq->page_pool = NULL;
 873                         goto err_free_by_rq_type;
 874                 }
 875                 if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
 876                         err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 877                                                          MEM_TYPE_PAGE_POOL, rq->page_pool);
 878         }
 879         if (err)
 880                 goto err_destroy_page_pool;
 881
 882         for (i = 0; i < wq_sz; i++) {
 883                 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 884                         struct mlx5e_rx_wqe_ll *wqe =
 885                                 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
 886                         u32 byte_count =
 887                                 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
 888                         u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) <<
 889                                 rq->mpwqe.page_shift;
 890                         u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ?
 891                                        0 : rq->buff.headroom;
 892
 893                         wqe->data[0].addr = cpu_to_be64(dma_offset + headroom);
 894                         wqe->data[0].byte_count = cpu_to_be32(byte_count);
 895                         wqe->data[0].lkey = rq->mpwqe.umr_mkey_be;
 896                 } else {
 897                         struct mlx5e_rx_wqe_cyc *wqe =
 898                                 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
 899                         int f;
 900
 901                         for (f = 0; f < rq->wqe.info.num_frags; f++) {
 902                                 u32 frag_size = rq->wqe.info.arr[f].frag_size |
 903                                         MLX5_HW_START_PADDING;
 904
 905                                 wqe->data[f].byte_count = cpu_to_be32(frag_size);
 906                                 wqe->data[f].lkey = rq->mkey_be;
 907                         }
 908                         /* check if num_frags is not a pow of two */
 909                         if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
 910                                 wqe->data[f].byte_count = 0;
 911                                 wqe->data[f].lkey = mlx5e_get_terminate_scatter_list_mkey(mdev);
 912                                 wqe->data[f].addr = 0;
 913                         }
 914                 }
 915         }
 916
 917         INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
 918
 919         switch (params->rx_cq_moderation.cq_period_mode) {
 920         case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
 921                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 922                 break;
 923         case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
 924         default:
 925                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 926         }
 927
 928         return 0;
 929
 930 err_destroy_page_pool:
 931         page_pool_destroy(rq->page_pool);
 932 err_free_by_rq_type:
 933         switch (rq->wq_type) {
 934         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 935                 mlx5e_rq_free_shampo(rq);
 936 err_free_mpwqe_info:
 937                 kvfree(rq->mpwqe.info);
 938 err_rq_mkey:
 939                 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
 940 err_rq_drop_page:
 941                 mlx5e_free_mpwqe_rq_drop_page(rq);
 942                 break;
 943         default: /* MLX5_WQ_TYPE_CYCLIC */
 944                 mlx5e_free_wqe_alloc_info(rq);
 945         }
 946 err_rq_wq_destroy:
 947         mlx5_wq_destroy(&rq->wq_ctrl);
 948 err_rq_xdp_prog:
 949         if (params->xdp_prog)
 950                 bpf_prog_put(params->xdp_prog);
 951
 952         return err;
 953 }
 954
 955 static void mlx5e_free_rq(struct mlx5e_rq *rq)
 956 {
 957         struct bpf_prog *old_prog;
 958
 959         if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
 960                 old_prog = rcu_dereference_protected(rq->xdp_prog,
 961                                                      lockdep_is_held(&rq->priv->state_lock));
 962                 if (old_prog)
 963                         bpf_prog_put(old_prog);
 964         }
 965
 966         switch (rq->wq_type) {
 967         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 968                 kvfree(rq->mpwqe.info);
 969                 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
 970                 mlx5e_free_mpwqe_rq_drop_page(rq);
 971                 mlx5e_rq_free_shampo(rq);
 972                 break;
 973         default: /* MLX5_WQ_TYPE_CYCLIC */
 974                 mlx5e_free_wqe_alloc_info(rq);
 975         }
 976
 977         xdp_rxq_info_unreg(&rq->xdp_rxq);
 978         page_pool_destroy(rq->page_pool);
 979         mlx5_wq_destroy(&rq->wq_ctrl);
 980 }
 981
 982 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 983 {
 984         struct mlx5_core_dev *mdev = rq->mdev;
 985         u8 ts_format;
 986         void *in;
 987         void *rqc;
 988         void *wq;
 989         int inlen;
 990         int err;
 991
 992         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 993                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 994         in = kvzalloc(inlen, GFP_KERNEL);
 995         if (!in)
 996                 return -ENOMEM;
 997
 998         ts_format = mlx5_is_real_time_rq(mdev) ?
 999                             MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1000                             MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1001         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1002         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
1003
1004         memcpy(rqc, param->rqc, sizeof(param->rqc));
1005
1006         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
1007         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
1008         MLX5_SET(rqc,  rqc, ts_format,          ts_format);
1009         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
1010                                                 MLX5_ADAPTER_PAGE_SHIFT);
1011         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
1012
1013         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
1014                 MLX5_SET(wq, wq, log_headers_buffer_entry_num,
1015                          order_base_2(rq->mpwqe.shampo->hd_per_wq));
1016                 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey);
1017         }
1018
1019         mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
1020                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1021
1022         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1023
1024         kvfree(in);
1025
1026         return err;
1027 }
1028
1029 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
1030 {
1031         struct mlx5_core_dev *mdev = rq->mdev;
1032
1033         void *in;
1034         void *rqc;
1035         int inlen;
1036         int err;
1037
1038         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1039         in = kvzalloc(inlen, GFP_KERNEL);
1040         if (!in)
1041                 return -ENOMEM;
1042
1043         if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
1044                 mlx5e_rqwq_reset(rq);
1045
1046         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1047
1048         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1049         MLX5_SET(rqc, rqc, state, next_state);
1050
1051         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
1052
1053         kvfree(in);
1054
1055         return err;
1056 }
1057
1058 static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state)
1059 {
1060         struct net_device *dev = rq->netdev;
1061         int err;
1062
1063         err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST);
1064         if (err) {
1065                 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn);
1066                 return err;
1067         }
1068         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1069         if (err) {
1070                 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn);
1071                 return err;
1072         }
1073
1074         return 0;
1075 }
1076
1077 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state)
1078 {
1079         mlx5e_free_rx_descs(rq);
1080
1081         return mlx5e_rq_to_ready(rq, curr_state);
1082 }
1083
1084 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
1085 {
1086         struct mlx5_core_dev *mdev = rq->mdev;
1087         void *in;
1088         void *rqc;
1089         int inlen;
1090         int err;
1091
1092         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1093         in = kvzalloc(inlen, GFP_KERNEL);
1094         if (!in)
1095                 return -ENOMEM;
1096
1097         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1098
1099         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
1100         MLX5_SET64(modify_rq_in, in, modify_bitmask,
1101                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
1102         MLX5_SET(rqc, rqc, vsd, vsd);
1103         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
1104
1105         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
1106
1107         kvfree(in);
1108
1109         return err;
1110 }
1111
1112 void mlx5e_destroy_rq(struct mlx5e_rq *rq)
1113 {
1114         mlx5_core_destroy_rq(rq->mdev, rq->rqn);
1115 }
1116
1117 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
1118 {
1119         unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
1120
1121         u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
1122
1123         do {
1124                 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
1125                         return 0;
1126
1127                 msleep(20);
1128         } while (time_before(jiffies, exp_time));
1129
1130         netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
1131                     rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
1132
1133         mlx5e_reporter_rx_timeout(rq);
1134         return -ETIMEDOUT;
1135 }
1136
1137 void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq)
1138 {
1139         struct mlx5_wq_ll *wq;
1140         u16 head;
1141         int i;
1142
1143         if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
1144                 return;
1145
1146         wq = &rq->mpwqe.wq;
1147         head = wq->head;
1148
1149         /* Release WQEs that are in missing state: they have been
1150          * popped from the list after completion but were not freed
1151          * due to deferred release.
1152          * Also free the linked-list reserved entry, hence the "+ 1".
1153          */
1154         for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) {
1155                 rq->dealloc_wqe(rq, head);
1156                 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
1157         }
1158
1159         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
1160                 u16 len;
1161
1162                 len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) &
1163                       (rq->mpwqe.shampo->hd_per_wq - 1);
1164                 mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false);
1165                 rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci;
1166         }
1167
1168         rq->mpwqe.actual_wq_head = wq->head;
1169         rq->mpwqe.umr_in_progress = 0;
1170         rq->mpwqe.umr_completed = 0;
1171 }
1172
1173 void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
1174 {
1175         __be16 wqe_ix_be;
1176         u16 wqe_ix;
1177
1178         if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
1179                 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
1180
1181                 mlx5e_free_rx_missing_descs(rq);
1182
1183                 while (!mlx5_wq_ll_is_empty(wq)) {
1184                         struct mlx5e_rx_wqe_ll *wqe;
1185
1186                         wqe_ix_be = *wq->tail_next;
1187                         wqe_ix    = be16_to_cpu(wqe_ix_be);
1188                         wqe       = mlx5_wq_ll_get_wqe(wq, wqe_ix);
1189                         rq->dealloc_wqe(rq, wqe_ix);
1190                         mlx5_wq_ll_pop(wq, wqe_ix_be,
1191                                        &wqe->next.next_wqe_index);
1192                 }
1193
1194                 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
1195                         mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq,
1196                                                 0, true);
1197         } else {
1198                 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1199                 u16 missing = mlx5_wq_cyc_missing(wq);
1200                 u16 head = mlx5_wq_cyc_get_head(wq);
1201
1202                 while (!mlx5_wq_cyc_is_empty(wq)) {
1203                         wqe_ix = mlx5_wq_cyc_get_tail(wq);
1204                         rq->dealloc_wqe(rq, wqe_ix);
1205                         mlx5_wq_cyc_pop(wq);
1206                 }
1207                 /* Missing slots might also contain unreleased pages due to
1208                  * deferred release.
1209                  */
1210                 while (missing--) {
1211                         wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++);
1212                         rq->dealloc_wqe(rq, wqe_ix);
1213                 }
1214         }
1215
1216 }
1217
1218 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
1219                   struct mlx5e_xsk_param *xsk, int node,
1220                   struct mlx5e_rq *rq)
1221 {
1222         struct mlx5_core_dev *mdev = rq->mdev;
1223         int err;
1224
1225         if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1226                 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
1227
1228         err = mlx5e_alloc_rq(params, xsk, param, node, rq);
1229         if (err)
1230                 return err;
1231
1232         err = mlx5e_create_rq(rq, param);
1233         if (err)
1234                 goto err_free_rq;
1235
1236         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1237         if (err)
1238                 goto err_destroy_rq;
1239
1240         if (MLX5_CAP_ETH(mdev, cqe_checksum_full))
1241                 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
1242
1243         if (params->rx_dim_enabled)
1244                 __set_bit(MLX5E_RQ_STATE_DIM, &rq->state);
1245
1246         /* We disable csum_complete when XDP is enabled since
1247          * XDP programs might manipulate packets which will render
1248          * skb->checksum incorrect.
1249          */
1250         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog)
1251                 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state);
1252
1253         /* For CQE compression on striding RQ, use stride index provided by
1254          * HW if capability is supported.
1255          */
1256         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
1257             MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index))
1258                 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state);
1259
1260         /* For enhanced CQE compression packet processing. decompress
1261          * session according to the enhanced layout.
1262          */
1263         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) &&
1264             MLX5_CAP_GEN(mdev, enhanced_cqe_compression))
1265                 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state);
1266
1267         return 0;
1268
1269 err_destroy_rq:
1270         mlx5e_destroy_rq(rq);
1271 err_free_rq:
1272         mlx5e_free_rq(rq);
1273
1274         return err;
1275 }
1276
1277 void mlx5e_activate_rq(struct mlx5e_rq *rq)
1278 {
1279         set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1280 }
1281
1282 void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
1283 {
1284         clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1285         synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
1286 }
1287
1288 void mlx5e_close_rq(struct mlx5e_rq *rq)
1289 {
1290         cancel_work_sync(&rq->dim.work);
1291         cancel_work_sync(&rq->recover_work);
1292         mlx5e_destroy_rq(rq);
1293         mlx5e_free_rx_descs(rq);
1294         mlx5e_free_rq(rq);
1295 }
1296
1297 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
1298 {
1299         kvfree(sq->db.xdpi_fifo.xi);
1300         kvfree(sq->db.wqe_info);
1301 }
1302
1303 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
1304 {
1305         struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
1306         int wq_sz        = mlx5_wq_cyc_get_size(&sq->wq);
1307         int entries = wq_sz * MLX5_SEND_WQEBB_NUM_DS * 2; /* upper bound for maximum num of
1308                                                            * entries of all xmit_modes.
1309                                                            */
1310         size_t size;
1311
1312         size = array_size(sizeof(*xdpi_fifo->xi), entries);
1313         xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa);
1314         if (!xdpi_fifo->xi)
1315                 return -ENOMEM;
1316
1317         xdpi_fifo->pc   = &sq->xdpi_fifo_pc;
1318         xdpi_fifo->cc   = &sq->xdpi_fifo_cc;
1319         xdpi_fifo->mask = entries - 1;
1320
1321         return 0;
1322 }
1323
1324 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
1325 {
1326         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1327         size_t size;
1328         int err;
1329
1330         size = array_size(sizeof(*sq->db.wqe_info), wq_sz);
1331         sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1332         if (!sq->db.wqe_info)
1333                 return -ENOMEM;
1334
1335         err = mlx5e_alloc_xdpsq_fifo(sq, numa);
1336         if (err) {
1337                 mlx5e_free_xdpsq_db(sq);
1338                 return err;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
1345                              struct mlx5e_params *params,
1346                              struct xsk_buff_pool *xsk_pool,
1347                              struct mlx5e_sq_param *param,
1348                              struct mlx5e_xdpsq *sq,
1349                              bool is_redirect)
1350 {
1351         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1352         struct mlx5_core_dev *mdev = c->mdev;
1353         struct mlx5_wq_cyc *wq = &sq->wq;
1354         int err;
1355
1356         sq->pdev      = c->pdev;
1357         sq->mkey_be   = c->mkey_be;
1358         sq->channel   = c;
1359         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1360         sq->min_inline_mode = params->tx_min_inline_mode;
1361         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN;
1362         sq->xsk_pool  = xsk_pool;
1363
1364         sq->stats = sq->xsk_pool ?
1365                 &c->priv->channel_stats[c->ix]->xsksq :
1366                 is_redirect ?
1367                         &c->priv->channel_stats[c->ix]->xdpsq :
1368                         &c->priv->channel_stats[c->ix]->rq_xdpsq;
1369         sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) :
1370                                         mlx5e_stop_room_for_max_wqe(mdev);
1371         sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
1372
1373         param->wq.db_numa_node = cpu_to_node(c->cpu);
1374         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1375         if (err)
1376                 return err;
1377         wq->db = &wq->db[MLX5_SND_DBR];
1378
1379         err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1380         if (err)
1381                 goto err_sq_wq_destroy;
1382
1383         return 0;
1384
1385 err_sq_wq_destroy:
1386         mlx5_wq_destroy(&sq->wq_ctrl);
1387
1388         return err;
1389 }
1390
1391 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1392 {
1393         mlx5e_free_xdpsq_db(sq);
1394         mlx5_wq_destroy(&sq->wq_ctrl);
1395 }
1396
1397 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1398 {
1399         kvfree(sq->db.wqe_info);
1400 }
1401
1402 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1403 {
1404         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1405         size_t size;
1406
1407         size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1408         sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1409         if (!sq->db.wqe_info)
1410                 return -ENOMEM;
1411
1412         return 0;
1413 }
1414
1415 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1416 {
1417         struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1418                                               recover_work);
1419
1420         mlx5e_reporter_icosq_cqe_err(sq);
1421 }
1422
1423 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
1424 {
1425         struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1426                                               recover_work);
1427
1428         /* Not implemented yet. */
1429
1430         netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n");
1431 }
1432
1433 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1434                              struct mlx5e_sq_param *param,
1435                              struct mlx5e_icosq *sq,
1436                              work_func_t recover_work_func)
1437 {
1438         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1439         struct mlx5_core_dev *mdev = c->mdev;
1440         struct mlx5_wq_cyc *wq = &sq->wq;
1441         int err;
1442
1443         sq->channel   = c;
1444         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1445         sq->reserved_room = param->stop_room;
1446
1447         param->wq.db_numa_node = cpu_to_node(c->cpu);
1448         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1449         if (err)
1450                 return err;
1451         wq->db = &wq->db[MLX5_SND_DBR];
1452
1453         err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1454         if (err)
1455                 goto err_sq_wq_destroy;
1456
1457         INIT_WORK(&sq->recover_work, recover_work_func);
1458
1459         return 0;
1460
1461 err_sq_wq_destroy:
1462         mlx5_wq_destroy(&sq->wq_ctrl);
1463
1464         return err;
1465 }
1466
1467 static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1468 {
1469         mlx5e_free_icosq_db(sq);
1470         mlx5_wq_destroy(&sq->wq_ctrl);
1471 }
1472
1473 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1474 {
1475         kvfree(sq->db.wqe_info);
1476         kvfree(sq->db.skb_fifo.fifo);
1477         kvfree(sq->db.dma_fifo);
1478 }
1479
1480 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1481 {
1482         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1483         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1484
1485         sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1486                                                    sizeof(*sq->db.dma_fifo)),
1487                                         GFP_KERNEL, numa);
1488         sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1489                                                         sizeof(*sq->db.skb_fifo.fifo)),
1490                                         GFP_KERNEL, numa);
1491         sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1492                                                    sizeof(*sq->db.wqe_info)),
1493                                         GFP_KERNEL, numa);
1494         if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
1495                 mlx5e_free_txqsq_db(sq);
1496                 return -ENOMEM;
1497         }
1498
1499         sq->dma_fifo_mask = df_sz - 1;
1500
1501         sq->db.skb_fifo.pc   = &sq->skb_fifo_pc;
1502         sq->db.skb_fifo.cc   = &sq->skb_fifo_cc;
1503         sq->db.skb_fifo.mask = df_sz - 1;
1504
1505         return 0;
1506 }
1507
1508 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1509                              int txq_ix,
1510                              struct mlx5e_params *params,
1511                              struct mlx5e_sq_param *param,
1512                              struct mlx5e_txqsq *sq,
1513                              int tc)
1514 {
1515         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1516         struct mlx5_core_dev *mdev = c->mdev;
1517         struct mlx5_wq_cyc *wq = &sq->wq;
1518         int err;
1519
1520         sq->pdev      = c->pdev;
1521         sq->clock     = &mdev->clock;
1522         sq->mkey_be   = c->mkey_be;
1523         sq->netdev    = c->netdev;
1524         sq->mdev      = c->mdev;
1525         sq->channel   = c;
1526         sq->priv      = c->priv;
1527         sq->ch_ix     = c->ix;
1528         sq->txq_ix    = txq_ix;
1529         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1530         sq->min_inline_mode = params->tx_min_inline_mode;
1531         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1532         sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
1533         INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
1534         if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1535                 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
1536         if (mlx5_ipsec_device_caps(c->priv->mdev))
1537                 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1538         if (param->is_mpw)
1539                 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
1540         sq->stop_room = param->stop_room;
1541         sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
1542
1543         param->wq.db_numa_node = cpu_to_node(c->cpu);
1544         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1545         if (err)
1546                 return err;
1547         wq->db    = &wq->db[MLX5_SND_DBR];
1548
1549         err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1550         if (err)
1551                 goto err_sq_wq_destroy;
1552
1553         INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1554         sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1555
1556         return 0;
1557
1558 err_sq_wq_destroy:
1559         mlx5_wq_destroy(&sq->wq_ctrl);
1560
1561         return err;
1562 }
1563
1564 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1565 {
1566         mlx5e_free_txqsq_db(sq);
1567         mlx5_wq_destroy(&sq->wq_ctrl);
1568 }
1569
1570 static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1571                            struct mlx5e_sq_param *param,
1572                            struct mlx5e_create_sq_param *csp,
1573                            u32 *sqn)
1574 {
1575         u8 ts_format;
1576         void *in;
1577         void *sqc;
1578         void *wq;
1579         int inlen;
1580         int err;
1581
1582         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1583                 sizeof(u64) * csp->wq_ctrl->buf.npages;
1584         in = kvzalloc(inlen, GFP_KERNEL);
1585         if (!in)
1586                 return -ENOMEM;
1587
1588         ts_format = mlx5_is_real_time_sq(mdev) ?
1589                             MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1590                             MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1591         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1592         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1593
1594         memcpy(sqc, param->sqc, sizeof(param->sqc));
1595         MLX5_SET(sqc,  sqc, tis_lst_sz, csp->tis_lst_sz);
1596         MLX5_SET(sqc,  sqc, tis_num_0, csp->tisn);
1597         MLX5_SET(sqc,  sqc, cqn, csp->cqn);
1598         MLX5_SET(sqc,  sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
1599         MLX5_SET(sqc,  sqc, ts_format, ts_format);
1600
1601
1602         if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1603                 MLX5_SET(sqc,  sqc, min_wqe_inline_mode, csp->min_inline_mode);
1604
1605         MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
1606         MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
1607
1608         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1609         MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
1610         MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
1611                                           MLX5_ADAPTER_PAGE_SHIFT);
1612         MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
1613
1614         mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1615                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1616
1617         err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1618
1619         kvfree(in);
1620
1621         return err;
1622 }
1623
1624 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1625                     struct mlx5e_modify_sq_param *p)
1626 {
1627         u64 bitmask = 0;
1628         void *in;
1629         void *sqc;
1630         int inlen;
1631         int err;
1632
1633         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1634         in = kvzalloc(inlen, GFP_KERNEL);
1635         if (!in)
1636                 return -ENOMEM;
1637
1638         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1639
1640         MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1641         MLX5_SET(sqc, sqc, state, p->next_state);
1642         if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1643                 bitmask |= 1;
1644                 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
1645         }
1646         if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1647                 bitmask |= 1 << 2;
1648                 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1649         }
1650         MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
1651
1652         err = mlx5_core_modify_sq(mdev, sqn, in);
1653
1654         kvfree(in);
1655
1656         return err;
1657 }
1658
1659 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1660 {
1661         mlx5_core_destroy_sq(mdev, sqn);
1662 }
1663
1664 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1665                         struct mlx5e_sq_param *param,
1666                         struct mlx5e_create_sq_param *csp,
1667                         u16 qos_queue_group_id,
1668                         u32 *sqn)
1669 {
1670         struct mlx5e_modify_sq_param msp = {0};
1671         int err;
1672
1673         err = mlx5e_create_sq(mdev, param, csp, sqn);
1674         if (err)
1675                 return err;
1676
1677         msp.curr_state = MLX5_SQC_STATE_RST;
1678         msp.next_state = MLX5_SQC_STATE_RDY;
1679         if (qos_queue_group_id) {
1680                 msp.qos_update = true;
1681                 msp.qos_queue_group_id = qos_queue_group_id;
1682         }
1683         err = mlx5e_modify_sq(mdev, *sqn, &msp);
1684         if (err)
1685                 mlx5e_destroy_sq(mdev, *sqn);
1686
1687         return err;
1688 }
1689
1690 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1691                                 struct mlx5e_txqsq *sq, u32 rate);
1692
1693 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1694                      struct mlx5e_params *params, struct mlx5e_sq_param *param,
1695                      struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1696                      struct mlx5e_sq_stats *sq_stats)
1697 {
1698         struct mlx5e_create_sq_param csp = {};
1699         u32 tx_rate;
1700         int err;
1701
1702         err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1703         if (err)
1704                 return err;
1705
1706         sq->stats = sq_stats;
1707
1708         csp.tisn            = tisn;
1709         csp.tis_lst_sz      = 1;
1710         csp.cqn             = sq->cq.mcq.cqn;
1711         csp.wq_ctrl         = &sq->wq_ctrl;
1712         csp.min_inline_mode = sq->min_inline_mode;
1713         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
1714         if (err)
1715                 goto err_free_txqsq;
1716
1717         tx_rate = c->priv->tx_rates[sq->txq_ix];
1718         if (tx_rate)
1719                 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1720
1721         if (params->tx_dim_enabled)
1722                 sq->state |= BIT(MLX5E_SQ_STATE_DIM);
1723
1724         return 0;
1725
1726 err_free_txqsq:
1727         mlx5e_free_txqsq(sq);
1728
1729         return err;
1730 }
1731
1732 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1733 {
1734         sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
1735         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1736         netdev_tx_reset_queue(sq->txq);
1737         netif_tx_start_queue(sq->txq);
1738 }
1739
1740 void mlx5e_tx_disable_queue(struct netdev_queue *txq)
1741 {
1742         __netif_tx_lock_bh(txq);
1743         netif_tx_stop_queue(txq);
1744         __netif_tx_unlock_bh(txq);
1745 }
1746
1747 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1748 {
1749         struct mlx5_wq_cyc *wq = &sq->wq;
1750
1751         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1752         synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1753
1754         mlx5e_tx_disable_queue(sq->txq);
1755
1756         /* last doorbell out, godspeed .. */
1757         if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1758                 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1759                 struct mlx5e_tx_wqe *nop;
1760
1761                 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1762                         .num_wqebbs = 1,
1763                 };
1764
1765                 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1766                 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1767         }
1768 }
1769
1770 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1771 {
1772         struct mlx5_core_dev *mdev = sq->mdev;
1773         struct mlx5_rate_limit rl = {0};
1774
1775         cancel_work_sync(&sq->dim.work);
1776         cancel_work_sync(&sq->recover_work);
1777         mlx5e_destroy_sq(mdev, sq->sqn);
1778         if (sq->rate_limit) {
1779                 rl.rate = sq->rate_limit;
1780                 mlx5_rl_remove_rate(mdev, &rl);
1781         }
1782         mlx5e_free_txqsq_descs(sq);
1783         mlx5e_free_txqsq(sq);
1784 }
1785
1786 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
1787 {
1788         struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1789                                               recover_work);
1790
1791         mlx5e_reporter_tx_err_cqe(sq);
1792 }
1793
1794 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1795                             struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
1796                             work_func_t recover_work_func)
1797 {
1798         struct mlx5e_create_sq_param csp = {};
1799         int err;
1800
1801         err = mlx5e_alloc_icosq(c, param, sq, recover_work_func);
1802         if (err)
1803                 return err;
1804
1805         csp.cqn             = sq->cq.mcq.cqn;
1806         csp.wq_ctrl         = &sq->wq_ctrl;
1807         csp.min_inline_mode = params->tx_min_inline_mode;
1808         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1809         if (err)
1810                 goto err_free_icosq;
1811
1812         if (param->is_tls) {
1813                 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list();
1814                 if (IS_ERR(sq->ktls_resync)) {
1815                         err = PTR_ERR(sq->ktls_resync);
1816                         goto err_destroy_icosq;
1817                 }
1818         }
1819         return 0;
1820
1821 err_destroy_icosq:
1822         mlx5e_destroy_sq(c->mdev, sq->sqn);
1823 err_free_icosq:
1824         mlx5e_free_icosq(sq);
1825
1826         return err;
1827 }
1828
1829 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
1830 {
1831         set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1832 }
1833
1834 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
1835 {
1836         clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1837         synchronize_net(); /* Sync with NAPI. */
1838 }
1839
1840 static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1841 {
1842         struct mlx5e_channel *c = sq->channel;
1843
1844         if (sq->ktls_resync)
1845                 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync);
1846         mlx5e_destroy_sq(c->mdev, sq->sqn);
1847         mlx5e_free_icosq_descs(sq);
1848         mlx5e_free_icosq(sq);
1849 }
1850
1851 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1852                      struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1853                      struct mlx5e_xdpsq *sq, bool is_redirect)
1854 {
1855         struct mlx5e_create_sq_param csp = {};
1856         int err;
1857
1858         err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
1859         if (err)
1860                 return err;
1861
1862         csp.tis_lst_sz      = 1;
1863         csp.tisn            = c->priv->tisn[c->lag_port][0]; /* tc = 0 */
1864         csp.cqn             = sq->cq.mcq.cqn;
1865         csp.wq_ctrl         = &sq->wq_ctrl;
1866         csp.min_inline_mode = sq->min_inline_mode;
1867         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1868
1869         if (param->is_xdp_mb)
1870                 set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state);
1871
1872         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1873         if (err)
1874                 goto err_free_xdpsq;
1875
1876         mlx5e_set_xmit_fp(sq, param->is_mpw);
1877
1878         if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) {
1879                 unsigned int ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1;
1880                 unsigned int inline_hdr_sz = 0;
1881                 int i;
1882
1883                 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1884                         inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1885                         ds_cnt++;
1886                 }
1887
1888                 /* Pre initialize fixed WQE fields */
1889                 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1890                         struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1891                         struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1892                         struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
1893
1894                         sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
1895                                 .num_wqebbs = 1,
1896                                 .num_pkts   = 1,
1897                         };
1898
1899                         cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1900                         eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1901                 }
1902         }
1903
1904         return 0;
1905
1906 err_free_xdpsq:
1907         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1908         mlx5e_free_xdpsq(sq);
1909
1910         return err;
1911 }
1912
1913 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1914 {
1915         struct mlx5e_channel *c = sq->channel;
1916
1917         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1918         synchronize_net(); /* Sync with NAPI. */
1919
1920         mlx5e_destroy_sq(c->mdev, sq->sqn);
1921         mlx5e_free_xdpsq_descs(sq);
1922         mlx5e_free_xdpsq(sq);
1923 }
1924
1925 static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv,
1926                                  struct mlx5e_cq_param *param,
1927                                  struct mlx5e_cq *cq)
1928 {
1929         struct mlx5_core_dev *mdev = priv->mdev;
1930         struct mlx5_core_cq *mcq = &cq->mcq;
1931         int err;
1932         u32 i;
1933
1934         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1935                                &cq->wq_ctrl);
1936         if (err)
1937                 return err;
1938
1939         mcq->cqe_sz     = 64;
1940         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1941         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1942         *mcq->set_ci_db = 0;
1943         *mcq->arm_db    = 0;
1944         mcq->vector     = param->eq_ix;
1945         mcq->comp       = mlx5e_completion_event;
1946         mcq->event      = mlx5e_cq_error_event;
1947
1948         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1949                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1950
1951                 cqe->op_own = 0xf1;
1952                 cqe->validity_iteration_count = 0xff;
1953         }
1954
1955         cq->mdev = mdev;
1956         cq->netdev = priv->netdev;
1957         cq->priv = priv;
1958
1959         return 0;
1960 }
1961
1962 static int mlx5e_alloc_cq(struct mlx5e_priv *priv,
1963                           struct mlx5e_cq_param *param,
1964                           struct mlx5e_create_cq_param *ccp,
1965                           struct mlx5e_cq *cq)
1966 {
1967         int err;
1968
1969         param->wq.buf_numa_node = ccp->node;
1970         param->wq.db_numa_node  = ccp->node;
1971         param->eq_ix            = ccp->ix;
1972
1973         err = mlx5e_alloc_cq_common(priv, param, cq);
1974
1975         cq->napi     = ccp->napi;
1976         cq->ch_stats = ccp->ch_stats;
1977
1978         return err;
1979 }
1980
1981 static void mlx5e_free_cq(struct mlx5e_cq *cq)
1982 {
1983         mlx5_wq_destroy(&cq->wq_ctrl);
1984 }
1985
1986 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1987 {
1988         u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1989         struct mlx5_core_dev *mdev = cq->mdev;
1990         struct mlx5_core_cq *mcq = &cq->mcq;
1991
1992         void *in;
1993         void *cqc;
1994         int inlen;
1995         int eqn;
1996         int err;
1997
1998         err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn);
1999         if (err)
2000                 return err;
2001
2002         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
2003                 sizeof(u64) * cq->wq_ctrl.buf.npages;
2004         in = kvzalloc(inlen, GFP_KERNEL);
2005         if (!in)
2006                 return -ENOMEM;
2007
2008         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2009
2010         memcpy(cqc, param->cqc, sizeof(param->cqc));
2011
2012         mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
2013                                   (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
2014
2015         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
2016         MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
2017         MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
2018         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
2019                                             MLX5_ADAPTER_PAGE_SHIFT);
2020         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
2021
2022         err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
2023
2024         kvfree(in);
2025
2026         if (err)
2027                 return err;
2028
2029         mlx5e_cq_arm(cq);
2030
2031         return 0;
2032 }
2033
2034 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
2035 {
2036         mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
2037 }
2038
2039 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
2040                   struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
2041                   struct mlx5e_cq *cq)
2042 {
2043         struct mlx5_core_dev *mdev = priv->mdev;
2044         int err;
2045
2046         err = mlx5e_alloc_cq(priv, param, ccp, cq);
2047         if (err)
2048                 return err;
2049
2050         err = mlx5e_create_cq(cq, param);
2051         if (err)
2052                 goto err_free_cq;
2053
2054         if (MLX5_CAP_GEN(mdev, cq_moderation))
2055                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
2056         return 0;
2057
2058 err_free_cq:
2059         mlx5e_free_cq(cq);
2060
2061         return err;
2062 }
2063
2064 void mlx5e_close_cq(struct mlx5e_cq *cq)
2065 {
2066         mlx5e_destroy_cq(cq);
2067         mlx5e_free_cq(cq);
2068 }
2069
2070 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2071                              struct mlx5e_params *params,
2072                              struct mlx5e_create_cq_param *ccp,
2073                              struct mlx5e_channel_param *cparam)
2074 {
2075         int err;
2076         int tc;
2077
2078         for (tc = 0; tc < c->num_tc; tc++) {
2079                 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp,
2080                                     ccp, &c->sq[tc].cq);
2081                 if (err)
2082                         goto err_close_tx_cqs;
2083         }
2084
2085         return 0;
2086
2087 err_close_tx_cqs:
2088         for (tc--; tc >= 0; tc--)
2089                 mlx5e_close_cq(&c->sq[tc].cq);
2090
2091         return err;
2092 }
2093
2094 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2095 {
2096         int tc;
2097
2098         for (tc = 0; tc < c->num_tc; tc++)
2099                 mlx5e_close_cq(&c->sq[tc].cq);
2100 }
2101
2102 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq)
2103 {
2104         int tc;
2105
2106         for (tc = 0; tc < TC_MAX_QUEUE; tc++)
2107                 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count)
2108                         return tc;
2109
2110         WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq);
2111         return -ENOENT;
2112 }
2113
2114 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix,
2115                                         u32 *hw_id)
2116 {
2117         int tc;
2118
2119         if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) {
2120                 *hw_id = 0;
2121                 return 0;
2122         }
2123
2124         tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix);
2125         if (tc < 0)
2126                 return tc;
2127
2128         if (tc >= params->mqprio.num_tc) {
2129                 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u",
2130                      tc, params->mqprio.num_tc);
2131                 return -EINVAL;
2132         }
2133
2134         *hw_id = params->mqprio.channel.hw_id[tc];
2135         return 0;
2136 }
2137
2138 static int mlx5e_open_sqs(struct mlx5e_channel *c,
2139                           struct mlx5e_params *params,
2140                           struct mlx5e_channel_param *cparam)
2141 {
2142         int err, tc;
2143
2144         for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) {
2145                 int txq_ix = c->ix + tc * params->num_channels;
2146                 u32 qos_queue_group_id;
2147
2148                 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id);
2149                 if (err)
2150                         goto err_close_sqs;
2151
2152                 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
2153                                        params, &cparam->txq_sq, &c->sq[tc], tc,
2154                                        qos_queue_group_id,
2155                                        &c->priv->channel_stats[c->ix]->sq[tc]);
2156                 if (err)
2157                         goto err_close_sqs;
2158         }
2159
2160         return 0;
2161
2162 err_close_sqs:
2163         for (tc--; tc >= 0; tc--)
2164                 mlx5e_close_txqsq(&c->sq[tc]);
2165
2166         return err;
2167 }
2168
2169 static void mlx5e_close_sqs(struct mlx5e_channel *c)
2170 {
2171         int tc;
2172
2173         for (tc = 0; tc < c->num_tc; tc++)
2174                 mlx5e_close_txqsq(&c->sq[tc]);
2175 }
2176
2177 static int mlx5e_set_sq_maxrate(struct net_device *dev,
2178                                 struct mlx5e_txqsq *sq, u32 rate)
2179 {
2180         struct mlx5e_priv *priv = netdev_priv(dev);
2181         struct mlx5_core_dev *mdev = priv->mdev;
2182         struct mlx5e_modify_sq_param msp = {0};
2183         struct mlx5_rate_limit rl = {0};
2184         u16 rl_index = 0;
2185         int err;
2186
2187         if (rate == sq->rate_limit)
2188                 /* nothing to do */
2189                 return 0;
2190
2191         if (sq->rate_limit) {
2192                 rl.rate = sq->rate_limit;
2193                 /* remove current rl index to free space to next ones */
2194                 mlx5_rl_remove_rate(mdev, &rl);
2195         }
2196
2197         sq->rate_limit = 0;
2198
2199         if (rate) {
2200                 rl.rate = rate;
2201                 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
2202                 if (err) {
2203                         netdev_err(dev, "Failed configuring rate %u: %d\n",
2204                                    rate, err);
2205                         return err;
2206                 }
2207         }
2208
2209         msp.curr_state = MLX5_SQC_STATE_RDY;
2210         msp.next_state = MLX5_SQC_STATE_RDY;
2211         msp.rl_index   = rl_index;
2212         msp.rl_update  = true;
2213         err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
2214         if (err) {
2215                 netdev_err(dev, "Failed configuring rate %u: %d\n",
2216                            rate, err);
2217                 /* remove the rate from the table */
2218                 if (rate)
2219                         mlx5_rl_remove_rate(mdev, &rl);
2220                 return err;
2221         }
2222
2223         sq->rate_limit = rate;
2224         return 0;
2225 }
2226
2227 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
2228 {
2229         struct mlx5e_priv *priv = netdev_priv(dev);
2230         struct mlx5_core_dev *mdev = priv->mdev;
2231         struct mlx5e_txqsq *sq = priv->txq2sq[index];
2232         int err = 0;
2233
2234         if (!mlx5_rl_is_supported(mdev)) {
2235                 netdev_err(dev, "Rate limiting is not supported on this device\n");
2236                 return -EINVAL;
2237         }
2238
2239         /* rate is given in Mb/sec, HW config is in Kb/sec */
2240         rate = rate << 10;
2241
2242         /* Check whether rate in valid range, 0 is always valid */
2243         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
2244                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
2245                 return -ERANGE;
2246         }
2247
2248         mutex_lock(&priv->state_lock);
2249         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
2250                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
2251         if (!err)
2252                 priv->tx_rates[index] = rate;
2253         mutex_unlock(&priv->state_lock);
2254
2255         return err;
2256 }
2257
2258 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
2259                              struct mlx5e_rq_param *rq_params)
2260 {
2261         int err;
2262
2263         err = mlx5e_init_rxq_rq(c, params, &c->rq);
2264         if (err)
2265                 return err;
2266
2267         return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq);
2268 }
2269
2270 static int mlx5e_open_queues(struct mlx5e_channel *c,
2271                              struct mlx5e_params *params,
2272                              struct mlx5e_channel_param *cparam)
2273 {
2274         struct dim_cq_moder icocq_moder = {0, 0};
2275         struct mlx5e_create_cq_param ccp;
2276         int err;
2277
2278         mlx5e_build_create_cq_param(&ccp, c);
2279
2280         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp,
2281                             &c->async_icosq.cq);
2282         if (err)
2283                 return err;
2284
2285         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp,
2286                             &c->icosq.cq);
2287         if (err)
2288                 goto err_close_async_icosq_cq;
2289
2290         err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
2291         if (err)
2292                 goto err_close_icosq_cq;
2293
2294         err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
2295                             &c->xdpsq.cq);
2296         if (err)
2297                 goto err_close_tx_cqs;
2298
2299         err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
2300                             &c->rq.cq);
2301         if (err)
2302                 goto err_close_xdp_tx_cqs;
2303
2304         err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
2305                                      &ccp, &c->rq_xdpsq.cq) : 0;
2306         if (err)
2307                 goto err_close_rx_cq;
2308
2309         spin_lock_init(&c->async_icosq_lock);
2310
2311         err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq,
2312                                mlx5e_async_icosq_err_cqe_work);
2313         if (err)
2314                 goto err_close_xdpsq_cq;
2315
2316         mutex_init(&c->icosq_recovery_lock);
2317
2318         err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq,
2319                                mlx5e_icosq_err_cqe_work);
2320         if (err)
2321                 goto err_close_async_icosq;
2322
2323         err = mlx5e_open_sqs(c, params, cparam);
2324         if (err)
2325                 goto err_close_icosq;
2326
2327         err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
2328         if (err)
2329                 goto err_close_sqs;
2330
2331         if (c->xdp) {
2332                 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
2333                                        &c->rq_xdpsq, false);
2334                 if (err)
2335                         goto err_close_rq;
2336         }
2337
2338         err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
2339         if (err)
2340                 goto err_close_xdp_sq;
2341
2342         return 0;
2343
2344 err_close_xdp_sq:
2345         if (c->xdp)
2346                 mlx5e_close_xdpsq(&c->rq_xdpsq);
2347
2348 err_close_rq:
2349         mlx5e_close_rq(&c->rq);
2350
2351 err_close_sqs:
2352         mlx5e_close_sqs(c);
2353
2354 err_close_icosq:
2355         mlx5e_close_icosq(&c->icosq);
2356
2357 err_close_async_icosq:
2358         mlx5e_close_icosq(&c->async_icosq);
2359
2360 err_close_xdpsq_cq:
2361         if (c->xdp)
2362                 mlx5e_close_cq(&c->rq_xdpsq.cq);
2363
2364 err_close_rx_cq:
2365         mlx5e_close_cq(&c->rq.cq);
2366
2367 err_close_xdp_tx_cqs:
2368         mlx5e_close_cq(&c->xdpsq.cq);
2369
2370 err_close_tx_cqs:
2371         mlx5e_close_tx_cqs(c);
2372
2373 err_close_icosq_cq:
2374         mlx5e_close_cq(&c->icosq.cq);
2375
2376 err_close_async_icosq_cq:
2377         mlx5e_close_cq(&c->async_icosq.cq);
2378
2379         return err;
2380 }
2381
2382 static void mlx5e_close_queues(struct mlx5e_channel *c)
2383 {
2384         mlx5e_close_xdpsq(&c->xdpsq);
2385         if (c->xdp)
2386                 mlx5e_close_xdpsq(&c->rq_xdpsq);
2387         /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */
2388         cancel_work_sync(&c->icosq.recover_work);
2389         mlx5e_close_rq(&c->rq);
2390         mlx5e_close_sqs(c);
2391         mlx5e_close_icosq(&c->icosq);
2392         mutex_destroy(&c->icosq_recovery_lock);
2393         mlx5e_close_icosq(&c->async_icosq);
2394         if (c->xdp)
2395                 mlx5e_close_cq(&c->rq_xdpsq.cq);
2396         mlx5e_close_cq(&c->rq.cq);
2397         mlx5e_close_cq(&c->xdpsq.cq);
2398         mlx5e_close_tx_cqs(c);
2399         mlx5e_close_cq(&c->icosq.cq);
2400         mlx5e_close_cq(&c->async_icosq.cq);
2401 }
2402
2403 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
2404 {
2405         u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
2406
2407         return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
2408 }
2409
2410 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
2411 {
2412         if (ix > priv->stats_nch)  {
2413                 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix,
2414                             priv->stats_nch);
2415                 return -EINVAL;
2416         }
2417
2418         if (priv->channel_stats[ix])
2419                 return 0;
2420
2421         /* Asymmetric dynamic memory allocation.
2422          * Freed in mlx5e_priv_arrays_free, not on channel closure.
2423          */
2424         mlx5e_dbg(DRV, priv, "Creating channel stats %d\n", ix);
2425         priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats),
2426                                                 GFP_KERNEL, cpu_to_node(cpu));
2427         if (!priv->channel_stats[ix])
2428                 return -ENOMEM;
2429         priv->stats_nch++;
2430
2431         return 0;
2432 }
2433
2434 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
2435 {
2436         spin_lock_bh(&c->async_icosq_lock);
2437         mlx5e_trigger_irq(&c->async_icosq);
2438         spin_unlock_bh(&c->async_icosq_lock);
2439 }
2440
2441 void mlx5e_trigger_napi_sched(struct napi_struct *napi)
2442 {
2443         local_bh_disable();
2444         napi_schedule(napi);
2445         local_bh_enable();
2446 }
2447
2448 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
2449                               struct mlx5e_params *params,
2450                               struct mlx5e_channel_param *cparam,
2451                               struct xsk_buff_pool *xsk_pool,
2452                               struct mlx5e_channel **cp)
2453 {
2454         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
2455         struct net_device *netdev = priv->netdev;
2456         struct mlx5e_xsk_param xsk;
2457         struct mlx5e_channel *c;
2458         unsigned int irq;
2459         int err;
2460
2461         err = mlx5_vector2irqn(priv->mdev, ix, &irq);
2462         if (err)
2463                 return err;
2464
2465         err = mlx5e_channel_stats_alloc(priv, ix, cpu);
2466         if (err)
2467                 return err;
2468
2469         c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
2470         if (!c)
2471                 return -ENOMEM;
2472
2473         c->priv     = priv;
2474         c->mdev     = priv->mdev;
2475         c->tstamp   = &priv->tstamp;
2476         c->ix       = ix;
2477         c->cpu      = cpu;
2478         c->pdev     = mlx5_core_dma_dev(priv->mdev);
2479         c->netdev   = priv->netdev;
2480         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
2481         c->num_tc   = mlx5e_get_dcb_num_tc(params);
2482         c->xdp      = !!params->xdp_prog;
2483         c->stats    = &priv->channel_stats[ix]->ch;
2484         c->aff_mask = irq_get_effective_affinity_mask(irq);
2485         c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
2486
2487         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll);
2488
2489         err = mlx5e_open_queues(c, params, cparam);
2490         if (unlikely(err))
2491                 goto err_napi_del;
2492
2493         if (xsk_pool) {
2494                 mlx5e_build_xsk_param(xsk_pool, &xsk);
2495                 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
2496                 if (unlikely(err))
2497                         goto err_close_queues;
2498         }
2499
2500         *cp = c;
2501
2502         return 0;
2503
2504 err_close_queues:
2505         mlx5e_close_queues(c);
2506
2507 err_napi_del:
2508         netif_napi_del(&c->napi);
2509
2510         kvfree(c);
2511
2512         return err;
2513 }
2514
2515 static void mlx5e_activate_channel(struct mlx5e_channel *c)
2516 {
2517         int tc;
2518
2519         napi_enable(&c->napi);
2520
2521         for (tc = 0; tc < c->num_tc; tc++)
2522                 mlx5e_activate_txqsq(&c->sq[tc]);
2523         mlx5e_activate_icosq(&c->icosq);
2524         mlx5e_activate_icosq(&c->async_icosq);
2525
2526         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2527                 mlx5e_activate_xsk(c);
2528         else
2529                 mlx5e_activate_rq(&c->rq);
2530 }
2531
2532 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2533 {
2534         int tc;
2535
2536         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2537                 mlx5e_deactivate_xsk(c);
2538         else
2539                 mlx5e_deactivate_rq(&c->rq);
2540
2541         mlx5e_deactivate_icosq(&c->async_icosq);
2542         mlx5e_deactivate_icosq(&c->icosq);
2543         for (tc = 0; tc < c->num_tc; tc++)
2544                 mlx5e_deactivate_txqsq(&c->sq[tc]);
2545         mlx5e_qos_deactivate_queues(c);
2546
2547         napi_disable(&c->napi);
2548 }
2549
2550 static void mlx5e_close_channel(struct mlx5e_channel *c)
2551 {
2552         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2553                 mlx5e_close_xsk(c);
2554         mlx5e_close_queues(c);
2555         mlx5e_qos_close_queues(c);
2556         netif_napi_del(&c->napi);
2557
2558         kvfree(c);
2559 }
2560
2561 int mlx5e_open_channels(struct mlx5e_priv *priv,
2562                         struct mlx5e_channels *chs)
2563 {
2564         struct mlx5e_channel_param *cparam;
2565         int err = -ENOMEM;
2566         int i;
2567
2568         chs->num = chs->params.num_channels;
2569
2570         chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2571         cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2572         if (!chs->c || !cparam)
2573                 goto err_free;
2574
2575         err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam);
2576         if (err)
2577                 goto err_free;
2578
2579         for (i = 0; i < chs->num; i++) {
2580                 struct xsk_buff_pool *xsk_pool = NULL;
2581
2582                 if (chs->params.xdp_prog)
2583                         xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
2584
2585                 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
2586                 if (err)
2587                         goto err_close_channels;
2588         }
2589
2590         if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) {
2591                 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
2592                 if (err)
2593                         goto err_close_channels;
2594         }
2595
2596         if (priv->htb) {
2597                 err = mlx5e_qos_open_queues(priv, chs);
2598                 if (err)
2599                         goto err_close_ptp;
2600         }
2601
2602         mlx5e_health_channels_update(priv);
2603         kvfree(cparam);
2604         return 0;
2605
2606 err_close_ptp:
2607         if (chs->ptp)
2608                 mlx5e_ptp_close(chs->ptp);
2609
2610 err_close_channels:
2611         for (i--; i >= 0; i--)
2612                 mlx5e_close_channel(chs->c[i]);
2613
2614 err_free:
2615         kfree(chs->c);
2616         kvfree(cparam);
2617         chs->num = 0;
2618         return err;
2619 }
2620
2621 static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
2622 {
2623         int i;
2624
2625         for (i = 0; i < chs->num; i++)
2626                 mlx5e_activate_channel(chs->c[i]);
2627
2628         if (priv->htb)
2629                 mlx5e_qos_activate_queues(priv);
2630
2631         for (i = 0; i < chs->num; i++)
2632                 mlx5e_trigger_napi_icosq(chs->c[i]);
2633
2634         if (chs->ptp)
2635                 mlx5e_ptp_activate_channel(chs->ptp);
2636 }
2637
2638 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2639 {
2640         int err = 0;
2641         int i;
2642
2643         for (i = 0; i < chs->num; i++) {
2644                 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2645                 struct mlx5e_channel *c = chs->c[i];
2646
2647                 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2648                         continue;
2649
2650                 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout);
2651
2652                 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2653                  * doesn't provide any Fill Ring entries at the setup stage.
2654                  */
2655         }
2656
2657         return err ? -ETIMEDOUT : 0;
2658 }
2659
2660 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2661 {
2662         int i;
2663
2664         if (chs->ptp)
2665                 mlx5e_ptp_deactivate_channel(chs->ptp);
2666
2667         for (i = 0; i < chs->num; i++)
2668                 mlx5e_deactivate_channel(chs->c[i]);
2669 }
2670
2671 void mlx5e_close_channels(struct mlx5e_channels *chs)
2672 {
2673         int i;
2674
2675         if (chs->ptp) {
2676                 mlx5e_ptp_close(chs->ptp);
2677                 chs->ptp = NULL;
2678         }
2679         for (i = 0; i < chs->num; i++)
2680                 mlx5e_close_channel(chs->c[i]);
2681
2682         kfree(chs->c);
2683         chs->num = 0;
2684 }
2685
2686 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
2687 {
2688         struct mlx5e_rx_res *res = priv->rx_res;
2689
2690         return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge);
2691 }
2692
2693 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);
2694
2695 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2696                          struct mlx5e_params *params, u16 mtu)
2697 {
2698         u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2699         int err;
2700
2701         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2702         if (err)
2703                 return err;
2704
2705         /* Update vport context MTU */
2706         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2707         return 0;
2708 }
2709
2710 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2711                             struct mlx5e_params *params, u16 *mtu)
2712 {
2713         u16 hw_mtu = 0;
2714         int err;
2715
2716         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2717         if (err || !hw_mtu) /* fallback to port oper mtu */
2718                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2719
2720         *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2721 }
2722
2723 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2724 {
2725         struct mlx5e_params *params = &priv->channels.params;
2726         struct net_device *netdev = priv->netdev;
2727         struct mlx5_core_dev *mdev = priv->mdev;
2728         u16 mtu;
2729         int err;
2730
2731         err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2732         if (err)
2733                 return err;
2734
2735         mlx5e_query_mtu(mdev, params, &mtu);
2736         if (mtu != params->sw_mtu)
2737                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2738                             __func__, mtu, params->sw_mtu);
2739
2740         params->sw_mtu = mtu;
2741         return 0;
2742 }
2743
2744 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2745
2746 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2747 {
2748         struct mlx5e_params *params = &priv->channels.params;
2749         struct net_device *netdev   = priv->netdev;
2750         struct mlx5_core_dev *mdev  = priv->mdev;
2751         u16 max_mtu;
2752
2753         /* MTU range: 68 - hw-specific max */
2754         netdev->min_mtu = ETH_MIN_MTU;
2755
2756         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2757         netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2758                                 ETH_MAX_MTU);
2759 }
2760
2761 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc,
2762                                 struct netdev_tc_txq *tc_to_txq)
2763 {
2764         int tc, err;
2765
2766         netdev_reset_tc(netdev);
2767
2768         if (ntc == 1)
2769                 return 0;
2770
2771         err = netdev_set_num_tc(netdev, ntc);
2772         if (err) {
2773                 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc);
2774                 return err;
2775         }
2776
2777         for (tc = 0; tc < ntc; tc++) {
2778                 u16 count, offset;
2779
2780                 count = tc_to_txq[tc].count;
2781                 offset = tc_to_txq[tc].offset;
2782                 netdev_set_tc_queue(netdev, tc, count, offset);
2783         }
2784
2785         return 0;
2786 }
2787
2788 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
2789 {
2790         int nch, ntc, num_txqs, err;
2791         int qos_queues = 0;
2792
2793         if (priv->htb)
2794                 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb);
2795
2796         nch = priv->channels.params.num_channels;
2797         ntc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2798         num_txqs = nch * ntc + qos_queues;
2799         if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
2800                 num_txqs += ntc;
2801
2802         mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs);
2803         err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
2804         if (err)
2805                 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
2806
2807         return err;
2808 }
2809
2810 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
2811 {
2812         struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq;
2813         struct net_device *netdev = priv->netdev;
2814         int old_num_txqs, old_ntc;
2815         int nch, ntc;
2816         int err;
2817         int i;
2818
2819         old_num_txqs = netdev->real_num_tx_queues;
2820         old_ntc = netdev->num_tc ? : 1;
2821         for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++)
2822                 old_tc_to_txq[i] = netdev->tc_to_txq[i];
2823
2824         nch = priv->channels.params.num_channels;
2825         ntc = priv->channels.params.mqprio.num_tc;
2826         tc_to_txq = priv->channels.params.mqprio.tc_to_txq;
2827
2828         err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq);
2829         if (err)
2830                 goto err_out;
2831         err = mlx5e_update_tx_netdev_queues(priv);
2832         if (err)
2833                 goto err_tcs;
2834         err = netif_set_real_num_rx_queues(netdev, nch);
2835         if (err) {
2836                 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
2837                 goto err_txqs;
2838         }
2839
2840         return 0;
2841
2842 err_txqs:
2843         /* netif_set_real_num_rx_queues could fail only when nch increased. Only
2844          * one of nch and ntc is changed in this function. That means, the call
2845          * to netif_set_real_num_tx_queues below should not fail, because it
2846          * decreases the number of TX queues.
2847          */
2848         WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs));
2849
2850 err_tcs:
2851         WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc,
2852                                           old_tc_to_txq));
2853 err_out:
2854         return err;
2855 }
2856
2857 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues);
2858
2859 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
2860                                            struct mlx5e_params *params)
2861 {
2862         struct mlx5_core_dev *mdev = priv->mdev;
2863         int num_comp_vectors, ix, irq;
2864
2865         num_comp_vectors = mlx5_comp_vectors_count(mdev);
2866
2867         for (ix = 0; ix < params->num_channels; ix++) {
2868                 cpumask_clear(priv->scratchpad.cpumask);
2869
2870                 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) {
2871                         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq));
2872
2873                         cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
2874                 }
2875
2876                 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
2877         }
2878 }
2879
2880 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
2881 {
2882         u16 count = priv->channels.params.num_channels;
2883         int err;
2884
2885         err = mlx5e_update_netdev_queues(priv);
2886         if (err)
2887                 return err;
2888
2889         mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
2890
2891         /* This function may be called on attach, before priv->rx_res is created. */
2892         if (!netif_is_rxfh_configured(priv->netdev) && priv->rx_res)
2893                 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count);
2894
2895         return 0;
2896 }
2897
2898 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
2899
2900 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
2901 {
2902         int i, ch, tc, num_tc;
2903
2904         ch = priv->channels.num;
2905         num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2906
2907         for (i = 0; i < ch; i++) {
2908                 for (tc = 0; tc < num_tc; tc++) {
2909                         struct mlx5e_channel *c = priv->channels.c[i];
2910                         struct mlx5e_txqsq *sq = &c->sq[tc];
2911
2912                         priv->txq2sq[sq->txq_ix] = sq;
2913                 }
2914         }
2915
2916         if (!priv->channels.ptp)
2917                 goto out;
2918
2919         if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state))
2920                 goto out;
2921
2922         for (tc = 0; tc < num_tc; tc++) {
2923                 struct mlx5e_ptp *c = priv->channels.ptp;
2924                 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
2925
2926                 priv->txq2sq[sq->txq_ix] = sq;
2927         }
2928
2929 out:
2930         /* Make the change to txq2sq visible before the queue is started.
2931          * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
2932          * which pairs with this barrier.
2933          */
2934         smp_wmb();
2935 }
2936
2937 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2938 {
2939         mlx5e_build_txq_maps(priv);
2940         mlx5e_activate_channels(priv, &priv->channels);
2941         mlx5e_xdp_tx_enable(priv);
2942
2943         /* dev_watchdog() wants all TX queues to be started when the carrier is
2944          * OK, including the ones in range real_num_tx_queues..num_tx_queues-1.
2945          * Make it happy to avoid TX timeout false alarms.
2946          */
2947         netif_tx_start_all_queues(priv->netdev);
2948
2949         if (mlx5e_is_vport_rep(priv))
2950                 mlx5e_rep_activate_channels(priv);
2951
2952         mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2953
2954         if (priv->rx_res)
2955                 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels);
2956 }
2957
2958 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2959 {
2960         if (priv->rx_res)
2961                 mlx5e_rx_res_channels_deactivate(priv->rx_res);
2962
2963         if (mlx5e_is_vport_rep(priv))
2964                 mlx5e_rep_deactivate_channels(priv);
2965
2966         /* The results of ndo_select_queue are unreliable, while netdev config
2967          * is being changed (real_num_tx_queues, num_tc). Stop all queues to
2968          * prevent ndo_start_xmit from being called, so that it can assume that
2969          * the selected queue is always valid.
2970          */
2971         netif_tx_disable(priv->netdev);
2972
2973         mlx5e_xdp_tx_disable(priv);
2974         mlx5e_deactivate_channels(&priv->channels);
2975 }
2976
2977 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
2978                                     struct mlx5e_params *new_params,
2979                                     mlx5e_fp_preactivate preactivate,
2980                                     void *context)
2981 {
2982         struct mlx5e_params old_params;
2983
2984         old_params = priv->channels.params;
2985         priv->channels.params = *new_params;
2986
2987         if (preactivate) {
2988                 int err;
2989
2990                 err = preactivate(priv, context);
2991                 if (err) {
2992                         priv->channels.params = old_params;
2993                         return err;
2994                 }
2995         }
2996
2997         return 0;
2998 }
2999
3000 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
3001                                       struct mlx5e_channels *new_chs,
3002                                       mlx5e_fp_preactivate preactivate,
3003                                       void *context)
3004 {
3005         struct net_device *netdev = priv->netdev;
3006         struct mlx5e_channels old_chs;
3007         int carrier_ok;
3008         int err = 0;
3009
3010         carrier_ok = netif_carrier_ok(netdev);
3011         netif_carrier_off(netdev);
3012
3013         mlx5e_deactivate_priv_channels(priv);
3014
3015         old_chs = priv->channels;
3016         priv->channels = *new_chs;
3017
3018         /* New channels are ready to roll, call the preactivate hook if needed
3019          * to modify HW settings or update kernel parameters.
3020          */
3021         if (preactivate) {
3022                 err = preactivate(priv, context);
3023                 if (err) {
3024                         priv->channels = old_chs;
3025                         goto out;
3026                 }
3027         }
3028
3029         mlx5e_close_channels(&old_chs);
3030         priv->profile->update_rx(priv);
3031
3032         mlx5e_selq_apply(&priv->selq);
3033 out:
3034         mlx5e_activate_priv_channels(priv);
3035
3036         /* return carrier back if needed */
3037         if (carrier_ok)
3038                 netif_carrier_on(netdev);
3039
3040         return err;
3041 }
3042
3043 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
3044                              struct mlx5e_params *params,
3045                              mlx5e_fp_preactivate preactivate,
3046                              void *context, bool reset)
3047 {
3048         struct mlx5e_channels *new_chs;
3049         int err;
3050
3051         reset &= test_bit(MLX5E_STATE_OPENED, &priv->state);
3052         if (!reset)
3053                 return mlx5e_switch_priv_params(priv, params, preactivate, context);
3054
3055         new_chs = kzalloc(sizeof(*new_chs), GFP_KERNEL);
3056         if (!new_chs)
3057                 return -ENOMEM;
3058         new_chs->params = *params;
3059
3060         mlx5e_selq_prepare_params(&priv->selq, &new_chs->params);
3061
3062         err = mlx5e_open_channels(priv, new_chs);
3063         if (err)
3064                 goto err_cancel_selq;
3065
3066         err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context);
3067         if (err)
3068                 goto err_close;
3069
3070         kfree(new_chs);
3071         return 0;
3072
3073 err_close:
3074         mlx5e_close_channels(new_chs);
3075
3076 err_cancel_selq:
3077         mlx5e_selq_cancel(&priv->selq);
3078         kfree(new_chs);
3079         return err;
3080 }
3081
3082 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
3083 {
3084         return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true);
3085 }
3086
3087 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
3088 {
3089         priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
3090         priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
3091 }
3092
3093 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
3094                                      enum mlx5_port_status state)
3095 {
3096         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3097         int vport_admin_state;
3098
3099         mlx5_set_port_admin_status(mdev, state);
3100
3101         if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
3102             !MLX5_CAP_GEN(mdev, uplink_follow))
3103                 return;
3104
3105         if (state == MLX5_PORT_UP)
3106                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
3107         else
3108                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3109
3110         mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
3111 }
3112
3113 int mlx5e_open_locked(struct net_device *netdev)
3114 {
3115         struct mlx5e_priv *priv = netdev_priv(netdev);
3116         int err;
3117
3118         mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params);
3119
3120         set_bit(MLX5E_STATE_OPENED, &priv->state);
3121
3122         err = mlx5e_open_channels(priv, &priv->channels);
3123         if (err)
3124                 goto err_clear_state_opened_flag;
3125
3126         err = priv->profile->update_rx(priv);
3127         if (err)
3128                 goto err_close_channels;
3129
3130         mlx5e_selq_apply(&priv->selq);
3131         mlx5e_activate_priv_channels(priv);
3132         mlx5e_apply_traps(priv, true);
3133         if (priv->profile->update_carrier)
3134                 priv->profile->update_carrier(priv);
3135
3136         mlx5e_queue_update_stats(priv);
3137         return 0;
3138
3139 err_close_channels:
3140         mlx5e_close_channels(&priv->channels);
3141 err_clear_state_opened_flag:
3142         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3143         mlx5e_selq_cancel(&priv->selq);
3144         return err;
3145 }
3146
3147 int mlx5e_open(struct net_device *netdev)
3148 {
3149         struct mlx5e_priv *priv = netdev_priv(netdev);
3150         int err;
3151
3152         mutex_lock(&priv->state_lock);
3153         err = mlx5e_open_locked(netdev);
3154         if (!err)
3155                 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
3156         mutex_unlock(&priv->state_lock);
3157
3158         return err;
3159 }
3160
3161 int mlx5e_close_locked(struct net_device *netdev)
3162 {
3163         struct mlx5e_priv *priv = netdev_priv(netdev);
3164
3165         /* May already be CLOSED in case a previous configuration operation
3166          * (e.g RX/TX queue size change) that involves close&open failed.
3167          */
3168         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3169                 return 0;
3170
3171         mlx5e_apply_traps(priv, false);
3172         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3173
3174         netif_carrier_off(priv->netdev);
3175         mlx5e_deactivate_priv_channels(priv);
3176         mlx5e_close_channels(&priv->channels);
3177
3178         return 0;
3179 }
3180
3181 int mlx5e_close(struct net_device *netdev)
3182 {
3183         struct mlx5e_priv *priv = netdev_priv(netdev);
3184         int err;
3185
3186         if (!netif_device_present(netdev))
3187                 return -ENODEV;
3188
3189         mutex_lock(&priv->state_lock);
3190         mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
3191         err = mlx5e_close_locked(netdev);
3192         mutex_unlock(&priv->state_lock);
3193
3194         return err;
3195 }
3196
3197 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
3198 {
3199         mlx5_wq_destroy(&rq->wq_ctrl);
3200 }
3201
3202 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3203                                struct mlx5e_rq *rq,
3204                                struct mlx5e_rq_param *param)
3205 {
3206         void *rqc = param->rqc;
3207         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3208         int err;
3209
3210         param->wq.db_numa_node = param->wq.buf_numa_node;
3211
3212         err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3213                                  &rq->wq_ctrl);
3214         if (err)
3215                 return err;
3216
3217         /* Mark as unused given "Drop-RQ" packets never reach XDP */
3218         xdp_rxq_info_unused(&rq->xdp_rxq);
3219
3220         rq->mdev = mdev;
3221
3222         return 0;
3223 }
3224
3225 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
3226                                struct mlx5e_cq *cq,
3227                                struct mlx5e_cq_param *param)
3228 {
3229         struct mlx5_core_dev *mdev = priv->mdev;
3230
3231         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3232         param->wq.db_numa_node  = dev_to_node(mlx5_core_dma_dev(mdev));
3233
3234         return mlx5e_alloc_cq_common(priv, param, cq);
3235 }
3236
3237 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3238                        struct mlx5e_rq *drop_rq)
3239 {
3240         struct mlx5_core_dev *mdev = priv->mdev;
3241         struct mlx5e_cq_param cq_param = {};
3242         struct mlx5e_rq_param rq_param = {};
3243         struct mlx5e_cq *cq = &drop_rq->cq;
3244         int err;
3245
3246         mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param);
3247
3248         err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
3249         if (err)
3250                 return err;
3251
3252         err = mlx5e_create_cq(cq, &cq_param);
3253         if (err)
3254                 goto err_free_cq;
3255
3256         err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
3257         if (err)
3258                 goto err_destroy_cq;
3259
3260         err = mlx5e_create_rq(drop_rq, &rq_param);
3261         if (err)
3262                 goto err_free_rq;
3263
3264         err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3265         if (err)
3266                 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3267
3268         return 0;
3269
3270 err_free_rq:
3271         mlx5e_free_drop_rq(drop_rq);
3272
3273 err_destroy_cq:
3274         mlx5e_destroy_cq(cq);
3275
3276 err_free_cq:
3277         mlx5e_free_cq(cq);
3278
3279         return err;
3280 }
3281
3282 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3283 {
3284         mlx5e_destroy_rq(drop_rq);
3285         mlx5e_free_drop_rq(drop_rq);
3286         mlx5e_destroy_cq(&drop_rq->cq);
3287         mlx5e_free_cq(&drop_rq->cq);
3288 }
3289
3290 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn)
3291 {
3292         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3293
3294         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
3295
3296         if (MLX5_GET(tisc, tisc, tls_en))
3297                 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn);
3298
3299         if (mlx5_lag_is_lacp_owner(mdev))
3300                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3301
3302         return mlx5_core_create_tis(mdev, in, tisn);
3303 }
3304
3305 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3306 {
3307         mlx5_core_destroy_tis(mdev, tisn);
3308 }
3309
3310 void mlx5e_destroy_tises(struct mlx5e_priv *priv)
3311 {
3312         int tc, i;
3313
3314         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++)
3315                 for (tc = 0; tc < priv->profile->max_tc; tc++)
3316                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3317 }
3318
3319 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev)
3320 {
3321         return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1;
3322 }
3323
3324 int mlx5e_create_tises(struct mlx5e_priv *priv)
3325 {
3326         int tc, i;
3327         int err;
3328
3329         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
3330                 for (tc = 0; tc < priv->profile->max_tc; tc++) {
3331                         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
3332                         void *tisc;
3333
3334                         tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3335
3336                         MLX5_SET(tisc, tisc, prio, tc << 1);
3337
3338                         if (mlx5e_lag_should_assign_affinity(priv->mdev))
3339                                 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1);
3340
3341                         err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]);
3342                         if (err)
3343                                 goto err_close_tises;
3344                 }
3345         }
3346
3347         return 0;
3348
3349 err_close_tises:
3350         for (; i >= 0; i--) {
3351                 for (tc--; tc >= 0; tc--)
3352                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3353                 tc = priv->profile->max_tc;
3354         }
3355
3356         return err;
3357 }
3358
3359 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3360 {
3361         if (priv->mqprio_rl) {
3362                 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3363                 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3364                 priv->mqprio_rl = NULL;
3365         }
3366         mlx5e_accel_cleanup_tx(priv);
3367         mlx5e_destroy_tises(priv);
3368 }
3369
3370 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3371 {
3372         int err;
3373         int i;
3374
3375         for (i = 0; i < chs->num; i++) {
3376                 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3377                 if (err)
3378                         return err;
3379         }
3380         if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state))
3381                 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd);
3382
3383         return 0;
3384 }
3385
3386 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3387                                                  int ntc, int nch)
3388 {
3389         int tc;
3390
3391         memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE);
3392
3393         /* Map netdev TCs to offset 0.
3394          * We have our own UP to TXQ mapping for DCB mode of QoS
3395          */
3396         for (tc = 0; tc < ntc; tc++) {
3397                 tc_to_txq[tc] = (struct netdev_tc_txq) {
3398                         .count = nch,
3399                         .offset = 0,
3400                 };
3401         }
3402 }
3403
3404 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3405                                          struct tc_mqprio_qopt *qopt)
3406 {
3407         int tc;
3408
3409         for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3410                 tc_to_txq[tc] = (struct netdev_tc_txq) {
3411                         .count = qopt->count[tc],
3412                         .offset = qopt->offset[tc],
3413                 };
3414         }
3415 }
3416
3417 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc)
3418 {
3419         params->mqprio.mode = TC_MQPRIO_MODE_DCB;
3420         params->mqprio.num_tc = num_tc;
3421         mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc,
3422                                              params->num_channels);
3423 }
3424
3425 static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params,
3426                                           struct mlx5e_mqprio_rl *rl)
3427 {
3428         int tc;
3429
3430         for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3431                 u32 hw_id = 0;
3432
3433                 if (rl)
3434                         mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id);
3435                 params->mqprio.channel.hw_id[tc] = hw_id;
3436         }
3437 }
3438
3439 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params,
3440                                             struct tc_mqprio_qopt_offload *mqprio,
3441                                             struct mlx5e_mqprio_rl *rl)
3442 {
3443         int tc;
3444
3445         params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL;
3446         params->mqprio.num_tc = mqprio->qopt.num_tc;
3447
3448         for (tc = 0; tc < TC_MAX_QUEUE; tc++)
3449                 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc];
3450
3451         mlx5e_mqprio_rl_update_params(params, rl);
3452         mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt);
3453 }
3454
3455 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params)
3456 {
3457         mlx5e_params_mqprio_dcb_set(params, 1);
3458 }
3459
3460 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv,
3461                                      struct tc_mqprio_qopt *mqprio)
3462 {
3463         struct mlx5e_params new_params;
3464         u8 tc = mqprio->num_tc;
3465         int err;
3466
3467         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3468
3469         if (tc && tc != MLX5E_MAX_NUM_TC)
3470                 return -EINVAL;
3471
3472         new_params = priv->channels.params;
3473         mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1);
3474
3475         err = mlx5e_safe_switch_params(priv, &new_params,
3476                                        mlx5e_num_channels_changed_ctx, NULL, true);
3477
3478         if (!err && priv->mqprio_rl) {
3479                 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3480                 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3481                 priv->mqprio_rl = NULL;
3482         }
3483
3484         priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3485                                     mlx5e_get_dcb_num_tc(&priv->channels.params));
3486         return err;
3487 }
3488
3489 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv,
3490                                          struct tc_mqprio_qopt_offload *mqprio)
3491 {
3492         struct net_device *netdev = priv->netdev;
3493         struct mlx5e_ptp *ptp_channel;
3494         int agg_count = 0;
3495         int i;
3496
3497         ptp_channel = priv->channels.ptp;
3498         if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) {
3499                 netdev_err(netdev,
3500                            "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n");
3501                 return -EINVAL;
3502         }
3503
3504         if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 ||
3505             mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC)
3506                 return -EINVAL;
3507
3508         for (i = 0; i < mqprio->qopt.num_tc; i++) {
3509                 if (!mqprio->qopt.count[i]) {
3510                         netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i);
3511                         return -EINVAL;
3512                 }
3513                 if (mqprio->min_rate[i]) {
3514                         netdev_err(netdev, "Min tx rate is not supported\n");
3515                         return -EINVAL;
3516                 }
3517
3518                 if (mqprio->max_rate[i]) {
3519                         int err;
3520
3521                         err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]);
3522                         if (err)
3523                                 return err;
3524                 }
3525
3526                 if (mqprio->qopt.offset[i] != agg_count) {
3527                         netdev_err(netdev, "Discontinuous queues config is not supported\n");
3528                         return -EINVAL;
3529                 }
3530                 agg_count += mqprio->qopt.count[i];
3531         }
3532
3533         if (priv->channels.params.num_channels != agg_count) {
3534                 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n",
3535                            agg_count, priv->channels.params.num_channels);
3536                 return -EINVAL;
3537         }
3538
3539         return 0;
3540 }
3541
3542 static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[])
3543 {
3544         int tc;
3545
3546         for (tc = 0; tc < num_tc; tc++)
3547                 if (max_rate[tc])
3548                         return true;
3549         return false;
3550 }
3551
3552 static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev,
3553                                                       u8 num_tc, u64 max_rate[])
3554 {
3555         struct mlx5e_mqprio_rl *rl;
3556         int err;
3557
3558         if (!mlx5e_mqprio_rate_limit(num_tc, max_rate))
3559                 return NULL;
3560
3561         rl = mlx5e_mqprio_rl_alloc();
3562         if (!rl)
3563                 return ERR_PTR(-ENOMEM);
3564
3565         err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate);
3566         if (err) {
3567                 mlx5e_mqprio_rl_free(rl);
3568                 return ERR_PTR(err);
3569         }
3570
3571         return rl;
3572 }
3573
3574 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv,
3575                                          struct tc_mqprio_qopt_offload *mqprio)
3576 {
3577         mlx5e_fp_preactivate preactivate;
3578         struct mlx5e_params new_params;
3579         struct mlx5e_mqprio_rl *rl;
3580         bool nch_changed;
3581         int err;
3582
3583         err = mlx5e_mqprio_channel_validate(priv, mqprio);
3584         if (err)
3585                 return err;
3586
3587         rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate);
3588         if (IS_ERR(rl))
3589                 return PTR_ERR(rl);
3590
3591         new_params = priv->channels.params;
3592         mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl);
3593
3594         nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1;
3595         preactivate = nch_changed ? mlx5e_num_channels_changed_ctx :
3596                 mlx5e_update_netdev_queues_ctx;
3597         err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true);
3598         if (err) {
3599                 if (rl) {
3600                         mlx5e_mqprio_rl_cleanup(rl);
3601                         mlx5e_mqprio_rl_free(rl);
3602                 }
3603                 return err;
3604         }
3605
3606         if (priv->mqprio_rl) {
3607                 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3608                 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3609         }
3610         priv->mqprio_rl = rl;
3611
3612         return 0;
3613 }
3614
3615 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3616                                  struct tc_mqprio_qopt_offload *mqprio)
3617 {
3618         /* MQPRIO is another toplevel qdisc that can't be attached
3619          * simultaneously with the offloaded HTB.
3620          */
3621         if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq)))
3622                 return -EINVAL;
3623
3624         switch (mqprio->mode) {
3625         case TC_MQPRIO_MODE_DCB:
3626                 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt);
3627         case TC_MQPRIO_MODE_CHANNEL:
3628                 return mlx5e_setup_tc_mqprio_channel(priv, mqprio);
3629         default:
3630                 return -EOPNOTSUPP;
3631         }
3632 }
3633
3634 static LIST_HEAD(mlx5e_block_cb_list);
3635
3636 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3637                           void *type_data)
3638 {
3639         struct mlx5e_priv *priv = netdev_priv(dev);
3640         bool tc_unbind = false;
3641         int err;
3642
3643         if (type == TC_SETUP_BLOCK &&
3644             ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND)
3645                 tc_unbind = true;
3646
3647         if (!netif_device_present(dev) && !tc_unbind)
3648                 return -ENODEV;
3649
3650         switch (type) {
3651         case TC_SETUP_BLOCK: {
3652                 struct flow_block_offload *f = type_data;
3653
3654                 f->unlocked_driver_cb = true;
3655                 return flow_block_cb_setup_simple(type_data,
3656                                                   &mlx5e_block_cb_list,
3657                                                   mlx5e_setup_tc_block_cb,
3658                                                   priv, priv, true);
3659         }
3660         case TC_SETUP_QDISC_MQPRIO:
3661                 mutex_lock(&priv->state_lock);
3662                 err = mlx5e_setup_tc_mqprio(priv, type_data);
3663                 mutex_unlock(&priv->state_lock);
3664                 return err;
3665         case TC_SETUP_QDISC_HTB:
3666                 mutex_lock(&priv->state_lock);
3667                 err = mlx5e_htb_setup_tc(priv, type_data);
3668                 mutex_unlock(&priv->state_lock);
3669                 return err;
3670         default:
3671                 return -EOPNOTSUPP;
3672         }
3673 }
3674
3675 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
3676 {
3677         int i;
3678
3679         for (i = 0; i < priv->stats_nch; i++) {
3680                 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
3681                 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3682                 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3683                 int j;
3684
3685                 s->rx_packets   += rq_stats->packets + xskrq_stats->packets;
3686                 s->rx_bytes     += rq_stats->bytes + xskrq_stats->bytes;
3687                 s->multicast    += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
3688
3689                 for (j = 0; j < priv->max_opened_tc; j++) {
3690                         struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3691
3692                         s->tx_packets    += sq_stats->packets;
3693                         s->tx_bytes      += sq_stats->bytes;
3694                         s->tx_dropped    += sq_stats->dropped;
3695                 }
3696         }
3697         if (priv->tx_ptp_opened) {
3698                 for (i = 0; i < priv->max_opened_tc; i++) {
3699                         struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i];
3700
3701                         s->tx_packets    += sq_stats->packets;
3702                         s->tx_bytes      += sq_stats->bytes;
3703                         s->tx_dropped    += sq_stats->dropped;
3704                 }
3705         }
3706         if (priv->rx_ptp_opened) {
3707                 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
3708
3709                 s->rx_packets   += rq_stats->packets;
3710                 s->rx_bytes     += rq_stats->bytes;
3711                 s->multicast    += rq_stats->mcast_packets;
3712         }
3713 }
3714
3715 void
3716 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3717 {
3718         struct mlx5e_priv *priv = netdev_priv(dev);
3719         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3720
3721         if (!netif_device_present(dev))
3722                 return;
3723
3724         /* In switchdev mode, monitor counters doesn't monitor
3725          * rx/tx stats of 802_3. The update stats mechanism
3726          * should keep the 802_3 layout counters updated
3727          */
3728         if (!mlx5e_monitor_counter_supported(priv) ||
3729             mlx5e_is_uplink_rep(priv)) {
3730                 /* update HW stats in background for next time */
3731                 mlx5e_queue_update_stats(priv);
3732         }
3733
3734         if (mlx5e_is_uplink_rep(priv)) {
3735                 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3736
3737                 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3738                 stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
3739                 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3740                 stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3741
3742                 /* vport multicast also counts packets that are dropped due to steering
3743                  * or rx out of buffer
3744                  */
3745                 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
3746         } else {
3747                 mlx5e_fold_sw_stats64(priv, stats);
3748         }
3749
3750         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3751
3752         stats->rx_length_errors =
3753                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3754                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3755                 PPORT_802_3_GET(pstats, a_frame_too_long_errors) +
3756                 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small);
3757         stats->rx_crc_errors =
3758                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3759         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3760         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3761         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3762                            stats->rx_frame_errors;
3763         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3764 }
3765
3766 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
3767 {
3768         if (mlx5e_is_uplink_rep(priv))
3769                 return; /* no rx mode for uplink rep */
3770
3771         queue_work(priv->wq, &priv->set_rx_mode_work);
3772 }
3773
3774 static void mlx5e_set_rx_mode(struct net_device *dev)
3775 {
3776         struct mlx5e_priv *priv = netdev_priv(dev);
3777
3778         mlx5e_nic_set_rx_mode(priv);
3779 }
3780
3781 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3782 {
3783         struct mlx5e_priv *priv = netdev_priv(netdev);
3784         struct sockaddr *saddr = addr;
3785
3786         if (!is_valid_ether_addr(saddr->sa_data))
3787                 return -EADDRNOTAVAIL;
3788
3789         netif_addr_lock_bh(netdev);
3790         eth_hw_addr_set(netdev, saddr->sa_data);
3791         netif_addr_unlock_bh(netdev);
3792
3793         mlx5e_nic_set_rx_mode(priv);
3794
3795         return 0;
3796 }
3797
3798 #define MLX5E_SET_FEATURE(features, feature, enable)    \
3799         do {                                            \
3800                 if (enable)                             \
3801                         *features |= feature;           \
3802                 else                                    \
3803                         *features &= ~feature;          \
3804         } while (0)
3805
3806 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3807
3808 static int set_feature_lro(struct net_device *netdev, bool enable)
3809 {
3810         struct mlx5e_priv *priv = netdev_priv(netdev);
3811         struct mlx5_core_dev *mdev = priv->mdev;
3812         struct mlx5e_params *cur_params;
3813         struct mlx5e_params new_params;
3814         bool reset = true;
3815         int err = 0;
3816
3817         mutex_lock(&priv->state_lock);
3818
3819         cur_params = &priv->channels.params;
3820         new_params = *cur_params;
3821
3822         if (enable)
3823                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO;
3824         else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)
3825                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3826         else
3827                 goto out;
3828
3829         if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO &&
3830               new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) {
3831                 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3832                         if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
3833                             mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
3834                                 reset = false;
3835                 }
3836         }
3837
3838         err = mlx5e_safe_switch_params(priv, &new_params,
3839                                        mlx5e_modify_tirs_packet_merge_ctx, NULL, reset);
3840 out:
3841         mutex_unlock(&priv->state_lock);
3842         return err;
3843 }
3844
3845 static int set_feature_hw_gro(struct net_device *netdev, bool enable)
3846 {
3847         struct mlx5e_priv *priv = netdev_priv(netdev);
3848         struct mlx5e_params new_params;
3849         bool reset = true;
3850         int err = 0;
3851
3852         mutex_lock(&priv->state_lock);
3853         new_params = priv->channels.params;
3854
3855         if (enable) {
3856                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO;
3857                 new_params.packet_merge.shampo.match_criteria_type =
3858                         MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED;
3859                 new_params.packet_merge.shampo.alignment_granularity =
3860                         MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE;
3861         } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
3862                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3863         } else {
3864                 goto out;
3865         }
3866
3867         err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
3868 out:
3869         mutex_unlock(&priv->state_lock);
3870         return err;
3871 }
3872
3873 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3874 {
3875         struct mlx5e_priv *priv = netdev_priv(netdev);
3876
3877         if (enable)
3878                 mlx5e_enable_cvlan_filter(priv->fs,
3879                                           !!(priv->netdev->flags & IFF_PROMISC));
3880         else
3881                 mlx5e_disable_cvlan_filter(priv->fs,
3882                                            !!(priv->netdev->flags & IFF_PROMISC));
3883
3884         return 0;
3885 }
3886
3887 static int set_feature_hw_tc(struct net_device *netdev, bool enable)
3888 {
3889         struct mlx5e_priv *priv = netdev_priv(netdev);
3890         int err = 0;
3891
3892 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3893         int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) :
3894                                                   MLX5_TC_FLAG(NIC_OFFLOAD);
3895         if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) {
3896                 netdev_err(netdev,
3897                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3898                 return -EINVAL;
3899         }
3900 #endif
3901
3902         mutex_lock(&priv->state_lock);
3903         if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) {
3904                 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
3905                 err = -EINVAL;
3906         }
3907         mutex_unlock(&priv->state_lock);
3908
3909         return err;
3910 }
3911
3912 static int set_feature_rx_all(struct net_device *netdev, bool enable)
3913 {
3914         struct mlx5e_priv *priv = netdev_priv(netdev);
3915         struct mlx5_core_dev *mdev = priv->mdev;
3916
3917         return mlx5_set_port_fcs(mdev, !enable);
3918 }
3919
3920 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable)
3921 {
3922         u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {};
3923         bool supported, curr_state;
3924         int err;
3925
3926         if (!MLX5_CAP_GEN(mdev, ports_check))
3927                 return 0;
3928
3929         err = mlx5_query_ports_check(mdev, in, sizeof(in));
3930         if (err)
3931                 return err;
3932
3933         supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap);
3934         curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc);
3935
3936         if (!supported || enable == curr_state)
3937                 return 0;
3938
3939         MLX5_SET(pcmr_reg, in, local_port, 1);
3940         MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable);
3941
3942         return mlx5_set_ports_check(mdev, in, sizeof(in));
3943 }
3944
3945 static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx)
3946 {
3947         struct mlx5_core_dev *mdev = priv->mdev;
3948         bool enable = *(bool *)ctx;
3949
3950         return mlx5e_set_rx_port_ts(mdev, enable);
3951 }
3952
3953 static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3954 {
3955         struct mlx5e_priv *priv = netdev_priv(netdev);
3956         struct mlx5e_channels *chs = &priv->channels;
3957         struct mlx5e_params new_params;
3958         int err;
3959
3960         mutex_lock(&priv->state_lock);
3961
3962         new_params = chs->params;
3963         new_params.scatter_fcs_en = enable;
3964         err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap,
3965                                        &new_params.scatter_fcs_en, true);
3966         mutex_unlock(&priv->state_lock);
3967         return err;
3968 }
3969
3970 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3971 {
3972         struct mlx5e_priv *priv = netdev_priv(netdev);
3973         int err = 0;
3974
3975         mutex_lock(&priv->state_lock);
3976
3977         mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable);
3978         priv->channels.params.vlan_strip_disable = !enable;
3979
3980         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3981                 goto unlock;
3982
3983         err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
3984         if (err) {
3985                 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable);
3986                 priv->channels.params.vlan_strip_disable = enable;
3987         }
3988 unlock:
3989         mutex_unlock(&priv->state_lock);
3990
3991         return err;
3992 }
3993
3994 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
3995 {
3996         struct mlx5e_priv *priv = netdev_priv(dev);
3997         struct mlx5e_flow_steering *fs = priv->fs;
3998
3999         if (mlx5e_is_uplink_rep(priv))
4000                 return 0; /* no vlan table for uplink rep */
4001
4002         return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid);
4003 }
4004
4005 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
4006 {
4007         struct mlx5e_priv *priv = netdev_priv(dev);
4008         struct mlx5e_flow_steering *fs = priv->fs;
4009
4010         if (mlx5e_is_uplink_rep(priv))
4011                 return 0; /* no vlan table for uplink rep */
4012
4013         return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid);
4014 }
4015
4016 #ifdef CONFIG_MLX5_EN_ARFS
4017 static int set_feature_arfs(struct net_device *netdev, bool enable)
4018 {
4019         struct mlx5e_priv *priv = netdev_priv(netdev);
4020         int err;
4021
4022         if (enable)
4023                 err = mlx5e_arfs_enable(priv->fs);
4024         else
4025                 err = mlx5e_arfs_disable(priv->fs);
4026
4027         return err;
4028 }
4029 #endif
4030
4031 static int mlx5e_handle_feature(struct net_device *netdev,
4032                                 netdev_features_t *features,
4033                                 netdev_features_t feature,
4034                                 mlx5e_feature_handler feature_handler)
4035 {
4036         netdev_features_t changes = *features ^ netdev->features;
4037         bool enable = !!(*features & feature);
4038         int err;
4039
4040         if (!(changes & feature))
4041                 return 0;
4042
4043         err = feature_handler(netdev, enable);
4044         if (err) {
4045                 MLX5E_SET_FEATURE(features, feature, !enable);
4046                 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
4047                            enable ? "Enable" : "Disable", &feature, err);
4048                 return err;
4049         }
4050
4051         return 0;
4052 }
4053
4054 void mlx5e_set_xdp_feature(struct net_device *netdev)
4055 {
4056         struct mlx5e_priv *priv = netdev_priv(netdev);
4057         struct mlx5e_params *params = &priv->channels.params;
4058         xdp_features_t val;
4059
4060         if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
4061                 xdp_clear_features_flag(netdev);
4062                 return;
4063         }
4064
4065         val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
4066               NETDEV_XDP_ACT_XSK_ZEROCOPY |
4067               NETDEV_XDP_ACT_RX_SG |
4068               NETDEV_XDP_ACT_NDO_XMIT |
4069               NETDEV_XDP_ACT_NDO_XMIT_SG;
4070         xdp_set_features_flag(netdev, val);
4071 }
4072
4073 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
4074 {
4075         netdev_features_t oper_features = features;
4076         int err = 0;
4077
4078 #define MLX5E_HANDLE_FEATURE(feature, handler) \
4079         mlx5e_handle_feature(netdev, &oper_features, feature, handler)
4080
4081         err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
4082         err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro);
4083         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
4084                                     set_feature_cvlan_filter);
4085         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc);
4086         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
4087         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
4088         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
4089 #ifdef CONFIG_MLX5_EN_ARFS
4090         err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
4091 #endif
4092         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx);
4093
4094         if (err) {
4095                 netdev->features = oper_features;
4096                 return -EINVAL;
4097         }
4098
4099         /* update XDP supported features */
4100         mlx5e_set_xdp_feature(netdev);
4101
4102         return 0;
4103 }
4104
4105 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev,
4106                                                        netdev_features_t features)
4107 {
4108         features &= ~NETIF_F_HW_TLS_RX;
4109         if (netdev->features & NETIF_F_HW_TLS_RX)
4110                 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n");
4111
4112         features &= ~NETIF_F_HW_TLS_TX;
4113         if (netdev->features & NETIF_F_HW_TLS_TX)
4114                 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n");
4115
4116         features &= ~NETIF_F_NTUPLE;
4117         if (netdev->features & NETIF_F_NTUPLE)
4118                 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n");
4119
4120         features &= ~NETIF_F_GRO_HW;
4121         if (netdev->features & NETIF_F_GRO_HW)
4122                 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n");
4123
4124         features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4125         if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)
4126                 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n");
4127
4128         return features;
4129 }
4130
4131 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
4132                                             netdev_features_t features)
4133 {
4134         struct mlx5e_priv *priv = netdev_priv(netdev);
4135         struct mlx5e_vlan_table *vlan;
4136         struct mlx5e_params *params;
4137
4138         if (!netif_device_present(netdev))
4139                 return features;
4140
4141         vlan = mlx5e_fs_get_vlan(priv->fs);
4142         mutex_lock(&priv->state_lock);
4143         params = &priv->channels.params;
4144         if (!vlan ||
4145             !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) {
4146                 /* HW strips the outer C-tag header, this is a problem
4147                  * for S-tag traffic.
4148                  */
4149                 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4150                 if (!params->vlan_strip_disable)
4151                         netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
4152         }
4153
4154         if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
4155                 if (features & NETIF_F_LRO) {
4156                         netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
4157                         features &= ~NETIF_F_LRO;
4158                 }
4159                 if (features & NETIF_F_GRO_HW) {
4160                         netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n");
4161                         features &= ~NETIF_F_GRO_HW;
4162                 }
4163         }
4164
4165         if (params->xdp_prog) {
4166                 if (features & NETIF_F_LRO) {
4167                         netdev_warn(netdev, "LRO is incompatible with XDP\n");
4168                         features &= ~NETIF_F_LRO;
4169                 }
4170                 if (features & NETIF_F_GRO_HW) {
4171                         netdev_warn(netdev, "HW GRO is incompatible with XDP\n");
4172                         features &= ~NETIF_F_GRO_HW;
4173                 }
4174         }
4175
4176         if (priv->xsk.refcnt) {
4177                 if (features & NETIF_F_LRO) {
4178                         netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
4179                                     priv->xsk.refcnt);
4180                         features &= ~NETIF_F_LRO;
4181                 }
4182                 if (features & NETIF_F_GRO_HW) {
4183                         netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n",
4184                                     priv->xsk.refcnt);
4185                         features &= ~NETIF_F_GRO_HW;
4186                 }
4187         }
4188
4189         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
4190                 features &= ~NETIF_F_RXHASH;
4191                 if (netdev->features & NETIF_F_RXHASH)
4192                         netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n");
4193
4194                 if (features & NETIF_F_GRO_HW) {
4195                         netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n");
4196                         features &= ~NETIF_F_GRO_HW;
4197                 }
4198         }
4199
4200         if (mlx5e_is_uplink_rep(priv)) {
4201                 features = mlx5e_fix_uplink_rep_features(netdev, features);
4202                 features |= NETIF_F_NETNS_LOCAL;
4203         } else {
4204                 features &= ~NETIF_F_NETNS_LOCAL;
4205         }
4206
4207         mutex_unlock(&priv->state_lock);
4208
4209         return features;
4210 }
4211
4212 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
4213                                    struct mlx5e_channels *chs,
4214                                    struct mlx5e_params *new_params,
4215                                    struct mlx5_core_dev *mdev)
4216 {
4217         u16 ix;
4218
4219         for (ix = 0; ix < chs->params.num_channels; ix++) {
4220                 struct xsk_buff_pool *xsk_pool =
4221                         mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
4222                 struct mlx5e_xsk_param xsk;
4223                 int max_xdp_mtu;
4224
4225                 if (!xsk_pool)
4226                         continue;
4227
4228                 mlx5e_build_xsk_param(xsk_pool, &xsk);
4229                 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk);
4230
4231                 /* Validate XSK params and XDP MTU in advance */
4232                 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) ||
4233                     new_params->sw_mtu > max_xdp_mtu) {
4234                         u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
4235                         int max_mtu_frame, max_mtu_page, max_mtu;
4236
4237                         /* Two criteria must be met:
4238                          * 1. HW MTU + all headrooms <= XSK frame size.
4239                          * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
4240                          */
4241                         max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
4242                         max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0));
4243                         max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu);
4244
4245                         netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n",
4246                                    new_params->sw_mtu, ix, max_mtu);
4247                         return false;
4248                 }
4249         }
4250
4251         return true;
4252 }
4253
4254 static bool mlx5e_params_validate_xdp(struct net_device *netdev,
4255                                       struct mlx5_core_dev *mdev,
4256                                       struct mlx5e_params *params)
4257 {
4258         bool is_linear;
4259
4260         /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4261          * the XDP program.
4262          */
4263         is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ?
4264                 mlx5e_rx_is_linear_skb(mdev, params, NULL) :
4265                 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL);
4266
4267         if (!is_linear) {
4268                 if (!params->xdp_prog->aux->xdp_has_frags) {
4269                         netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n",
4270                                     params->sw_mtu,
4271                                     mlx5e_xdp_max_mtu(params, NULL));
4272                         return false;
4273                 }
4274                 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
4275                     !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) {
4276                         netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
4277                                     params->sw_mtu,
4278                                     mlx5e_xdp_max_mtu(params, NULL));
4279                         return false;
4280                 }
4281         }
4282
4283         return true;
4284 }
4285
4286 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
4287                      mlx5e_fp_preactivate preactivate)
4288 {
4289         struct mlx5e_priv *priv = netdev_priv(netdev);
4290         struct mlx5e_params new_params;
4291         struct mlx5e_params *params;
4292         bool reset = true;
4293         int err = 0;
4294
4295         mutex_lock(&priv->state_lock);
4296
4297         params = &priv->channels.params;
4298
4299         new_params = *params;
4300         new_params.sw_mtu = new_mtu;
4301         err = mlx5e_validate_params(priv->mdev, &new_params);
4302         if (err)
4303                 goto out;
4304
4305         if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev,
4306                                                               &new_params)) {
4307                 err = -EINVAL;
4308                 goto out;
4309         }
4310
4311         if (priv->xsk.refcnt &&
4312             !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
4313                                     &new_params, priv->mdev)) {
4314                 err = -EINVAL;
4315                 goto out;
4316         }
4317
4318         if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
4319                 reset = false;
4320
4321         if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
4322             params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) {
4323                 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL);
4324                 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
4325                                                                   &new_params, NULL);
4326                 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL);
4327                 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL);
4328
4329                 /* Always reset in linear mode - hw_mtu is used in data path.
4330                  * Check that the mode was non-linear and didn't change.
4331                  * If XSK is active, XSK RQs are linear.
4332                  * Reset if the RQ size changed, even if it's non-linear.
4333                  */
4334                 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt &&
4335                     sz_old == sz_new)
4336                         reset = false;
4337         }
4338
4339         err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset);
4340
4341 out:
4342         netdev->mtu = params->sw_mtu;
4343         mutex_unlock(&priv->state_lock);
4344         return err;
4345 }
4346
4347 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
4348 {
4349         return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx);
4350 }
4351
4352 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx)
4353 {
4354         bool set  = *(bool *)ctx;
4355
4356         return mlx5e_ptp_rx_manage_fs(priv, set);
4357 }
4358
4359 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter)
4360 {
4361         bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4362         int err;
4363
4364         if (!rx_filter)
4365                 /* Reset CQE compression to Admin default */
4366                 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false);
4367
4368         if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS))
4369                 return 0;
4370
4371         /* Disable CQE compression */
4372         netdev_warn(priv->netdev, "Disabling RX cqe compression\n");
4373         err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true);
4374         if (err)
4375                 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
4376
4377         return err;
4378 }
4379
4380 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx)
4381 {
4382         struct mlx5e_params new_params;
4383
4384         if (ptp_rx == priv->channels.params.ptp_rx)
4385                 return 0;
4386
4387         new_params = priv->channels.params;
4388         new_params.ptp_rx = ptp_rx;
4389         return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx,
4390                                         &new_params.ptp_rx, true);
4391 }
4392
4393 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
4394 {
4395         struct hwtstamp_config config;
4396         bool rx_cqe_compress_def;
4397         bool ptp_rx;
4398         int err;
4399
4400         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
4401             (mlx5_clock_get_ptp_index(priv->mdev) == -1))
4402                 return -EOPNOTSUPP;
4403
4404         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
4405                 return -EFAULT;
4406
4407         /* TX HW timestamp */
4408         switch (config.tx_type) {
4409         case HWTSTAMP_TX_OFF:
4410         case HWTSTAMP_TX_ON:
4411                 break;
4412         default:
4413                 return -ERANGE;
4414         }
4415
4416         mutex_lock(&priv->state_lock);
4417         rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4418
4419         /* RX HW timestamp */
4420         switch (config.rx_filter) {
4421         case HWTSTAMP_FILTER_NONE:
4422                 ptp_rx = false;
4423                 break;
4424         case HWTSTAMP_FILTER_ALL:
4425         case HWTSTAMP_FILTER_SOME:
4426         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4427         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4428         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4429         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4430         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4431         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4432         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4433         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4434         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4435         case HWTSTAMP_FILTER_PTP_V2_EVENT:
4436         case HWTSTAMP_FILTER_PTP_V2_SYNC:
4437         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4438         case HWTSTAMP_FILTER_NTP_ALL:
4439                 config.rx_filter = HWTSTAMP_FILTER_ALL;
4440                 /* ptp_rx is set if both HW TS is set and CQE
4441                  * compression is set
4442                  */
4443                 ptp_rx = rx_cqe_compress_def;
4444                 break;
4445         default:
4446                 err = -ERANGE;
4447                 goto err_unlock;
4448         }
4449
4450         if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
4451                 err = mlx5e_hwstamp_config_no_ptp_rx(priv,
4452                                                      config.rx_filter != HWTSTAMP_FILTER_NONE);
4453         else
4454                 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx);
4455         if (err)
4456                 goto err_unlock;
4457
4458         memcpy(&priv->tstamp, &config, sizeof(config));
4459         mutex_unlock(&priv->state_lock);
4460
4461         /* might need to fix some features */
4462         netdev_update_features(priv->netdev);
4463
4464         return copy_to_user(ifr->ifr_data, &config,
4465                             sizeof(config)) ? -EFAULT : 0;
4466 err_unlock:
4467         mutex_unlock(&priv->state_lock);
4468         return err;
4469 }
4470
4471 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
4472 {
4473         struct hwtstamp_config *cfg = &priv->tstamp;
4474
4475         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
4476                 return -EOPNOTSUPP;
4477
4478         return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
4479 }
4480
4481 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4482 {
4483         struct mlx5e_priv *priv = netdev_priv(dev);
4484
4485         switch (cmd) {
4486         case SIOCSHWTSTAMP:
4487                 return mlx5e_hwstamp_set(priv, ifr);
4488         case SIOCGHWTSTAMP:
4489                 return mlx5e_hwstamp_get(priv, ifr);
4490         default:
4491                 return -EOPNOTSUPP;
4492         }
4493 }
4494
4495 #ifdef CONFIG_MLX5_ESWITCH
4496 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
4497 {
4498         struct mlx5e_priv *priv = netdev_priv(dev);
4499         struct mlx5_core_dev *mdev = priv->mdev;
4500
4501         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
4502 }
4503
4504 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
4505                              __be16 vlan_proto)
4506 {
4507         struct mlx5e_priv *priv = netdev_priv(dev);
4508         struct mlx5_core_dev *mdev = priv->mdev;
4509
4510         if (vlan_proto != htons(ETH_P_8021Q))
4511                 return -EPROTONOSUPPORT;
4512
4513         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
4514                                            vlan, qos);
4515 }
4516
4517 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
4518 {
4519         struct mlx5e_priv *priv = netdev_priv(dev);
4520         struct mlx5_core_dev *mdev = priv->mdev;
4521
4522         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
4523 }
4524
4525 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
4526 {
4527         struct mlx5e_priv *priv = netdev_priv(dev);
4528         struct mlx5_core_dev *mdev = priv->mdev;
4529
4530         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
4531 }
4532
4533 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
4534                       int max_tx_rate)
4535 {
4536         struct mlx5e_priv *priv = netdev_priv(dev);
4537         struct mlx5_core_dev *mdev = priv->mdev;
4538
4539         return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
4540                                            max_tx_rate, min_tx_rate);
4541 }
4542
4543 static int mlx5_vport_link2ifla(u8 esw_link)
4544 {
4545         switch (esw_link) {
4546         case MLX5_VPORT_ADMIN_STATE_DOWN:
4547                 return IFLA_VF_LINK_STATE_DISABLE;
4548         case MLX5_VPORT_ADMIN_STATE_UP:
4549                 return IFLA_VF_LINK_STATE_ENABLE;
4550         }
4551         return IFLA_VF_LINK_STATE_AUTO;
4552 }
4553
4554 static int mlx5_ifla_link2vport(u8 ifla_link)
4555 {
4556         switch (ifla_link) {
4557         case IFLA_VF_LINK_STATE_DISABLE:
4558                 return MLX5_VPORT_ADMIN_STATE_DOWN;
4559         case IFLA_VF_LINK_STATE_ENABLE:
4560                 return MLX5_VPORT_ADMIN_STATE_UP;
4561         }
4562         return MLX5_VPORT_ADMIN_STATE_AUTO;
4563 }
4564
4565 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
4566                                    int link_state)
4567 {
4568         struct mlx5e_priv *priv = netdev_priv(dev);
4569         struct mlx5_core_dev *mdev = priv->mdev;
4570
4571         if (mlx5e_is_uplink_rep(priv))
4572                 return -EOPNOTSUPP;
4573
4574         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
4575                                             mlx5_ifla_link2vport(link_state));
4576 }
4577
4578 int mlx5e_get_vf_config(struct net_device *dev,
4579                         int vf, struct ifla_vf_info *ivi)
4580 {
4581         struct mlx5e_priv *priv = netdev_priv(dev);
4582         struct mlx5_core_dev *mdev = priv->mdev;
4583         int err;
4584
4585         if (!netif_device_present(dev))
4586                 return -EOPNOTSUPP;
4587
4588         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
4589         if (err)
4590                 return err;
4591         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
4592         return 0;
4593 }
4594
4595 int mlx5e_get_vf_stats(struct net_device *dev,
4596                        int vf, struct ifla_vf_stats *vf_stats)
4597 {
4598         struct mlx5e_priv *priv = netdev_priv(dev);
4599         struct mlx5_core_dev *mdev = priv->mdev;
4600
4601         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
4602                                             vf_stats);
4603 }
4604
4605 static bool
4606 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id)
4607 {
4608         struct mlx5e_priv *priv = netdev_priv(dev);
4609
4610         if (!netif_device_present(dev))
4611                 return false;
4612
4613         if (!mlx5e_is_uplink_rep(priv))
4614                 return false;
4615
4616         return mlx5e_rep_has_offload_stats(dev, attr_id);
4617 }
4618
4619 static int
4620 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev,
4621                         void *sp)
4622 {
4623         struct mlx5e_priv *priv = netdev_priv(dev);
4624
4625         if (!mlx5e_is_uplink_rep(priv))
4626                 return -EOPNOTSUPP;
4627
4628         return mlx5e_rep_get_offload_stats(attr_id, dev, sp);
4629 }
4630 #endif
4631
4632 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
4633 {
4634         switch (proto_type) {
4635         case IPPROTO_GRE:
4636                 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
4637         case IPPROTO_IPIP:
4638         case IPPROTO_IPV6:
4639                 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
4640                         MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx));
4641         default:
4642                 return false;
4643         }
4644 }
4645
4646 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
4647                                                            struct sk_buff *skb)
4648 {
4649         switch (skb->inner_protocol) {
4650         case htons(ETH_P_IP):
4651         case htons(ETH_P_IPV6):
4652         case htons(ETH_P_TEB):
4653                 return true;
4654         case htons(ETH_P_MPLS_UC):
4655         case htons(ETH_P_MPLS_MC):
4656                 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
4657         }
4658         return false;
4659 }
4660
4661 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4662                                                      struct sk_buff *skb,
4663                                                      netdev_features_t features)
4664 {
4665         unsigned int offset = 0;
4666         struct udphdr *udph;
4667         u8 proto;
4668         u16 port;
4669
4670         switch (vlan_get_protocol(skb)) {
4671         case htons(ETH_P_IP):
4672                 proto = ip_hdr(skb)->protocol;
4673                 break;
4674         case htons(ETH_P_IPV6):
4675                 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
4676                 break;
4677         default:
4678                 goto out;
4679         }
4680
4681         switch (proto) {
4682         case IPPROTO_GRE:
4683                 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
4684                         return features;
4685                 break;
4686         case IPPROTO_IPIP:
4687         case IPPROTO_IPV6:
4688                 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP))
4689                         return features;
4690                 break;
4691         case IPPROTO_UDP:
4692                 udph = udp_hdr(skb);
4693                 port = be16_to_cpu(udph->dest);
4694
4695                 /* Verify if UDP port is being offloaded by HW */
4696                 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
4697                         return features;
4698
4699 #if IS_ENABLED(CONFIG_GENEVE)
4700                 /* Support Geneve offload for default UDP port */
4701                 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev))
4702                         return features;
4703 #endif
4704                 break;
4705 #ifdef CONFIG_MLX5_EN_IPSEC
4706         case IPPROTO_ESP:
4707                 return mlx5e_ipsec_feature_check(skb, features);
4708 #endif
4709         }
4710
4711 out:
4712         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4713         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4714 }
4715
4716 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4717                                        struct net_device *netdev,
4718                                        netdev_features_t features)
4719 {
4720         struct mlx5e_priv *priv = netdev_priv(netdev);
4721
4722         features = vlan_features_check(skb, features);
4723         features = vxlan_features_check(skb, features);
4724
4725         /* Validate if the tunneled packet is being offloaded by HW */
4726         if (skb->encapsulation &&
4727             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
4728                 return mlx5e_tunnel_features_check(priv, skb, features);
4729
4730         return features;
4731 }
4732
4733 static void mlx5e_tx_timeout_work(struct work_struct *work)
4734 {
4735         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4736                                                tx_timeout_work);
4737         struct net_device *netdev = priv->netdev;
4738         int i;
4739
4740         rtnl_lock();
4741         mutex_lock(&priv->state_lock);
4742
4743         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4744                 goto unlock;
4745
4746         for (i = 0; i < netdev->real_num_tx_queues; i++) {
4747                 struct netdev_queue *dev_queue =
4748                         netdev_get_tx_queue(netdev, i);
4749                 struct mlx5e_txqsq *sq = priv->txq2sq[i];
4750
4751                 if (!netif_xmit_stopped(dev_queue))
4752                         continue;
4753
4754                 if (mlx5e_reporter_tx_timeout(sq))
4755                 /* break if tried to reopened channels */
4756                         break;
4757         }
4758
4759 unlock:
4760         mutex_unlock(&priv->state_lock);
4761         rtnl_unlock();
4762 }
4763
4764 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
4765 {
4766         struct mlx5e_priv *priv = netdev_priv(dev);
4767
4768         netdev_err(dev, "TX timeout detected\n");
4769         queue_work(priv->wq, &priv->tx_timeout_work);
4770 }
4771
4772 static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev,
4773                              struct mlx5e_params *params)
4774 {
4775         if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
4776                 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
4777                 return -EINVAL;
4778         }
4779
4780         if (!mlx5e_params_validate_xdp(netdev, mdev, params))
4781                 return -EINVAL;
4782
4783         return 0;
4784 }
4785
4786 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
4787 {
4788         struct bpf_prog *old_prog;
4789
4790         old_prog = rcu_replace_pointer(rq->xdp_prog, prog,
4791                                        lockdep_is_held(&rq->priv->state_lock));
4792         if (old_prog)
4793                 bpf_prog_put(old_prog);
4794 }
4795
4796 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4797 {
4798         struct mlx5e_priv *priv = netdev_priv(netdev);
4799         struct mlx5e_params new_params;
4800         struct bpf_prog *old_prog;
4801         int err = 0;
4802         bool reset;
4803         int i;
4804
4805         mutex_lock(&priv->state_lock);
4806
4807         new_params = priv->channels.params;
4808         new_params.xdp_prog = prog;
4809
4810         if (prog) {
4811                 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params);
4812                 if (err)
4813                         goto unlock;
4814         }
4815
4816         /* no need for full reset when exchanging programs */
4817         reset = (!priv->channels.params.xdp_prog || !prog);
4818
4819         old_prog = priv->channels.params.xdp_prog;
4820
4821         err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
4822         if (err)
4823                 goto unlock;
4824
4825         if (old_prog)
4826                 bpf_prog_put(old_prog);
4827
4828         if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4829                 goto unlock;
4830
4831         /* exchanging programs w/o reset, we update ref counts on behalf
4832          * of the channels RQs here.
4833          */
4834         bpf_prog_add(prog, priv->channels.num);
4835         for (i = 0; i < priv->channels.num; i++) {
4836                 struct mlx5e_channel *c = priv->channels.c[i];
4837
4838                 mlx5e_rq_replace_xdp_prog(&c->rq, prog);
4839                 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) {
4840                         bpf_prog_inc(prog);
4841                         mlx5e_rq_replace_xdp_prog(&c->xskrq, prog);
4842                 }
4843         }
4844
4845 unlock:
4846         mutex_unlock(&priv->state_lock);
4847
4848         /* Need to fix some features. */
4849         if (!err)
4850                 netdev_update_features(netdev);
4851
4852         return err;
4853 }
4854
4855 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4856 {
4857         switch (xdp->command) {
4858         case XDP_SETUP_PROG:
4859                 return mlx5e_xdp_set(dev, xdp->prog);
4860         case XDP_SETUP_XSK_POOL:
4861                 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
4862                                             xdp->xsk.queue_id);
4863         default:
4864                 return -EINVAL;
4865         }
4866 }
4867
4868 #ifdef CONFIG_MLX5_ESWITCH
4869 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4870                                 struct net_device *dev, u32 filter_mask,
4871                                 int nlflags)
4872 {
4873         struct mlx5e_priv *priv = netdev_priv(dev);
4874         struct mlx5_core_dev *mdev = priv->mdev;
4875         u8 mode, setting;
4876         int err;
4877
4878         err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting);
4879         if (err)
4880                 return err;
4881         mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
4882         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4883                                        mode,
4884                                        0, 0, nlflags, filter_mask, NULL);
4885 }
4886
4887 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4888                                 u16 flags, struct netlink_ext_ack *extack)
4889 {
4890         struct mlx5e_priv *priv = netdev_priv(dev);
4891         struct mlx5_core_dev *mdev = priv->mdev;
4892         struct nlattr *attr, *br_spec;
4893         u16 mode = BRIDGE_MODE_UNDEF;
4894         u8 setting;
4895         int rem;
4896
4897         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4898         if (!br_spec)
4899                 return -EINVAL;
4900
4901         nla_for_each_nested(attr, br_spec, rem) {
4902                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4903                         continue;
4904
4905                 if (nla_len(attr) < sizeof(mode))
4906                         return -EINVAL;
4907
4908                 mode = nla_get_u16(attr);
4909                 if (mode > BRIDGE_MODE_VEPA)
4910                         return -EINVAL;
4911
4912                 break;
4913         }
4914
4915         if (mode == BRIDGE_MODE_UNDEF)
4916                 return -EINVAL;
4917
4918         setting = (mode == BRIDGE_MODE_VEPA) ?  1 : 0;
4919         return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting);
4920 }
4921 #endif
4922
4923 const struct net_device_ops mlx5e_netdev_ops = {
4924         .ndo_open                = mlx5e_open,
4925         .ndo_stop                = mlx5e_close,
4926         .ndo_start_xmit          = mlx5e_xmit,
4927         .ndo_setup_tc            = mlx5e_setup_tc,
4928         .ndo_select_queue        = mlx5e_select_queue,
4929         .ndo_get_stats64         = mlx5e_get_stats,
4930         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
4931         .ndo_set_mac_address     = mlx5e_set_mac,
4932         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
4933         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
4934         .ndo_set_features        = mlx5e_set_features,
4935         .ndo_fix_features        = mlx5e_fix_features,
4936         .ndo_change_mtu          = mlx5e_change_nic_mtu,
4937         .ndo_eth_ioctl            = mlx5e_ioctl,
4938         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
4939         .ndo_features_check      = mlx5e_features_check,
4940         .ndo_tx_timeout          = mlx5e_tx_timeout,
4941         .ndo_bpf                 = mlx5e_xdp,
4942         .ndo_xdp_xmit            = mlx5e_xdp_xmit,
4943         .ndo_xsk_wakeup          = mlx5e_xsk_wakeup,
4944 #ifdef CONFIG_MLX5_EN_ARFS
4945         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
4946 #endif
4947 #ifdef CONFIG_MLX5_ESWITCH
4948         .ndo_bridge_setlink      = mlx5e_bridge_setlink,
4949         .ndo_bridge_getlink      = mlx5e_bridge_getlink,
4950
4951         /* SRIOV E-Switch NDOs */
4952         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
4953         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
4954         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
4955         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
4956         .ndo_set_vf_rate         = mlx5e_set_vf_rate,
4957         .ndo_get_vf_config       = mlx5e_get_vf_config,
4958         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
4959         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
4960         .ndo_has_offload_stats   = mlx5e_has_offload_stats,
4961         .ndo_get_offload_stats   = mlx5e_get_offload_stats,
4962 #endif
4963 };
4964
4965 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
4966 {
4967         int i;
4968
4969         /* The supported periods are organized in ascending order */
4970         for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4971                 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4972                         break;
4973
4974         return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4975 }
4976
4977 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
4978 {
4979         struct mlx5e_params *params = &priv->channels.params;
4980         struct mlx5_core_dev *mdev = priv->mdev;
4981         u8 rx_cq_period_mode;
4982
4983         params->sw_mtu = mtu;
4984         params->hard_mtu = MLX5E_ETH_HARD_MTU;
4985         params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2,
4986                                      priv->max_nch);
4987         mlx5e_params_mqprio_reset(params);
4988
4989         /* SQ */
4990         params->log_sq_size = is_kdump_kernel() ?
4991                 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4992                 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
4993         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4994
4995         /* XDP SQ */
4996         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4997
4998         /* set CQE compression */
4999         params->rx_cqe_compress_def = false;
5000         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
5001             MLX5_CAP_GEN(mdev, vport_group_manager))
5002                 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
5003
5004         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
5005         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false);
5006
5007         /* RQ */
5008         mlx5e_build_rq_params(mdev, params);
5009
5010         params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
5011
5012         /* CQ moderation params */
5013         rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
5014                         MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
5015                         MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
5016         params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
5017         params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
5018         mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
5019         mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
5020
5021         /* TX inline */
5022         mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
5023
5024         /* AF_XDP */
5025         params->xsk = xsk;
5026
5027         /* Do not update netdev->features directly in here
5028          * on mlx5e_attach_netdev() we will call mlx5e_update_features()
5029          * To update netdev->features please modify mlx5e_fix_features()
5030          */
5031 }
5032
5033 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
5034 {
5035         struct mlx5e_priv *priv = netdev_priv(netdev);
5036         u8 addr[ETH_ALEN];
5037
5038         mlx5_query_mac_address(priv->mdev, addr);
5039         if (is_zero_ether_addr(addr) &&
5040             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
5041                 eth_hw_addr_random(netdev);
5042                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
5043                 return;
5044         }
5045
5046         eth_hw_addr_set(netdev, addr);
5047 }
5048
5049 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
5050                                 unsigned int entry, struct udp_tunnel_info *ti)
5051 {
5052         struct mlx5e_priv *priv = netdev_priv(netdev);
5053
5054         return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port));
5055 }
5056
5057 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
5058                                   unsigned int entry, struct udp_tunnel_info *ti)
5059 {
5060         struct mlx5e_priv *priv = netdev_priv(netdev);
5061
5062         return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port));
5063 }
5064
5065 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
5066 {
5067         if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
5068                 return;
5069
5070         priv->nic_info.set_port = mlx5e_vxlan_set_port;
5071         priv->nic_info.unset_port = mlx5e_vxlan_unset_port;
5072         priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
5073                                 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN;
5074         priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN;
5075         /* Don't count the space hard-coded to the IANA port */
5076         priv->nic_info.tables[0].n_entries =
5077                 mlx5_vxlan_max_udp_ports(priv->mdev) - 1;
5078
5079         priv->netdev->udp_tunnel_nic_info = &priv->nic_info;
5080 }
5081
5082 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
5083 {
5084         int tt;
5085
5086         for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
5087                 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt)))
5088                         return true;
5089         }
5090         return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev));
5091 }
5092
5093 static void mlx5e_build_nic_netdev(struct net_device *netdev)
5094 {
5095         struct mlx5e_priv *priv = netdev_priv(netdev);
5096         struct mlx5_core_dev *mdev = priv->mdev;
5097         bool fcs_supported;
5098         bool fcs_enabled;
5099
5100         SET_NETDEV_DEV(netdev, mdev->device);
5101
5102         netdev->netdev_ops = &mlx5e_netdev_ops;
5103         netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops;
5104
5105         mlx5e_dcbnl_build_netdev(netdev);
5106
5107         netdev->watchdog_timeo    = 15 * HZ;
5108
5109         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
5110
5111         netdev->vlan_features    |= NETIF_F_SG;
5112         netdev->vlan_features    |= NETIF_F_HW_CSUM;
5113         netdev->vlan_features    |= NETIF_F_HW_MACSEC;
5114         netdev->vlan_features    |= NETIF_F_GRO;
5115         netdev->vlan_features    |= NETIF_F_TSO;
5116         netdev->vlan_features    |= NETIF_F_TSO6;
5117         netdev->vlan_features    |= NETIF_F_RXCSUM;
5118         netdev->vlan_features    |= NETIF_F_RXHASH;
5119         netdev->vlan_features    |= NETIF_F_GSO_PARTIAL;
5120
5121         netdev->mpls_features    |= NETIF_F_SG;
5122         netdev->mpls_features    |= NETIF_F_HW_CSUM;
5123         netdev->mpls_features    |= NETIF_F_TSO;
5124         netdev->mpls_features    |= NETIF_F_TSO6;
5125
5126         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_TX;
5127         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_RX;
5128
5129         /* Tunneled LRO is not supported in the driver, and the same RQs are
5130          * shared between inner and outer TIRs, so the driver can't disable LRO
5131          * for inner TIRs while having it enabled for outer TIRs. Due to this,
5132          * block LRO altogether if the firmware declares tunneled LRO support.
5133          */
5134         if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
5135             !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
5136             !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
5137             mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT,
5138                                                    MLX5E_MPWRQ_UMR_MODE_ALIGNED))
5139                 netdev->vlan_features    |= NETIF_F_LRO;
5140
5141         netdev->hw_features       = netdev->vlan_features;
5142         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
5143         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
5144         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
5145         netdev->hw_features      |= NETIF_F_HW_VLAN_STAG_TX;
5146
5147         if (mlx5e_tunnel_any_tx_proto_supported(mdev)) {
5148                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
5149                 netdev->hw_enc_features |= NETIF_F_TSO;
5150                 netdev->hw_enc_features |= NETIF_F_TSO6;
5151                 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
5152         }
5153
5154         if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
5155                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
5156                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
5157                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
5158                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
5159                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
5160                 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
5161                                          NETIF_F_GSO_UDP_TUNNEL_CSUM;
5162         }
5163
5164         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
5165                 netdev->hw_features     |= NETIF_F_GSO_GRE |
5166                                            NETIF_F_GSO_GRE_CSUM;
5167                 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
5168                                            NETIF_F_GSO_GRE_CSUM;
5169                 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
5170                                                 NETIF_F_GSO_GRE_CSUM;
5171         }
5172
5173         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
5174                 netdev->hw_features |= NETIF_F_GSO_IPXIP4 |
5175                                        NETIF_F_GSO_IPXIP6;
5176                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 |
5177                                            NETIF_F_GSO_IPXIP6;
5178                 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 |
5179                                                 NETIF_F_GSO_IPXIP6;
5180         }
5181
5182         netdev->gso_partial_features             |= NETIF_F_GSO_UDP_L4;
5183         netdev->hw_features                      |= NETIF_F_GSO_UDP_L4;
5184         netdev->features                         |= NETIF_F_GSO_UDP_L4;
5185
5186         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
5187
5188         if (fcs_supported)
5189                 netdev->hw_features |= NETIF_F_RXALL;
5190
5191         if (MLX5_CAP_ETH(mdev, scatter_fcs))
5192                 netdev->hw_features |= NETIF_F_RXFCS;
5193
5194         if (mlx5_qos_is_supported(mdev))
5195                 netdev->hw_features |= NETIF_F_HW_TC;
5196
5197         netdev->features          = netdev->hw_features;
5198
5199         /* Defaults */
5200         if (fcs_enabled)
5201                 netdev->features  &= ~NETIF_F_RXALL;
5202         netdev->features  &= ~NETIF_F_LRO;
5203         netdev->features  &= ~NETIF_F_GRO_HW;
5204         netdev->features  &= ~NETIF_F_RXFCS;
5205
5206 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
5207         if (FT_CAP(flow_modify_en) &&
5208             FT_CAP(modify_root) &&
5209             FT_CAP(identified_miss_table_mode) &&
5210             FT_CAP(flow_table_modify)) {
5211 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
5212                 netdev->hw_features      |= NETIF_F_HW_TC;
5213 #endif
5214 #ifdef CONFIG_MLX5_EN_ARFS
5215                 netdev->hw_features      |= NETIF_F_NTUPLE;
5216 #endif
5217         }
5218
5219         netdev->features         |= NETIF_F_HIGHDMA;
5220         netdev->features         |= NETIF_F_HW_VLAN_STAG_FILTER;
5221
5222         netdev->priv_flags       |= IFF_UNICAST_FLT;
5223
5224         netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
5225         mlx5e_set_xdp_feature(netdev);
5226         mlx5e_set_netdev_dev_addr(netdev);
5227         mlx5e_macsec_build_netdev(priv);
5228         mlx5e_ipsec_build_netdev(priv);
5229         mlx5e_ktls_build_netdev(priv);
5230 }
5231
5232 void mlx5e_create_q_counters(struct mlx5e_priv *priv)
5233 {
5234         u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
5235         u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
5236         struct mlx5_core_dev *mdev = priv->mdev;
5237         int err;
5238
5239         MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
5240         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5241         if (!err)
5242                 priv->q_counter =
5243                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5244
5245         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5246         if (!err)
5247                 priv->drop_rq_q_counter =
5248                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5249 }
5250
5251 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
5252 {
5253         u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
5254
5255         MLX5_SET(dealloc_q_counter_in, in, opcode,
5256                  MLX5_CMD_OP_DEALLOC_Q_COUNTER);
5257         if (priv->q_counter) {
5258                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5259                          priv->q_counter);
5260                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5261         }
5262
5263         if (priv->drop_rq_q_counter) {
5264                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5265                          priv->drop_rq_q_counter);
5266                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5267         }
5268 }
5269
5270 static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
5271                           struct net_device *netdev)
5272 {
5273         struct mlx5e_priv *priv = netdev_priv(netdev);
5274         struct mlx5e_flow_steering *fs;
5275         int err;
5276
5277         mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu);
5278         mlx5e_vxlan_set_netdev_info(priv);
5279
5280         mlx5e_timestamp_init(priv);
5281
5282         fs = mlx5e_fs_init(priv->profile, mdev,
5283                            !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
5284                            priv->dfs_root);
5285         if (!fs) {
5286                 err = -ENOMEM;
5287                 mlx5_core_err(mdev, "FS initialization failed, %d\n", err);
5288                 return err;
5289         }
5290         priv->fs = fs;
5291
5292         err = mlx5e_ktls_init(priv);
5293         if (err)
5294                 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
5295
5296         mlx5e_health_create_reporters(priv);
5297         /* update XDP supported features */
5298         mlx5e_set_xdp_feature(netdev);
5299
5300         return 0;
5301 }
5302
5303 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
5304 {
5305         mlx5e_health_destroy_reporters(priv);
5306         mlx5e_ktls_cleanup(priv);
5307         mlx5e_fs_cleanup(priv->fs);
5308         priv->fs = NULL;
5309 }
5310
5311 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
5312 {
5313         struct mlx5_core_dev *mdev = priv->mdev;
5314         enum mlx5e_rx_res_features features;
5315         int err;
5316
5317         priv->rx_res = mlx5e_rx_res_alloc();
5318         if (!priv->rx_res)
5319                 return -ENOMEM;
5320
5321         mlx5e_create_q_counters(priv);
5322
5323         err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
5324         if (err) {
5325                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
5326                 goto err_destroy_q_counters;
5327         }
5328
5329         features = MLX5E_RX_RES_FEATURE_PTP;
5330         if (mlx5_tunnel_inner_ft_supported(mdev))
5331                 features |= MLX5E_RX_RES_FEATURE_INNER_FT;
5332         err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features,
5333                                 priv->max_nch, priv->drop_rq.rqn,
5334                                 &priv->channels.params.packet_merge,
5335                                 priv->channels.params.num_channels);
5336         if (err)
5337                 goto err_close_drop_rq;
5338
5339         err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile,
5340                                          priv->netdev);
5341         if (err) {
5342                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
5343                 goto err_destroy_rx_res;
5344         }
5345
5346         err = mlx5e_tc_nic_init(priv);
5347         if (err)
5348                 goto err_destroy_flow_steering;
5349
5350         err = mlx5e_accel_init_rx(priv);
5351         if (err)
5352                 goto err_tc_nic_cleanup;
5353
5354 #ifdef CONFIG_MLX5_EN_ARFS
5355         priv->netdev->rx_cpu_rmap =  mlx5_eq_table_get_rmap(priv->mdev);
5356 #endif
5357
5358         return 0;
5359
5360 err_tc_nic_cleanup:
5361         mlx5e_tc_nic_cleanup(priv);
5362 err_destroy_flow_steering:
5363         mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
5364                                     priv->profile);
5365 err_destroy_rx_res:
5366         mlx5e_rx_res_destroy(priv->rx_res);
5367 err_close_drop_rq:
5368         mlx5e_close_drop_rq(&priv->drop_rq);
5369 err_destroy_q_counters:
5370         mlx5e_destroy_q_counters(priv);
5371         mlx5e_rx_res_free(priv->rx_res);
5372         priv->rx_res = NULL;
5373         return err;
5374 }
5375
5376 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
5377 {
5378         mlx5e_accel_cleanup_rx(priv);
5379         mlx5e_tc_nic_cleanup(priv);
5380         mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
5381                                     priv->profile);
5382         mlx5e_rx_res_destroy(priv->rx_res);
5383         mlx5e_close_drop_rq(&priv->drop_rq);
5384         mlx5e_destroy_q_counters(priv);
5385         mlx5e_rx_res_free(priv->rx_res);
5386         priv->rx_res = NULL;
5387 }
5388
5389 static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv)
5390 {
5391         struct mlx5e_params *params;
5392         struct mlx5e_mqprio_rl *rl;
5393
5394         params = &priv->channels.params;
5395         if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL)
5396                 return;
5397
5398         rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc,
5399                                     params->mqprio.channel.max_rate);
5400         if (IS_ERR(rl))
5401                 rl = NULL;
5402         priv->mqprio_rl = rl;
5403         mlx5e_mqprio_rl_update_params(params, rl);
5404 }
5405
5406 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
5407 {
5408         int err;
5409
5410         err = mlx5e_create_tises(priv);
5411         if (err) {
5412                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
5413                 return err;
5414         }
5415
5416         err = mlx5e_accel_init_tx(priv);
5417         if (err)
5418                 goto err_destroy_tises;
5419
5420         mlx5e_set_mqprio_rl(priv);
5421         mlx5e_dcbnl_initialize(priv);
5422         return 0;
5423
5424 err_destroy_tises:
5425         mlx5e_destroy_tises(priv);
5426         return err;
5427 }
5428
5429 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
5430 {
5431         struct net_device *netdev = priv->netdev;
5432         struct mlx5_core_dev *mdev = priv->mdev;
5433         int err;
5434
5435         mlx5e_fs_init_l2_addr(priv->fs, netdev);
5436         mlx5e_ipsec_init(priv);
5437
5438         err = mlx5e_macsec_init(priv);
5439         if (err)
5440                 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err);
5441
5442         /* Marking the link as currently not needed by the Driver */
5443         if (!netif_running(netdev))
5444                 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
5445
5446         mlx5e_set_netdev_mtu_boundaries(priv);
5447         mlx5e_set_dev_port_mtu(priv);
5448
5449         mlx5_lag_add_netdev(mdev, netdev);
5450
5451         mlx5e_enable_async_events(priv);
5452         mlx5e_enable_blocking_events(priv);
5453         if (mlx5e_monitor_counter_supported(priv))
5454                 mlx5e_monitor_counter_init(priv);
5455
5456         mlx5e_hv_vhca_stats_create(priv);
5457         if (netdev->reg_state != NETREG_REGISTERED)
5458                 return;
5459         mlx5e_dcbnl_init_app(priv);
5460
5461         mlx5e_nic_set_rx_mode(priv);
5462
5463         rtnl_lock();
5464         if (netif_running(netdev))
5465                 mlx5e_open(netdev);
5466         udp_tunnel_nic_reset_ntf(priv->netdev);
5467         netif_device_attach(netdev);
5468         rtnl_unlock();
5469 }
5470
5471 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
5472 {
5473         struct mlx5_core_dev *mdev = priv->mdev;
5474
5475         if (priv->netdev->reg_state == NETREG_REGISTERED)
5476                 mlx5e_dcbnl_delete_app(priv);
5477
5478         rtnl_lock();
5479         if (netif_running(priv->netdev))
5480                 mlx5e_close(priv->netdev);
5481         netif_device_detach(priv->netdev);
5482         rtnl_unlock();
5483
5484         mlx5e_nic_set_rx_mode(priv);
5485
5486         mlx5e_hv_vhca_stats_destroy(priv);
5487         if (mlx5e_monitor_counter_supported(priv))
5488                 mlx5e_monitor_counter_cleanup(priv);
5489
5490         mlx5e_disable_blocking_events(priv);
5491         if (priv->en_trap) {
5492                 mlx5e_deactivate_trap(priv);
5493                 mlx5e_close_trap(priv->en_trap);
5494                 priv->en_trap = NULL;
5495         }
5496         mlx5e_disable_async_events(priv);
5497         mlx5_lag_remove_netdev(mdev, priv->netdev);
5498         mlx5_vxlan_reset_to_default(mdev->vxlan);
5499         mlx5e_macsec_cleanup(priv);
5500         mlx5e_ipsec_cleanup(priv);
5501 }
5502
5503 int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
5504 {
5505         return mlx5e_refresh_tirs(priv, false, false);
5506 }
5507
5508 static const struct mlx5e_profile mlx5e_nic_profile = {
5509         .init              = mlx5e_nic_init,
5510         .cleanup           = mlx5e_nic_cleanup,
5511         .init_rx           = mlx5e_init_nic_rx,
5512         .cleanup_rx        = mlx5e_cleanup_nic_rx,
5513         .init_tx           = mlx5e_init_nic_tx,
5514         .cleanup_tx        = mlx5e_cleanup_nic_tx,
5515         .enable            = mlx5e_nic_enable,
5516         .disable           = mlx5e_nic_disable,
5517         .update_rx         = mlx5e_update_nic_rx,
5518         .update_stats      = mlx5e_stats_update_ndo_stats,
5519         .update_carrier    = mlx5e_update_carrier,
5520         .rx_handlers       = &mlx5e_rx_handlers_nic,
5521         .max_tc            = MLX5E_MAX_NUM_TC,
5522         .stats_grps        = mlx5e_nic_stats_grps,
5523         .stats_grps_num    = mlx5e_nic_stats_grps_num,
5524         .features          = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) |
5525                 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) |
5526                 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) |
5527                 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) |
5528                 BIT(MLX5E_PROFILE_FEATURE_FS_TC),
5529 };
5530
5531 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev,
5532                                           const struct mlx5e_profile *profile)
5533 {
5534         int nch;
5535
5536         nch = mlx5e_get_max_num_channels(mdev);
5537
5538         if (profile->max_nch_limit)
5539                 nch = min_t(int, nch, profile->max_nch_limit(mdev));
5540         return nch;
5541 }
5542
5543 static unsigned int
5544 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
5545                    const struct mlx5e_profile *profile)
5546
5547 {
5548         unsigned int max_nch, tmp;
5549
5550         /* core resources */
5551         max_nch = mlx5e_profile_max_num_channels(mdev, profile);
5552
5553         /* netdev rx queues */
5554         max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues);
5555
5556         /* netdev tx queues */
5557         tmp = netdev->num_tx_queues;
5558         if (mlx5_qos_is_supported(mdev))
5559                 tmp -= mlx5e_qos_max_leaf_nodes(mdev);
5560         if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn))
5561                 tmp -= profile->max_tc;
5562         tmp = tmp / profile->max_tc;
5563         max_nch = min_t(unsigned int, max_nch, tmp);
5564
5565         return max_nch;
5566 }
5567
5568 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev)
5569 {
5570         /* Indirect TIRS: 2 sets of TTCs (inner + outer steering)
5571          * and 1 set of direct TIRS
5572          */
5573         return 2 * MLX5E_NUM_INDIR_TIRS
5574                 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile);
5575 }
5576
5577 void mlx5e_set_rx_mode_work(struct work_struct *work)
5578 {
5579         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
5580                                                set_rx_mode_work);
5581
5582         return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev);
5583 }
5584
5585 /* mlx5e generic netdev management API (move to en_common.c) */
5586 int mlx5e_priv_init(struct mlx5e_priv *priv,
5587                     const struct mlx5e_profile *profile,
5588                     struct net_device *netdev,
5589                     struct mlx5_core_dev *mdev)
5590 {
5591         int nch, num_txqs, node;
5592         int err;
5593
5594         num_txqs = netdev->num_tx_queues;
5595         nch = mlx5e_calc_max_nch(mdev, netdev, profile);
5596         node = dev_to_node(mlx5_core_dma_dev(mdev));
5597
5598         /* priv init */
5599         priv->mdev        = mdev;
5600         priv->netdev      = netdev;
5601         priv->msglevel    = MLX5E_MSG_LEVEL;
5602         priv->max_nch     = nch;
5603         priv->max_opened_tc = 1;
5604
5605         if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL))
5606                 return -ENOMEM;
5607
5608         mutex_init(&priv->state_lock);
5609
5610         err = mlx5e_selq_init(&priv->selq, &priv->state_lock);
5611         if (err)
5612                 goto err_free_cpumask;
5613
5614         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
5615         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
5616         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
5617         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
5618
5619         priv->wq = create_singlethread_workqueue("mlx5e");
5620         if (!priv->wq)
5621                 goto err_free_selq;
5622
5623         priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node);
5624         if (!priv->txq2sq)
5625                 goto err_destroy_workqueue;
5626
5627         priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node);
5628         if (!priv->tx_rates)
5629                 goto err_free_txq2sq;
5630
5631         priv->channel_stats =
5632                 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node);
5633         if (!priv->channel_stats)
5634                 goto err_free_tx_rates;
5635
5636         return 0;
5637
5638 err_free_tx_rates:
5639         kfree(priv->tx_rates);
5640 err_free_txq2sq:
5641         kfree(priv->txq2sq);
5642 err_destroy_workqueue:
5643         destroy_workqueue(priv->wq);
5644 err_free_selq:
5645         mlx5e_selq_cleanup(&priv->selq);
5646 err_free_cpumask:
5647         free_cpumask_var(priv->scratchpad.cpumask);
5648         return -ENOMEM;
5649 }
5650
5651 void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
5652 {
5653         int i;
5654
5655         /* bail if change profile failed and also rollback failed */
5656         if (!priv->mdev)
5657                 return;
5658
5659         for (i = 0; i < priv->stats_nch; i++)
5660                 kvfree(priv->channel_stats[i]);
5661         kfree(priv->channel_stats);
5662         kfree(priv->tx_rates);
5663         kfree(priv->txq2sq);
5664         destroy_workqueue(priv->wq);
5665         mutex_lock(&priv->state_lock);
5666         mlx5e_selq_cleanup(&priv->selq);
5667         mutex_unlock(&priv->state_lock);
5668         free_cpumask_var(priv->scratchpad.cpumask);
5669
5670         for (i = 0; i < priv->htb_max_qos_sqs; i++)
5671                 kfree(priv->htb_qos_sq_stats[i]);
5672         kvfree(priv->htb_qos_sq_stats);
5673
5674         memset(priv, 0, sizeof(*priv));
5675 }
5676
5677 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
5678                                            const struct mlx5e_profile *profile)
5679 {
5680         unsigned int nch, ptp_txqs, qos_txqs;
5681
5682         nch = mlx5e_profile_max_num_channels(mdev, profile);
5683
5684         ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) &&
5685                 mlx5e_profile_feature_cap(profile, PTP_TX) ?
5686                 profile->max_tc : 0;
5687
5688         qos_txqs = mlx5_qos_is_supported(mdev) &&
5689                 mlx5e_profile_feature_cap(profile, QOS_HTB) ?
5690                 mlx5e_qos_max_leaf_nodes(mdev) : 0;
5691
5692         return nch * profile->max_tc + ptp_txqs + qos_txqs;
5693 }
5694
5695 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
5696                                            const struct mlx5e_profile *profile)
5697 {
5698         return mlx5e_profile_max_num_channels(mdev, profile);
5699 }
5700
5701 struct net_device *
5702 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile)
5703 {
5704         struct net_device *netdev;
5705         unsigned int txqs, rxqs;
5706         int err;
5707
5708         txqs = mlx5e_get_max_num_txqs(mdev, profile);
5709         rxqs = mlx5e_get_max_num_rxqs(mdev, profile);
5710
5711         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs);
5712         if (!netdev) {
5713                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
5714                 return NULL;
5715         }
5716
5717         err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev);
5718         if (err) {
5719                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5720                 goto err_free_netdev;
5721         }
5722
5723         netif_carrier_off(netdev);
5724         netif_tx_disable(netdev);
5725         dev_net_set(netdev, mlx5_core_net(mdev));
5726
5727         return netdev;
5728
5729 err_free_netdev:
5730         free_netdev(netdev);
5731
5732         return NULL;
5733 }
5734
5735 static void mlx5e_update_features(struct net_device *netdev)
5736 {
5737         if (netdev->reg_state != NETREG_REGISTERED)
5738                 return; /* features will be updated on netdev registration */
5739
5740         rtnl_lock();
5741         netdev_update_features(netdev);
5742         rtnl_unlock();
5743 }
5744
5745 static void mlx5e_reset_channels(struct net_device *netdev)
5746 {
5747         netdev_reset_tc(netdev);
5748 }
5749
5750 int mlx5e_attach_netdev(struct mlx5e_priv *priv)
5751 {
5752         const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED;
5753         const struct mlx5e_profile *profile = priv->profile;
5754         int max_nch;
5755         int err;
5756
5757         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
5758         if (priv->fs)
5759                 mlx5e_fs_set_state_destroy(priv->fs,
5760                                            !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5761
5762         /* Validate the max_wqe_size_sq capability. */
5763         if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) {
5764                 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n",
5765                                mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS);
5766                 return -EIO;
5767         }
5768
5769         /* max number of channels may have changed */
5770         max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile);
5771         if (priv->channels.params.num_channels > max_nch) {
5772                 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch);
5773                 /* Reducing the number of channels - RXFH has to be reset, and
5774                  * mlx5e_num_channels_changed below will build the RQT.
5775                  */
5776                 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED;
5777                 priv->channels.params.num_channels = max_nch;
5778                 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) {
5779                         mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n");
5780                         mlx5e_params_mqprio_reset(&priv->channels.params);
5781                 }
5782         }
5783         if (max_nch != priv->max_nch) {
5784                 mlx5_core_warn(priv->mdev,
5785                                "MLX5E: Updating max number of channels from %u to %u\n",
5786                                priv->max_nch, max_nch);
5787                 priv->max_nch = max_nch;
5788         }
5789
5790         /* 1. Set the real number of queues in the kernel the first time.
5791          * 2. Set our default XPS cpumask.
5792          * 3. Build the RQT.
5793          *
5794          * rtnl_lock is required by netif_set_real_num_*_queues in case the
5795          * netdev has been registered by this point (if this function was called
5796          * in the reload or resume flow).
5797          */
5798         if (take_rtnl)
5799                 rtnl_lock();
5800         err = mlx5e_num_channels_changed(priv);
5801         if (take_rtnl)
5802                 rtnl_unlock();
5803         if (err)
5804                 goto out;
5805
5806         err = profile->init_tx(priv);
5807         if (err)
5808                 goto out;
5809
5810         err = profile->init_rx(priv);
5811         if (err)
5812                 goto err_cleanup_tx;
5813
5814         if (profile->enable)
5815                 profile->enable(priv);
5816
5817         mlx5e_update_features(priv->netdev);
5818
5819         return 0;
5820
5821 err_cleanup_tx:
5822         profile->cleanup_tx(priv);
5823
5824 out:
5825         mlx5e_reset_channels(priv->netdev);
5826         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5827         if (priv->fs)
5828                 mlx5e_fs_set_state_destroy(priv->fs,
5829                                            !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5830         cancel_work_sync(&priv->update_stats_work);
5831         return err;
5832 }
5833
5834 void mlx5e_detach_netdev(struct mlx5e_priv *priv)
5835 {
5836         const struct mlx5e_profile *profile = priv->profile;
5837
5838         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5839         if (priv->fs)
5840                 mlx5e_fs_set_state_destroy(priv->fs,
5841                                            !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5842
5843         if (profile->disable)
5844                 profile->disable(priv);
5845         flush_workqueue(priv->wq);
5846
5847         profile->cleanup_rx(priv);
5848         profile->cleanup_tx(priv);
5849         mlx5e_reset_channels(priv->netdev);
5850         cancel_work_sync(&priv->update_stats_work);
5851 }
5852
5853 static int
5854 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
5855                             const struct mlx5e_profile *new_profile, void *new_ppriv)
5856 {
5857         struct mlx5e_priv *priv = netdev_priv(netdev);
5858         int err;
5859
5860         err = mlx5e_priv_init(priv, new_profile, netdev, mdev);
5861         if (err) {
5862                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5863                 return err;
5864         }
5865         netif_carrier_off(netdev);
5866         priv->profile = new_profile;
5867         priv->ppriv = new_ppriv;
5868         err = new_profile->init(priv->mdev, priv->netdev);
5869         if (err)
5870                 goto priv_cleanup;
5871         err = mlx5e_attach_netdev(priv);
5872         if (err)
5873                 goto profile_cleanup;
5874         return err;
5875
5876 profile_cleanup:
5877         new_profile->cleanup(priv);
5878 priv_cleanup:
5879         mlx5e_priv_cleanup(priv);
5880         return err;
5881 }
5882
5883 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
5884                                 const struct mlx5e_profile *new_profile, void *new_ppriv)
5885 {
5886         const struct mlx5e_profile *orig_profile = priv->profile;
5887         struct net_device *netdev = priv->netdev;
5888         struct mlx5_core_dev *mdev = priv->mdev;
5889         void *orig_ppriv = priv->ppriv;
5890         int err, rollback_err;
5891
5892         /* cleanup old profile */
5893         mlx5e_detach_netdev(priv);
5894         priv->profile->cleanup(priv);
5895         mlx5e_priv_cleanup(priv);
5896
5897         err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv);
5898         if (err) { /* roll back to original profile */
5899                 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err);
5900                 goto rollback;
5901         }
5902
5903         return 0;
5904
5905 rollback:
5906         rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv);
5907         if (rollback_err)
5908                 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n",
5909                            __func__, rollback_err);
5910         return err;
5911 }
5912
5913 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv)
5914 {
5915         mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL);
5916 }
5917
5918 void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
5919 {
5920         struct net_device *netdev = priv->netdev;
5921
5922         mlx5e_priv_cleanup(priv);
5923         free_netdev(netdev);
5924 }
5925
5926 static int mlx5e_resume(struct auxiliary_device *adev)
5927 {
5928         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5929         struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
5930         struct mlx5e_priv *priv = mlx5e_dev->priv;
5931         struct net_device *netdev = priv->netdev;
5932         struct mlx5_core_dev *mdev = edev->mdev;
5933         int err;
5934
5935         if (netif_device_present(netdev))
5936                 return 0;
5937
5938         err = mlx5e_create_mdev_resources(mdev);
5939         if (err)
5940                 return err;
5941
5942         err = mlx5e_attach_netdev(priv);
5943         if (err) {
5944                 mlx5e_destroy_mdev_resources(mdev);
5945                 return err;
5946         }
5947
5948         return 0;
5949 }
5950
5951 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
5952 {
5953         struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
5954         struct mlx5e_priv *priv = mlx5e_dev->priv;
5955         struct net_device *netdev = priv->netdev;
5956         struct mlx5_core_dev *mdev = priv->mdev;
5957
5958         if (!netif_device_present(netdev))
5959                 return -ENODEV;
5960
5961         mlx5e_detach_netdev(priv);
5962         mlx5e_destroy_mdev_resources(mdev);
5963         return 0;
5964 }
5965
5966 static int mlx5e_probe(struct auxiliary_device *adev,
5967                        const struct auxiliary_device_id *id)
5968 {
5969         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5970         const struct mlx5e_profile *profile = &mlx5e_nic_profile;
5971         struct mlx5_core_dev *mdev = edev->mdev;
5972         struct mlx5e_dev *mlx5e_dev;
5973         struct net_device *netdev;
5974         pm_message_t state = {};
5975         struct mlx5e_priv *priv;
5976         int err;
5977
5978         mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev);
5979         if (IS_ERR(mlx5e_dev))
5980                 return PTR_ERR(mlx5e_dev);
5981         auxiliary_set_drvdata(adev, mlx5e_dev);
5982
5983         err = mlx5e_devlink_port_register(mlx5e_dev, mdev);
5984         if (err) {
5985                 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err);
5986                 goto err_devlink_unregister;
5987         }
5988
5989         netdev = mlx5e_create_netdev(mdev, profile);
5990         if (!netdev) {
5991                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
5992                 err = -ENOMEM;
5993                 goto err_devlink_port_unregister;
5994         }
5995         SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port);
5996
5997         mlx5e_build_nic_netdev(netdev);
5998
5999         priv = netdev_priv(netdev);
6000         mlx5e_dev->priv = priv;
6001
6002         priv->profile = profile;
6003         priv->ppriv = NULL;
6004
6005         priv->dfs_root = debugfs_create_dir("nic",
6006                                             mlx5_debugfs_get_dev_root(priv->mdev));
6007
6008         err = profile->init(mdev, netdev);
6009         if (err) {
6010                 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err);
6011                 goto err_destroy_netdev;
6012         }
6013
6014         err = mlx5e_resume(adev);
6015         if (err) {
6016                 mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err);
6017                 goto err_profile_cleanup;
6018         }
6019
6020         err = register_netdev(netdev);
6021         if (err) {
6022                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
6023                 goto err_resume;
6024         }
6025
6026         mlx5e_dcbnl_init_app(priv);
6027         mlx5_core_uplink_netdev_set(mdev, netdev);
6028         mlx5e_params_print_info(mdev, &priv->channels.params);
6029         return 0;
6030
6031 err_resume:
6032         mlx5e_suspend(adev, state);
6033 err_profile_cleanup:
6034         profile->cleanup(priv);
6035 err_destroy_netdev:
6036         debugfs_remove_recursive(priv->dfs_root);
6037         mlx5e_destroy_netdev(priv);
6038 err_devlink_port_unregister:
6039         mlx5e_devlink_port_unregister(mlx5e_dev);
6040 err_devlink_unregister:
6041         mlx5e_destroy_devlink(mlx5e_dev);
6042         return err;
6043 }
6044
6045 static void mlx5e_remove(struct auxiliary_device *adev)
6046 {
6047         struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
6048         struct mlx5e_priv *priv = mlx5e_dev->priv;
6049         pm_message_t state = {};
6050
6051         mlx5_core_uplink_netdev_set(priv->mdev, NULL);
6052         mlx5e_dcbnl_delete_app(priv);
6053         unregister_netdev(priv->netdev);
6054         mlx5e_suspend(adev, state);
6055         priv->profile->cleanup(priv);
6056         debugfs_remove_recursive(priv->dfs_root);
6057         mlx5e_destroy_netdev(priv);
6058         mlx5e_devlink_port_unregister(mlx5e_dev);
6059         mlx5e_destroy_devlink(mlx5e_dev);
6060 }
6061
6062 static const struct auxiliary_device_id mlx5e_id_table[] = {
6063         { .name = MLX5_ADEV_NAME ".eth", },
6064         {},
6065 };
6066
6067 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);
6068
6069 static struct auxiliary_driver mlx5e_driver = {
6070         .name = "eth",
6071         .probe = mlx5e_probe,
6072         .remove = mlx5e_remove,
6073         .suspend = mlx5e_suspend,
6074         .resume = mlx5e_resume,
6075         .id_table = mlx5e_id_table,
6076 };
6077
6078 int mlx5e_init(void)
6079 {
6080         int ret;
6081
6082         mlx5e_build_ptys2ethtool_map();
6083         ret = auxiliary_driver_register(&mlx5e_driver);
6084         if (ret)
6085                 return ret;
6086
6087         ret = mlx5e_rep_init();
6088         if (ret)
6089                 auxiliary_driver_unregister(&mlx5e_driver);
6090         return ret;
6091 }
6092
6093 void mlx5e_cleanup(void)
6094 {
6095         mlx5e_rep_cleanup();
6096         auxiliary_driver_unregister(&mlx5e_driver);
6097 }