drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include <net/geneve.h>
  38 #include <linux/bpf.h>
  39 #include <linux/if_bridge.h>
  40 #include <linux/filter.h>
  41 #include <net/page_pool.h>
  42 #include <net/xdp_sock_drv.h>
  43 #include "eswitch.h"
  44 #include "en.h"
  45 #include "en/txrx.h"
  46 #include "en_tc.h"
  47 #include "en_rep.h"
  48 #include "en_accel/ipsec.h"
  49 #include "en_accel/en_accel.h"
  50 #include "en_accel/tls.h"
  51 #include "accel/ipsec.h"
  52 #include "accel/tls.h"
  53 #include "lib/vxlan.h"
  54 #include "lib/clock.h"
  55 #include "en/port.h"
  56 #include "en/xdp.h"
  57 #include "lib/eq.h"
  58 #include "en/monitor_stats.h"
  59 #include "en/health.h"
  60 #include "en/params.h"
  61 #include "en/xsk/pool.h"
  62 #include "en/xsk/setup.h"
  63 #include "en/xsk/rx.h"
  64 #include "en/xsk/tx.h"
  65 #include "en/hv_vhca_stats.h"
  66 #include "en/devlink.h"
  67 #include "lib/mlx5.h"
  68 #include "en/ptp.h"
  69 #include "qos.h"
  70 #include "en/trap.h"
  71 #include "fpga/ipsec.h"
  72
  73 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  74 {
  75         bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
  76                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  77                 MLX5_CAP_ETH(mdev, reg_umr_sq);
  78         u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
  79         bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
  80
  81         if (!striding_rq_umr)
  82                 return false;
  83         if (!inline_umr) {
  84                 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
  85                                (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
  86                 return false;
  87         }
  88         return true;
  89 }
  90
  91 void mlx5e_update_carrier(struct mlx5e_priv *priv)
  92 {
  93         struct mlx5_core_dev *mdev = priv->mdev;
  94         u8 port_state;
  95         bool up;
  96
  97         port_state = mlx5_query_vport_state(mdev,
  98                                             MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
  99                                             0);
 100
 101         up = port_state == VPORT_STATE_UP;
 102         if (up == netif_carrier_ok(priv->netdev))
 103                 netif_carrier_event(priv->netdev);
 104         if (up) {
 105                 netdev_info(priv->netdev, "Link up\n");
 106                 netif_carrier_on(priv->netdev);
 107         } else {
 108                 netdev_info(priv->netdev, "Link down\n");
 109                 netif_carrier_off(priv->netdev);
 110         }
 111 }
 112
 113 static void mlx5e_update_carrier_work(struct work_struct *work)
 114 {
 115         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 116                                                update_carrier_work);
 117
 118         mutex_lock(&priv->state_lock);
 119         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 120                 if (priv->profile->update_carrier)
 121                         priv->profile->update_carrier(priv);
 122         mutex_unlock(&priv->state_lock);
 123 }
 124
 125 static void mlx5e_update_stats_work(struct work_struct *work)
 126 {
 127         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 128                                                update_stats_work);
 129
 130         mutex_lock(&priv->state_lock);
 131         priv->profile->update_stats(priv);
 132         mutex_unlock(&priv->state_lock);
 133 }
 134
 135 void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
 136 {
 137         if (!priv->profile->update_stats)
 138                 return;
 139
 140         if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
 141                 return;
 142
 143         queue_work(priv->wq, &priv->update_stats_work);
 144 }
 145
 146 static int async_event(struct notifier_block *nb, unsigned long event, void *data)
 147 {
 148         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
 149         struct mlx5_eqe   *eqe = data;
 150
 151         if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
 152                 return NOTIFY_DONE;
 153
 154         switch (eqe->sub_type) {
 155         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
 156         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
 157                 queue_work(priv->wq, &priv->update_carrier_work);
 158                 break;
 159         default:
 160                 return NOTIFY_DONE;
 161         }
 162
 163         return NOTIFY_OK;
 164 }
 165
 166 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 167 {
 168         priv->events_nb.notifier_call = async_event;
 169         mlx5_notifier_register(priv->mdev, &priv->events_nb);
 170 }
 171
 172 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 173 {
 174         mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
 175 }
 176
 177 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
 178 {
 179         struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
 180         int err;
 181
 182         switch (event) {
 183         case MLX5_DRIVER_EVENT_TYPE_TRAP:
 184                 err = mlx5e_handle_trap_event(priv, data);
 185                 break;
 186         default:
 187                 netdev_warn(priv->netdev, "Sync event: Unknown event %ld\n", event);
 188                 err = -EINVAL;
 189         }
 190         return err;
 191 }
 192
 193 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
 194 {
 195         priv->blocking_events_nb.notifier_call = blocking_event;
 196         mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
 197 }
 198
 199 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
 200 {
 201         mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
 202 }
 203
 204 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
 205                                        struct mlx5e_icosq *sq,
 206                                        struct mlx5e_umr_wqe *wqe)
 207 {
 208         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 209         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 210         u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
 211
 212         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 213                                       ds_cnt);
 214         cseg->umr_mkey  = rq->mkey_be;
 215
 216         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
 217         ucseg->xlt_octowords =
 218                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 219         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 220 }
 221
 222 static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node)
 223 {
 224         rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo),
 225                                          GFP_KERNEL, node);
 226         if (!rq->mpwqe.shampo)
 227                 return -ENOMEM;
 228         return 0;
 229 }
 230
 231 static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq)
 232 {
 233         kvfree(rq->mpwqe.shampo);
 234 }
 235
 236 static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
 237 {
 238         struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 239
 240         shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL,
 241                                             node);
 242         if (!shampo->bitmap)
 243                 return -ENOMEM;
 244
 245         shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq,
 246                                                 sizeof(*shampo->info)),
 247                                      GFP_KERNEL, node);
 248         if (!shampo->info) {
 249                 kvfree(shampo->bitmap);
 250                 return -ENOMEM;
 251         }
 252         return 0;
 253 }
 254
 255 static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
 256 {
 257         kvfree(rq->mpwqe.shampo->bitmap);
 258         kvfree(rq->mpwqe.shampo->info);
 259 }
 260
 261 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 262 {
 263         int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 264
 265         rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
 266                                                   sizeof(*rq->mpwqe.info)),
 267                                        GFP_KERNEL, node);
 268         if (!rq->mpwqe.info)
 269                 return -ENOMEM;
 270
 271         mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
 272
 273         return 0;
 274 }
 275
 276 static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
 277                                      u64 npages, u8 page_shift, u32 *umr_mkey,
 278                                      dma_addr_t filler_addr)
 279 {
 280         struct mlx5_mtt *mtt;
 281         int inlen;
 282         void *mkc;
 283         u32 *in;
 284         int err;
 285         int i;
 286
 287         inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages;
 288
 289         in = kvzalloc(inlen, GFP_KERNEL);
 290         if (!in)
 291                 return -ENOMEM;
 292
 293         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 294
 295         MLX5_SET(mkc, mkc, free, 1);
 296         MLX5_SET(mkc, mkc, umr_en, 1);
 297         MLX5_SET(mkc, mkc, lw, 1);
 298         MLX5_SET(mkc, mkc, lr, 1);
 299         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 300         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 301         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 302         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 303         MLX5_SET64(mkc, mkc, len, npages << page_shift);
 304         MLX5_SET(mkc, mkc, translations_octword_size,
 305                  MLX5_MTT_OCTW(npages));
 306         MLX5_SET(mkc, mkc, log_page_size, page_shift);
 307         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 308                  MLX5_MTT_OCTW(npages));
 309
 310         /* Initialize the mkey with all MTTs pointing to a default
 311          * page (filler_addr). When the channels are activated, UMR
 312          * WQEs will redirect the RX WQEs to the actual memory from
 313          * the RQ's pool, while the gaps (wqe_overflow) remain mapped
 314          * to the default page.
 315          */
 316         mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 317         for (i = 0 ; i < npages ; i++)
 318                 mtt[i].ptag = cpu_to_be64(filler_addr);
 319
 320         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 321
 322         kvfree(in);
 323         return err;
 324 }
 325
 326 static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
 327                                      u64 nentries,
 328                                      u32 *umr_mkey)
 329 {
 330         int inlen;
 331         void *mkc;
 332         u32 *in;
 333         int err;
 334
 335         inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 336
 337         in = kvzalloc(inlen, GFP_KERNEL);
 338         if (!in)
 339                 return -ENOMEM;
 340
 341         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 342
 343         MLX5_SET(mkc, mkc, free, 1);
 344         MLX5_SET(mkc, mkc, umr_en, 1);
 345         MLX5_SET(mkc, mkc, lw, 1);
 346         MLX5_SET(mkc, mkc, lr, 1);
 347         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
 348         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 349         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 350         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 351         MLX5_SET(mkc, mkc, translations_octword_size, nentries);
 352         MLX5_SET(mkc, mkc, length64, 1);
 353         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 354
 355         kvfree(in);
 356         return err;
 357 }
 358
 359 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
 360 {
 361         u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
 362
 363         return mlx5e_create_umr_mtt_mkey(mdev, num_mtts, PAGE_SHIFT,
 364                                          &rq->umr_mkey, rq->wqe_overflow.addr);
 365 }
 366
 367 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
 368                                        struct mlx5e_rq *rq)
 369 {
 370         u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size));
 371
 372         if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) {
 373                 mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n",
 374                               max_klm_size, rq->mpwqe.shampo->hd_per_wq);
 375                 return -EINVAL;
 376         }
 377         return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq,
 378                                          &rq->mpwqe.shampo->mkey);
 379 }
 380
 381 static u64 mlx5e_get_mpwqe_offset(u16 wqe_ix)
 382 {
 383         return MLX5E_REQUIRED_MTTS(wqe_ix) << PAGE_SHIFT;
 384 }
 385
 386 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 387 {
 388         struct mlx5e_wqe_frag_info next_frag = {};
 389         struct mlx5e_wqe_frag_info *prev = NULL;
 390         int i;
 391
 392         next_frag.di = &rq->wqe.di[0];
 393
 394         for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 395                 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 396                 struct mlx5e_wqe_frag_info *frag =
 397                         &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
 398                 int f;
 399
 400                 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 401                         if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
 402                                 next_frag.di++;
 403                                 next_frag.offset = 0;
 404                                 if (prev)
 405                                         prev->last_in_page = true;
 406                         }
 407                         *frag = next_frag;
 408
 409                         /* prepare next */
 410                         next_frag.offset += frag_info[f].frag_stride;
 411                         prev = frag;
 412                 }
 413         }
 414
 415         if (prev)
 416                 prev->last_in_page = true;
 417 }
 418
 419 int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
 420 {
 421         int len = wq_sz << rq->wqe.info.log_num_frags;
 422
 423         rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node);
 424         if (!rq->wqe.di)
 425                 return -ENOMEM;
 426
 427         mlx5e_init_frags_partition(rq);
 428
 429         return 0;
 430 }
 431
 432 void mlx5e_free_di_list(struct mlx5e_rq *rq)
 433 {
 434         kvfree(rq->wqe.di);
 435 }
 436
 437 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
 438 {
 439         struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
 440
 441         mlx5e_reporter_rq_cqe_err(rq);
 442 }
 443
 444 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 445 {
 446         rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
 447         if (!rq->wqe_overflow.page)
 448                 return -ENOMEM;
 449
 450         rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
 451                                              PAGE_SIZE, rq->buff.map_dir);
 452         if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
 453                 __free_page(rq->wqe_overflow.page);
 454                 return -ENOMEM;
 455         }
 456         return 0;
 457 }
 458
 459 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 460 {
 461          dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
 462                         rq->buff.map_dir);
 463          __free_page(rq->wqe_overflow.page);
 464 }
 465
 466 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
 467                              struct mlx5e_rq *rq)
 468 {
 469         struct mlx5_core_dev *mdev = c->mdev;
 470         int err;
 471
 472         rq->wq_type      = params->rq_wq_type;
 473         rq->pdev         = c->pdev;
 474         rq->netdev       = c->netdev;
 475         rq->priv         = c->priv;
 476         rq->tstamp       = c->tstamp;
 477         rq->clock        = &mdev->clock;
 478         rq->icosq        = &c->icosq;
 479         rq->ix           = c->ix;
 480         rq->mdev         = mdev;
 481         rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 482         rq->xdpsq        = &c->rq_xdpsq;
 483         rq->stats        = &c->priv->channel_stats[c->ix]->rq;
 484         rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
 485         err = mlx5e_rq_set_handlers(rq, params, NULL);
 486         if (err)
 487                 return err;
 488
 489         return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, 0);
 490 }
 491
 492 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
 493                                 struct mlx5e_params *params,
 494                                 struct mlx5e_rq_param *rqp,
 495                                 struct mlx5e_rq *rq,
 496                                 u32 *pool_size,
 497                                 int node)
 498 {
 499         void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
 500         int wq_size;
 501         int err;
 502
 503         if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 504                 return 0;
 505         err = mlx5e_rq_shampo_hd_alloc(rq, node);
 506         if (err)
 507                 goto out;
 508         rq->mpwqe.shampo->hd_per_wq =
 509                 mlx5e_shampo_hd_per_wq(mdev, params, rqp);
 510         err = mlx5e_create_rq_hd_umr_mkey(mdev, rq);
 511         if (err)
 512                 goto err_shampo_hd;
 513         err = mlx5e_rq_shampo_hd_info_alloc(rq, node);
 514         if (err)
 515                 goto err_shampo_info;
 516         rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node);
 517         if (!rq->hw_gro_data) {
 518                 err = -ENOMEM;
 519                 goto err_hw_gro_data;
 520         }
 521         rq->mpwqe.shampo->key =
 522                 cpu_to_be32(rq->mpwqe.shampo->mkey);
 523         rq->mpwqe.shampo->hd_per_wqe =
 524                 mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
 525         wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
 526         *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
 527                      MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
 528         return 0;
 529
 530 err_hw_gro_data:
 531         mlx5e_rq_shampo_hd_info_free(rq);
 532 err_shampo_info:
 533         mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey);
 534 err_shampo_hd:
 535         mlx5e_rq_shampo_hd_free(rq);
 536 out:
 537         return err;
 538 }
 539
 540 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
 541 {
 542         if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 543                 return;
 544
 545         kvfree(rq->hw_gro_data);
 546         mlx5e_rq_shampo_hd_info_free(rq);
 547         mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey);
 548         mlx5e_rq_shampo_hd_free(rq);
 549 }
 550
 551 static int mlx5e_alloc_rq(struct mlx5e_params *params,
 552                           struct mlx5e_xsk_param *xsk,
 553                           struct mlx5e_rq_param *rqp,
 554                           int node, struct mlx5e_rq *rq)
 555 {
 556         struct page_pool_params pp_params = { 0 };
 557         struct mlx5_core_dev *mdev = rq->mdev;
 558         void *rqc = rqp->rqc;
 559         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 560         u32 pool_size;
 561         int wq_sz;
 562         int err;
 563         int i;
 564
 565         rqp->wq.db_numa_node = node;
 566         INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
 567
 568         if (params->xdp_prog)
 569                 bpf_prog_inc(params->xdp_prog);
 570         RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
 571
 572         rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 573         rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
 574         pool_size = 1 << params->log_rq_mtu_frames;
 575
 576         switch (rq->wq_type) {
 577         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 578                 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
 579                                         &rq->wq_ctrl);
 580                 if (err)
 581                         goto err_rq_xdp_prog;
 582
 583                 err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
 584                 if (err)
 585                         goto err_rq_wq_destroy;
 586
 587                 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
 588
 589                 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 590
 591                 pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
 592                         mlx5e_mpwqe_get_log_rq_size(params, xsk);
 593
 594                 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 595                 rq->mpwqe.num_strides =
 596                         BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
 597
 598                 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
 599
 600                 err = mlx5e_create_rq_umr_mkey(mdev, rq);
 601                 if (err)
 602                         goto err_rq_drop_page;
 603                 rq->mkey_be = cpu_to_be32(rq->umr_mkey);
 604
 605                 err = mlx5e_rq_alloc_mpwqe_info(rq, node);
 606                 if (err)
 607                         goto err_rq_mkey;
 608
 609                 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node);
 610                 if (err)
 611                         goto err_free_by_rq_type;
 612
 613                 break;
 614         default: /* MLX5_WQ_TYPE_CYCLIC */
 615                 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
 616                                          &rq->wq_ctrl);
 617                 if (err)
 618                         goto err_rq_xdp_prog;
 619
 620                 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
 621
 622                 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 623
 624                 rq->wqe.info = rqp->frags_info;
 625                 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 626
 627                 rq->wqe.frags =
 628                         kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
 629                                         (wq_sz << rq->wqe.info.log_num_frags)),
 630                                       GFP_KERNEL, node);
 631                 if (!rq->wqe.frags) {
 632                         err = -ENOMEM;
 633                         goto err_rq_wq_destroy;
 634                 }
 635
 636                 err = mlx5e_init_di_list(rq, wq_sz, node);
 637                 if (err)
 638                         goto err_rq_frags;
 639
 640                 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
 641         }
 642
 643         if (xsk) {
 644                 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 645                                                  MEM_TYPE_XSK_BUFF_POOL, NULL);
 646                 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
 647         } else {
 648                 /* Create a page_pool and register it with rxq */
 649                 pp_params.order     = 0;
 650                 pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
 651                 pp_params.pool_size = pool_size;
 652                 pp_params.nid       = node;
 653                 pp_params.dev       = rq->pdev;
 654                 pp_params.dma_dir   = rq->buff.map_dir;
 655
 656                 /* page_pool can be used even when there is no rq->xdp_prog,
 657                  * given page_pool does not handle DMA mapping there is no
 658                  * required state to clear. And page_pool gracefully handle
 659                  * elevated refcnt.
 660                  */
 661                 rq->page_pool = page_pool_create(&pp_params);
 662                 if (IS_ERR(rq->page_pool)) {
 663                         err = PTR_ERR(rq->page_pool);
 664                         rq->page_pool = NULL;
 665                         goto err_free_shampo;
 666                 }
 667                 if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
 668                         err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 669                                                          MEM_TYPE_PAGE_POOL, rq->page_pool);
 670         }
 671         if (err)
 672                 goto err_free_shampo;
 673
 674         for (i = 0; i < wq_sz; i++) {
 675                 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 676                         struct mlx5e_rx_wqe_ll *wqe =
 677                                 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
 678                         u32 byte_count =
 679                                 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
 680                         u64 dma_offset = mlx5e_get_mpwqe_offset(i);
 681                         u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ?
 682                                        0 : rq->buff.headroom;
 683
 684                         wqe->data[0].addr = cpu_to_be64(dma_offset + headroom);
 685                         wqe->data[0].byte_count = cpu_to_be32(byte_count);
 686                         wqe->data[0].lkey = rq->mkey_be;
 687                 } else {
 688                         struct mlx5e_rx_wqe_cyc *wqe =
 689                                 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
 690                         int f;
 691
 692                         for (f = 0; f < rq->wqe.info.num_frags; f++) {
 693                                 u32 frag_size = rq->wqe.info.arr[f].frag_size |
 694                                         MLX5_HW_START_PADDING;
 695
 696                                 wqe->data[f].byte_count = cpu_to_be32(frag_size);
 697                                 wqe->data[f].lkey = rq->mkey_be;
 698                         }
 699                         /* check if num_frags is not a pow of two */
 700                         if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
 701                                 wqe->data[f].byte_count = 0;
 702                                 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
 703                                 wqe->data[f].addr = 0;
 704                         }
 705                 }
 706         }
 707
 708         INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
 709
 710         switch (params->rx_cq_moderation.cq_period_mode) {
 711         case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
 712                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 713                 break;
 714         case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
 715         default:
 716                 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 717         }
 718
 719         rq->page_cache.head = 0;
 720         rq->page_cache.tail = 0;
 721
 722         return 0;
 723
 724 err_free_shampo:
 725         mlx5e_rq_free_shampo(rq);
 726 err_free_by_rq_type:
 727         switch (rq->wq_type) {
 728         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 729                 kvfree(rq->mpwqe.info);
 730 err_rq_mkey:
 731                 mlx5_core_destroy_mkey(mdev, rq->umr_mkey);
 732 err_rq_drop_page:
 733                 mlx5e_free_mpwqe_rq_drop_page(rq);
 734                 break;
 735         default: /* MLX5_WQ_TYPE_CYCLIC */
 736                 mlx5e_free_di_list(rq);
 737 err_rq_frags:
 738                 kvfree(rq->wqe.frags);
 739         }
 740 err_rq_wq_destroy:
 741         mlx5_wq_destroy(&rq->wq_ctrl);
 742 err_rq_xdp_prog:
 743         if (params->xdp_prog)
 744                 bpf_prog_put(params->xdp_prog);
 745
 746         return err;
 747 }
 748
 749 static void mlx5e_free_rq(struct mlx5e_rq *rq)
 750 {
 751         struct bpf_prog *old_prog;
 752         int i;
 753
 754         if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
 755                 old_prog = rcu_dereference_protected(rq->xdp_prog,
 756                                                      lockdep_is_held(&rq->priv->state_lock));
 757                 if (old_prog)
 758                         bpf_prog_put(old_prog);
 759         }
 760
 761         switch (rq->wq_type) {
 762         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 763                 kvfree(rq->mpwqe.info);
 764                 mlx5_core_destroy_mkey(rq->mdev, rq->umr_mkey);
 765                 mlx5e_free_mpwqe_rq_drop_page(rq);
 766                 mlx5e_rq_free_shampo(rq);
 767                 break;
 768         default: /* MLX5_WQ_TYPE_CYCLIC */
 769                 kvfree(rq->wqe.frags);
 770                 mlx5e_free_di_list(rq);
 771         }
 772
 773         for (i = rq->page_cache.head; i != rq->page_cache.tail;
 774              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 775                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 776
 777                 /* With AF_XDP, page_cache is not used, so this loop is not
 778                  * entered, and it's safe to call mlx5e_page_release_dynamic
 779                  * directly.
 780                  */
 781                 mlx5e_page_release_dynamic(rq, dma_info, false);
 782         }
 783
 784         xdp_rxq_info_unreg(&rq->xdp_rxq);
 785         page_pool_destroy(rq->page_pool);
 786         mlx5_wq_destroy(&rq->wq_ctrl);
 787 }
 788
 789 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 790 {
 791         struct mlx5_core_dev *mdev = rq->mdev;
 792         u8 ts_format;
 793         void *in;
 794         void *rqc;
 795         void *wq;
 796         int inlen;
 797         int err;
 798
 799         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 800                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 801         in = kvzalloc(inlen, GFP_KERNEL);
 802         if (!in)
 803                 return -ENOMEM;
 804
 805         ts_format = mlx5_is_real_time_rq(mdev) ?
 806                             MLX5_TIMESTAMP_FORMAT_REAL_TIME :
 807                             MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
 808         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 809         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 810
 811         memcpy(rqc, param->rqc, sizeof(param->rqc));
 812
 813         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 814         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 815         MLX5_SET(rqc,  rqc, ts_format,          ts_format);
 816         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 817                                                 MLX5_ADAPTER_PAGE_SHIFT);
 818         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 819
 820         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
 821                 MLX5_SET(wq, wq, log_headers_buffer_entry_num,
 822                          order_base_2(rq->mpwqe.shampo->hd_per_wq));
 823                 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey);
 824         }
 825
 826         mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
 827                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 828
 829         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 830
 831         kvfree(in);
 832
 833         return err;
 834 }
 835
 836 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 837 {
 838         struct mlx5_core_dev *mdev = rq->mdev;
 839
 840         void *in;
 841         void *rqc;
 842         int inlen;
 843         int err;
 844
 845         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 846         in = kvzalloc(inlen, GFP_KERNEL);
 847         if (!in)
 848                 return -ENOMEM;
 849
 850         if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
 851                 mlx5e_rqwq_reset(rq);
 852
 853         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 854
 855         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 856         MLX5_SET(rqc, rqc, state, next_state);
 857
 858         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 859
 860         kvfree(in);
 861
 862         return err;
 863 }
 864
 865 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
 866 {
 867         struct mlx5_core_dev *mdev = rq->mdev;
 868
 869         void *in;
 870         void *rqc;
 871         int inlen;
 872         int err;
 873
 874         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 875         in = kvzalloc(inlen, GFP_KERNEL);
 876         if (!in)
 877                 return -ENOMEM;
 878
 879         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 880
 881         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 882         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 883                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
 884         MLX5_SET(rqc, rqc, scatter_fcs, enable);
 885         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 886
 887         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 888
 889         kvfree(in);
 890
 891         return err;
 892 }
 893
 894 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 895 {
 896         struct mlx5_core_dev *mdev = rq->mdev;
 897         void *in;
 898         void *rqc;
 899         int inlen;
 900         int err;
 901
 902         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 903         in = kvzalloc(inlen, GFP_KERNEL);
 904         if (!in)
 905                 return -ENOMEM;
 906
 907         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 908
 909         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 910         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 911                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 912         MLX5_SET(rqc, rqc, vsd, vsd);
 913         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 914
 915         err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 916
 917         kvfree(in);
 918
 919         return err;
 920 }
 921
 922 void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 923 {
 924         mlx5_core_destroy_rq(rq->mdev, rq->rqn);
 925 }
 926
 927 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
 928 {
 929         unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
 930
 931         u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
 932
 933         do {
 934                 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
 935                         return 0;
 936
 937                 msleep(20);
 938         } while (time_before(jiffies, exp_time));
 939
 940         netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
 941                     rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
 942
 943         mlx5e_reporter_rx_timeout(rq);
 944         return -ETIMEDOUT;
 945 }
 946
 947 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
 948 {
 949         struct mlx5_wq_ll *wq;
 950         u16 head;
 951         int i;
 952
 953         if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 954                 return;
 955
 956         wq = &rq->mpwqe.wq;
 957         head = wq->head;
 958
 959         /* Outstanding UMR WQEs (in progress) start at wq->head */
 960         for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
 961                 rq->dealloc_wqe(rq, head);
 962                 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 963         }
 964
 965         if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
 966                 u16 len;
 967
 968                 len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) &
 969                       (rq->mpwqe.shampo->hd_per_wq - 1);
 970                 mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false);
 971                 rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci;
 972         }
 973
 974         rq->mpwqe.actual_wq_head = wq->head;
 975         rq->mpwqe.umr_in_progress = 0;
 976         rq->mpwqe.umr_completed = 0;
 977 }
 978
 979 void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 980 {
 981         __be16 wqe_ix_be;
 982         u16 wqe_ix;
 983
 984         if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 985                 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 986
 987                 mlx5e_free_rx_in_progress_descs(rq);
 988
 989                 while (!mlx5_wq_ll_is_empty(wq)) {
 990                         struct mlx5e_rx_wqe_ll *wqe;
 991
 992                         wqe_ix_be = *wq->tail_next;
 993                         wqe_ix    = be16_to_cpu(wqe_ix_be);
 994                         wqe       = mlx5_wq_ll_get_wqe(wq, wqe_ix);
 995                         rq->dealloc_wqe(rq, wqe_ix);
 996                         mlx5_wq_ll_pop(wq, wqe_ix_be,
 997                                        &wqe->next.next_wqe_index);
 998                 }
 999
1000                 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
1001                         mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq,
1002                                                 0, true);
1003         } else {
1004                 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1005
1006                 while (!mlx5_wq_cyc_is_empty(wq)) {
1007                         wqe_ix = mlx5_wq_cyc_get_tail(wq);
1008                         rq->dealloc_wqe(rq, wqe_ix);
1009                         mlx5_wq_cyc_pop(wq);
1010                 }
1011         }
1012
1013 }
1014
1015 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
1016                   struct mlx5e_xsk_param *xsk, int node,
1017                   struct mlx5e_rq *rq)
1018 {
1019         struct mlx5_core_dev *mdev = rq->mdev;
1020         int err;
1021
1022         if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1023                 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
1024
1025         err = mlx5e_alloc_rq(params, xsk, param, node, rq);
1026         if (err)
1027                 return err;
1028
1029         err = mlx5e_create_rq(rq, param);
1030         if (err)
1031                 goto err_free_rq;
1032
1033         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1034         if (err)
1035                 goto err_destroy_rq;
1036
1037         if (mlx5e_is_tls_on(rq->priv) && !mlx5e_accel_is_ktls_device(mdev))
1038                 __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &rq->state); /* must be FPGA */
1039
1040         if (MLX5_CAP_ETH(mdev, cqe_checksum_full))
1041                 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
1042
1043         if (params->rx_dim_enabled)
1044                 __set_bit(MLX5E_RQ_STATE_AM, &rq->state);
1045
1046         /* We disable csum_complete when XDP is enabled since
1047          * XDP programs might manipulate packets which will render
1048          * skb->checksum incorrect.
1049          */
1050         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog)
1051                 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state);
1052
1053         /* For CQE compression on striding RQ, use stride index provided by
1054          * HW if capability is supported.
1055          */
1056         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
1057             MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index))
1058                 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state);
1059
1060         return 0;
1061
1062 err_destroy_rq:
1063         mlx5e_destroy_rq(rq);
1064 err_free_rq:
1065         mlx5e_free_rq(rq);
1066
1067         return err;
1068 }
1069
1070 void mlx5e_activate_rq(struct mlx5e_rq *rq)
1071 {
1072         set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1073         if (rq->icosq) {
1074                 mlx5e_trigger_irq(rq->icosq);
1075         } else {
1076                 local_bh_disable();
1077                 napi_schedule(rq->cq.napi);
1078                 local_bh_enable();
1079         }
1080 }
1081
1082 void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
1083 {
1084         clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
1085         synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
1086 }
1087
1088 void mlx5e_close_rq(struct mlx5e_rq *rq)
1089 {
1090         cancel_work_sync(&rq->dim.work);
1091         cancel_work_sync(&rq->recover_work);
1092         mlx5e_destroy_rq(rq);
1093         mlx5e_free_rx_descs(rq);
1094         mlx5e_free_rq(rq);
1095 }
1096
1097 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
1098 {
1099         kvfree(sq->db.xdpi_fifo.xi);
1100         kvfree(sq->db.wqe_info);
1101 }
1102
1103 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
1104 {
1105         struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
1106         int wq_sz        = mlx5_wq_cyc_get_size(&sq->wq);
1107         int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1108         size_t size;
1109
1110         size = array_size(sizeof(*xdpi_fifo->xi), dsegs_per_wq);
1111         xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa);
1112         if (!xdpi_fifo->xi)
1113                 return -ENOMEM;
1114
1115         xdpi_fifo->pc   = &sq->xdpi_fifo_pc;
1116         xdpi_fifo->cc   = &sq->xdpi_fifo_cc;
1117         xdpi_fifo->mask = dsegs_per_wq - 1;
1118
1119         return 0;
1120 }
1121
1122 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
1123 {
1124         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1125         size_t size;
1126         int err;
1127
1128         size = array_size(sizeof(*sq->db.wqe_info), wq_sz);
1129         sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1130         if (!sq->db.wqe_info)
1131                 return -ENOMEM;
1132
1133         err = mlx5e_alloc_xdpsq_fifo(sq, numa);
1134         if (err) {
1135                 mlx5e_free_xdpsq_db(sq);
1136                 return err;
1137         }
1138
1139         return 0;
1140 }
1141
1142 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
1143                              struct mlx5e_params *params,
1144                              struct xsk_buff_pool *xsk_pool,
1145                              struct mlx5e_sq_param *param,
1146                              struct mlx5e_xdpsq *sq,
1147                              bool is_redirect)
1148 {
1149         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1150         struct mlx5_core_dev *mdev = c->mdev;
1151         struct mlx5_wq_cyc *wq = &sq->wq;
1152         int err;
1153
1154         sq->pdev      = c->pdev;
1155         sq->mkey_be   = c->mkey_be;
1156         sq->channel   = c;
1157         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1158         sq->min_inline_mode = params->tx_min_inline_mode;
1159         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1160         sq->xsk_pool  = xsk_pool;
1161
1162         sq->stats = sq->xsk_pool ?
1163                 &c->priv->channel_stats[c->ix]->xsksq :
1164                 is_redirect ?
1165                         &c->priv->channel_stats[c->ix]->xdpsq :
1166                         &c->priv->channel_stats[c->ix]->rq_xdpsq;
1167
1168         param->wq.db_numa_node = cpu_to_node(c->cpu);
1169         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1170         if (err)
1171                 return err;
1172         wq->db = &wq->db[MLX5_SND_DBR];
1173
1174         err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1175         if (err)
1176                 goto err_sq_wq_destroy;
1177
1178         return 0;
1179
1180 err_sq_wq_destroy:
1181         mlx5_wq_destroy(&sq->wq_ctrl);
1182
1183         return err;
1184 }
1185
1186 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1187 {
1188         mlx5e_free_xdpsq_db(sq);
1189         mlx5_wq_destroy(&sq->wq_ctrl);
1190 }
1191
1192 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1193 {
1194         kvfree(sq->db.wqe_info);
1195 }
1196
1197 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1198 {
1199         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1200         size_t size;
1201
1202         size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1203         sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1204         if (!sq->db.wqe_info)
1205                 return -ENOMEM;
1206
1207         return 0;
1208 }
1209
1210 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1211 {
1212         struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1213                                               recover_work);
1214
1215         mlx5e_reporter_icosq_cqe_err(sq);
1216 }
1217
1218 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
1219 {
1220         struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1221                                               recover_work);
1222
1223         /* Not implemented yet. */
1224
1225         netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n");
1226 }
1227
1228 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1229                              struct mlx5e_sq_param *param,
1230                              struct mlx5e_icosq *sq,
1231                              work_func_t recover_work_func)
1232 {
1233         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1234         struct mlx5_core_dev *mdev = c->mdev;
1235         struct mlx5_wq_cyc *wq = &sq->wq;
1236         int err;
1237
1238         sq->channel   = c;
1239         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1240         sq->reserved_room = param->stop_room;
1241
1242         param->wq.db_numa_node = cpu_to_node(c->cpu);
1243         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1244         if (err)
1245                 return err;
1246         wq->db = &wq->db[MLX5_SND_DBR];
1247
1248         err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1249         if (err)
1250                 goto err_sq_wq_destroy;
1251
1252         INIT_WORK(&sq->recover_work, recover_work_func);
1253
1254         return 0;
1255
1256 err_sq_wq_destroy:
1257         mlx5_wq_destroy(&sq->wq_ctrl);
1258
1259         return err;
1260 }
1261
1262 static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1263 {
1264         mlx5e_free_icosq_db(sq);
1265         mlx5_wq_destroy(&sq->wq_ctrl);
1266 }
1267
1268 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1269 {
1270         kvfree(sq->db.wqe_info);
1271         kvfree(sq->db.skb_fifo.fifo);
1272         kvfree(sq->db.dma_fifo);
1273 }
1274
1275 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1276 {
1277         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1278         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1279
1280         sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1281                                                    sizeof(*sq->db.dma_fifo)),
1282                                         GFP_KERNEL, numa);
1283         sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1284                                                         sizeof(*sq->db.skb_fifo.fifo)),
1285                                         GFP_KERNEL, numa);
1286         sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1287                                                    sizeof(*sq->db.wqe_info)),
1288                                         GFP_KERNEL, numa);
1289         if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
1290                 mlx5e_free_txqsq_db(sq);
1291                 return -ENOMEM;
1292         }
1293
1294         sq->dma_fifo_mask = df_sz - 1;
1295
1296         sq->db.skb_fifo.pc   = &sq->skb_fifo_pc;
1297         sq->db.skb_fifo.cc   = &sq->skb_fifo_cc;
1298         sq->db.skb_fifo.mask = df_sz - 1;
1299
1300         return 0;
1301 }
1302
1303 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1304                              int txq_ix,
1305                              struct mlx5e_params *params,
1306                              struct mlx5e_sq_param *param,
1307                              struct mlx5e_txqsq *sq,
1308                              int tc)
1309 {
1310         void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1311         struct mlx5_core_dev *mdev = c->mdev;
1312         struct mlx5_wq_cyc *wq = &sq->wq;
1313         int err;
1314
1315         sq->pdev      = c->pdev;
1316         sq->tstamp    = c->tstamp;
1317         sq->clock     = &mdev->clock;
1318         sq->mkey_be   = c->mkey_be;
1319         sq->netdev    = c->netdev;
1320         sq->mdev      = c->mdev;
1321         sq->priv      = c->priv;
1322         sq->ch_ix     = c->ix;
1323         sq->txq_ix    = txq_ix;
1324         sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1325         sq->min_inline_mode = params->tx_min_inline_mode;
1326         sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1327         INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
1328         if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1329                 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
1330         if (MLX5_IPSEC_DEV(c->priv->mdev))
1331                 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1332         if (param->is_mpw)
1333                 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
1334         sq->stop_room = param->stop_room;
1335         sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
1336
1337         param->wq.db_numa_node = cpu_to_node(c->cpu);
1338         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1339         if (err)
1340                 return err;
1341         wq->db    = &wq->db[MLX5_SND_DBR];
1342
1343         err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1344         if (err)
1345                 goto err_sq_wq_destroy;
1346
1347         INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1348         sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1349
1350         return 0;
1351
1352 err_sq_wq_destroy:
1353         mlx5_wq_destroy(&sq->wq_ctrl);
1354
1355         return err;
1356 }
1357
1358 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1359 {
1360         mlx5e_free_txqsq_db(sq);
1361         mlx5_wq_destroy(&sq->wq_ctrl);
1362 }
1363
1364 static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1365                            struct mlx5e_sq_param *param,
1366                            struct mlx5e_create_sq_param *csp,
1367                            u32 *sqn)
1368 {
1369         u8 ts_format;
1370         void *in;
1371         void *sqc;
1372         void *wq;
1373         int inlen;
1374         int err;
1375
1376         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1377                 sizeof(u64) * csp->wq_ctrl->buf.npages;
1378         in = kvzalloc(inlen, GFP_KERNEL);
1379         if (!in)
1380                 return -ENOMEM;
1381
1382         ts_format = mlx5_is_real_time_sq(mdev) ?
1383                             MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1384                             MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1385         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1386         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1387
1388         memcpy(sqc, param->sqc, sizeof(param->sqc));
1389         MLX5_SET(sqc,  sqc, tis_lst_sz, csp->tis_lst_sz);
1390         MLX5_SET(sqc,  sqc, tis_num_0, csp->tisn);
1391         MLX5_SET(sqc,  sqc, cqn, csp->cqn);
1392         MLX5_SET(sqc,  sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
1393         MLX5_SET(sqc,  sqc, ts_format, ts_format);
1394
1395
1396         if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1397                 MLX5_SET(sqc,  sqc, min_wqe_inline_mode, csp->min_inline_mode);
1398
1399         MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
1400         MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
1401
1402         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1403         MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
1404         MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
1405                                           MLX5_ADAPTER_PAGE_SHIFT);
1406         MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
1407
1408         mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1409                                   (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1410
1411         err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1412
1413         kvfree(in);
1414
1415         return err;
1416 }
1417
1418 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1419                     struct mlx5e_modify_sq_param *p)
1420 {
1421         u64 bitmask = 0;
1422         void *in;
1423         void *sqc;
1424         int inlen;
1425         int err;
1426
1427         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1428         in = kvzalloc(inlen, GFP_KERNEL);
1429         if (!in)
1430                 return -ENOMEM;
1431
1432         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1433
1434         MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1435         MLX5_SET(sqc, sqc, state, p->next_state);
1436         if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1437                 bitmask |= 1;
1438                 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
1439         }
1440         if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1441                 bitmask |= 1 << 2;
1442                 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1443         }
1444         MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
1445
1446         err = mlx5_core_modify_sq(mdev, sqn, in);
1447
1448         kvfree(in);
1449
1450         return err;
1451 }
1452
1453 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1454 {
1455         mlx5_core_destroy_sq(mdev, sqn);
1456 }
1457
1458 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1459                         struct mlx5e_sq_param *param,
1460                         struct mlx5e_create_sq_param *csp,
1461                         u16 qos_queue_group_id,
1462                         u32 *sqn)
1463 {
1464         struct mlx5e_modify_sq_param msp = {0};
1465         int err;
1466
1467         err = mlx5e_create_sq(mdev, param, csp, sqn);
1468         if (err)
1469                 return err;
1470
1471         msp.curr_state = MLX5_SQC_STATE_RST;
1472         msp.next_state = MLX5_SQC_STATE_RDY;
1473         if (qos_queue_group_id) {
1474                 msp.qos_update = true;
1475                 msp.qos_queue_group_id = qos_queue_group_id;
1476         }
1477         err = mlx5e_modify_sq(mdev, *sqn, &msp);
1478         if (err)
1479                 mlx5e_destroy_sq(mdev, *sqn);
1480
1481         return err;
1482 }
1483
1484 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1485                                 struct mlx5e_txqsq *sq, u32 rate);
1486
1487 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1488                      struct mlx5e_params *params, struct mlx5e_sq_param *param,
1489                      struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1490                      struct mlx5e_sq_stats *sq_stats)
1491 {
1492         struct mlx5e_create_sq_param csp = {};
1493         u32 tx_rate;
1494         int err;
1495
1496         err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1497         if (err)
1498                 return err;
1499
1500         sq->stats = sq_stats;
1501
1502         csp.tisn            = tisn;
1503         csp.tis_lst_sz      = 1;
1504         csp.cqn             = sq->cq.mcq.cqn;
1505         csp.wq_ctrl         = &sq->wq_ctrl;
1506         csp.min_inline_mode = sq->min_inline_mode;
1507         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
1508         if (err)
1509                 goto err_free_txqsq;
1510
1511         tx_rate = c->priv->tx_rates[sq->txq_ix];
1512         if (tx_rate)
1513                 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1514
1515         if (params->tx_dim_enabled)
1516                 sq->state |= BIT(MLX5E_SQ_STATE_AM);
1517
1518         return 0;
1519
1520 err_free_txqsq:
1521         mlx5e_free_txqsq(sq);
1522
1523         return err;
1524 }
1525
1526 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1527 {
1528         sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
1529         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1530         netdev_tx_reset_queue(sq->txq);
1531         netif_tx_start_queue(sq->txq);
1532 }
1533
1534 void mlx5e_tx_disable_queue(struct netdev_queue *txq)
1535 {
1536         __netif_tx_lock_bh(txq);
1537         netif_tx_stop_queue(txq);
1538         __netif_tx_unlock_bh(txq);
1539 }
1540
1541 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1542 {
1543         struct mlx5_wq_cyc *wq = &sq->wq;
1544
1545         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1546         synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1547
1548         mlx5e_tx_disable_queue(sq->txq);
1549
1550         /* last doorbell out, godspeed .. */
1551         if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1552                 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1553                 struct mlx5e_tx_wqe *nop;
1554
1555                 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1556                         .num_wqebbs = 1,
1557                 };
1558
1559                 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1560                 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1561         }
1562 }
1563
1564 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1565 {
1566         struct mlx5_core_dev *mdev = sq->mdev;
1567         struct mlx5_rate_limit rl = {0};
1568
1569         cancel_work_sync(&sq->dim.work);
1570         cancel_work_sync(&sq->recover_work);
1571         mlx5e_destroy_sq(mdev, sq->sqn);
1572         if (sq->rate_limit) {
1573                 rl.rate = sq->rate_limit;
1574                 mlx5_rl_remove_rate(mdev, &rl);
1575         }
1576         mlx5e_free_txqsq_descs(sq);
1577         mlx5e_free_txqsq(sq);
1578 }
1579
1580 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
1581 {
1582         struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1583                                               recover_work);
1584
1585         mlx5e_reporter_tx_err_cqe(sq);
1586 }
1587
1588 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1589                             struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
1590                             work_func_t recover_work_func)
1591 {
1592         struct mlx5e_create_sq_param csp = {};
1593         int err;
1594
1595         err = mlx5e_alloc_icosq(c, param, sq, recover_work_func);
1596         if (err)
1597                 return err;
1598
1599         csp.cqn             = sq->cq.mcq.cqn;
1600         csp.wq_ctrl         = &sq->wq_ctrl;
1601         csp.min_inline_mode = params->tx_min_inline_mode;
1602         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1603         if (err)
1604                 goto err_free_icosq;
1605
1606         if (param->is_tls) {
1607                 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list();
1608                 if (IS_ERR(sq->ktls_resync)) {
1609                         err = PTR_ERR(sq->ktls_resync);
1610                         goto err_destroy_icosq;
1611                 }
1612         }
1613         return 0;
1614
1615 err_destroy_icosq:
1616         mlx5e_destroy_sq(c->mdev, sq->sqn);
1617 err_free_icosq:
1618         mlx5e_free_icosq(sq);
1619
1620         return err;
1621 }
1622
1623 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
1624 {
1625         set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1626 }
1627
1628 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
1629 {
1630         clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1631         synchronize_net(); /* Sync with NAPI. */
1632 }
1633
1634 static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1635 {
1636         struct mlx5e_channel *c = sq->channel;
1637
1638         if (sq->ktls_resync)
1639                 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync);
1640         mlx5e_destroy_sq(c->mdev, sq->sqn);
1641         mlx5e_free_icosq_descs(sq);
1642         mlx5e_free_icosq(sq);
1643 }
1644
1645 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1646                      struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1647                      struct mlx5e_xdpsq *sq, bool is_redirect)
1648 {
1649         struct mlx5e_create_sq_param csp = {};
1650         int err;
1651
1652         err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
1653         if (err)
1654                 return err;
1655
1656         csp.tis_lst_sz      = 1;
1657         csp.tisn            = c->priv->tisn[c->lag_port][0]; /* tc = 0 */
1658         csp.cqn             = sq->cq.mcq.cqn;
1659         csp.wq_ctrl         = &sq->wq_ctrl;
1660         csp.min_inline_mode = sq->min_inline_mode;
1661         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1662         err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1663         if (err)
1664                 goto err_free_xdpsq;
1665
1666         mlx5e_set_xmit_fp(sq, param->is_mpw);
1667
1668         if (!param->is_mpw) {
1669                 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1670                 unsigned int inline_hdr_sz = 0;
1671                 int i;
1672
1673                 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1674                         inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1675                         ds_cnt++;
1676                 }
1677
1678                 /* Pre initialize fixed WQE fields */
1679                 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1680                         struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1681                         struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1682                         struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
1683                         struct mlx5_wqe_data_seg *dseg;
1684
1685                         sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
1686                                 .num_wqebbs = 1,
1687                                 .num_pkts   = 1,
1688                         };
1689
1690                         cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1691                         eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1692
1693                         dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1694                         dseg->lkey = sq->mkey_be;
1695                 }
1696         }
1697
1698         return 0;
1699
1700 err_free_xdpsq:
1701         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1702         mlx5e_free_xdpsq(sq);
1703
1704         return err;
1705 }
1706
1707 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1708 {
1709         struct mlx5e_channel *c = sq->channel;
1710
1711         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1712         synchronize_net(); /* Sync with NAPI. */
1713
1714         mlx5e_destroy_sq(c->mdev, sq->sqn);
1715         mlx5e_free_xdpsq_descs(sq);
1716         mlx5e_free_xdpsq(sq);
1717 }
1718
1719 static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv,
1720                                  struct mlx5e_cq_param *param,
1721                                  struct mlx5e_cq *cq)
1722 {
1723         struct mlx5_core_dev *mdev = priv->mdev;
1724         struct mlx5_core_cq *mcq = &cq->mcq;
1725         int err;
1726         u32 i;
1727
1728         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1729                                &cq->wq_ctrl);
1730         if (err)
1731                 return err;
1732
1733         mcq->cqe_sz     = 64;
1734         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1735         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1736         *mcq->set_ci_db = 0;
1737         *mcq->arm_db    = 0;
1738         mcq->vector     = param->eq_ix;
1739         mcq->comp       = mlx5e_completion_event;
1740         mcq->event      = mlx5e_cq_error_event;
1741
1742         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1743                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1744
1745                 cqe->op_own = 0xf1;
1746         }
1747
1748         cq->mdev = mdev;
1749         cq->netdev = priv->netdev;
1750         cq->priv = priv;
1751
1752         return 0;
1753 }
1754
1755 static int mlx5e_alloc_cq(struct mlx5e_priv *priv,
1756                           struct mlx5e_cq_param *param,
1757                           struct mlx5e_create_cq_param *ccp,
1758                           struct mlx5e_cq *cq)
1759 {
1760         int err;
1761
1762         param->wq.buf_numa_node = ccp->node;
1763         param->wq.db_numa_node  = ccp->node;
1764         param->eq_ix            = ccp->ix;
1765
1766         err = mlx5e_alloc_cq_common(priv, param, cq);
1767
1768         cq->napi     = ccp->napi;
1769         cq->ch_stats = ccp->ch_stats;
1770
1771         return err;
1772 }
1773
1774 static void mlx5e_free_cq(struct mlx5e_cq *cq)
1775 {
1776         mlx5_wq_destroy(&cq->wq_ctrl);
1777 }
1778
1779 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1780 {
1781         u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1782         struct mlx5_core_dev *mdev = cq->mdev;
1783         struct mlx5_core_cq *mcq = &cq->mcq;
1784
1785         void *in;
1786         void *cqc;
1787         int inlen;
1788         int eqn;
1789         int err;
1790
1791         err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn);
1792         if (err)
1793                 return err;
1794
1795         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1796                 sizeof(u64) * cq->wq_ctrl.buf.npages;
1797         in = kvzalloc(inlen, GFP_KERNEL);
1798         if (!in)
1799                 return -ENOMEM;
1800
1801         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1802
1803         memcpy(cqc, param->cqc, sizeof(param->cqc));
1804
1805         mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1806                                   (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1807
1808         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1809         MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
1810         MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
1811         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1812                                             MLX5_ADAPTER_PAGE_SHIFT);
1813         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1814
1815         err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
1816
1817         kvfree(in);
1818
1819         if (err)
1820                 return err;
1821
1822         mlx5e_cq_arm(cq);
1823
1824         return 0;
1825 }
1826
1827 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1828 {
1829         mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1830 }
1831
1832 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
1833                   struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1834                   struct mlx5e_cq *cq)
1835 {
1836         struct mlx5_core_dev *mdev = priv->mdev;
1837         int err;
1838
1839         err = mlx5e_alloc_cq(priv, param, ccp, cq);
1840         if (err)
1841                 return err;
1842
1843         err = mlx5e_create_cq(cq, param);
1844         if (err)
1845                 goto err_free_cq;
1846
1847         if (MLX5_CAP_GEN(mdev, cq_moderation))
1848                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
1849         return 0;
1850
1851 err_free_cq:
1852         mlx5e_free_cq(cq);
1853
1854         return err;
1855 }
1856
1857 void mlx5e_close_cq(struct mlx5e_cq *cq)
1858 {
1859         mlx5e_destroy_cq(cq);
1860         mlx5e_free_cq(cq);
1861 }
1862
1863 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1864                              struct mlx5e_params *params,
1865                              struct mlx5e_create_cq_param *ccp,
1866                              struct mlx5e_channel_param *cparam)
1867 {
1868         int err;
1869         int tc;
1870
1871         for (tc = 0; tc < c->num_tc; tc++) {
1872                 err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp,
1873                                     ccp, &c->sq[tc].cq);
1874                 if (err)
1875                         goto err_close_tx_cqs;
1876         }
1877
1878         return 0;
1879
1880 err_close_tx_cqs:
1881         for (tc--; tc >= 0; tc--)
1882                 mlx5e_close_cq(&c->sq[tc].cq);
1883
1884         return err;
1885 }
1886
1887 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1888 {
1889         int tc;
1890
1891         for (tc = 0; tc < c->num_tc; tc++)
1892                 mlx5e_close_cq(&c->sq[tc].cq);
1893 }
1894
1895 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq)
1896 {
1897         int tc;
1898
1899         for (tc = 0; tc < TC_MAX_QUEUE; tc++)
1900                 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count)
1901                         return tc;
1902
1903         WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq);
1904         return -ENOENT;
1905 }
1906
1907 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix,
1908                                         u32 *hw_id)
1909 {
1910         int tc;
1911
1912         if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL ||
1913             !params->mqprio.channel.rl) {
1914                 *hw_id = 0;
1915                 return 0;
1916         }
1917
1918         tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix);
1919         if (tc < 0)
1920                 return tc;
1921
1922         return mlx5e_mqprio_rl_get_node_hw_id(params->mqprio.channel.rl, tc, hw_id);
1923 }
1924
1925 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1926                           struct mlx5e_params *params,
1927                           struct mlx5e_channel_param *cparam)
1928 {
1929         int err, tc;
1930
1931         for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) {
1932                 int txq_ix = c->ix + tc * params->num_channels;
1933                 u32 qos_queue_group_id;
1934
1935                 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id);
1936                 if (err)
1937                         goto err_close_sqs;
1938
1939                 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
1940                                        params, &cparam->txq_sq, &c->sq[tc], tc,
1941                                        qos_queue_group_id,
1942                                        &c->priv->channel_stats[c->ix]->sq[tc]);
1943                 if (err)
1944                         goto err_close_sqs;
1945         }
1946
1947         return 0;
1948
1949 err_close_sqs:
1950         for (tc--; tc >= 0; tc--)
1951                 mlx5e_close_txqsq(&c->sq[tc]);
1952
1953         return err;
1954 }
1955
1956 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1957 {
1958         int tc;
1959
1960         for (tc = 0; tc < c->num_tc; tc++)
1961                 mlx5e_close_txqsq(&c->sq[tc]);
1962 }
1963
1964 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1965                                 struct mlx5e_txqsq *sq, u32 rate)
1966 {
1967         struct mlx5e_priv *priv = netdev_priv(dev);
1968         struct mlx5_core_dev *mdev = priv->mdev;
1969         struct mlx5e_modify_sq_param msp = {0};
1970         struct mlx5_rate_limit rl = {0};
1971         u16 rl_index = 0;
1972         int err;
1973
1974         if (rate == sq->rate_limit)
1975                 /* nothing to do */
1976                 return 0;
1977
1978         if (sq->rate_limit) {
1979                 rl.rate = sq->rate_limit;
1980                 /* remove current rl index to free space to next ones */
1981                 mlx5_rl_remove_rate(mdev, &rl);
1982         }
1983
1984         sq->rate_limit = 0;
1985
1986         if (rate) {
1987                 rl.rate = rate;
1988                 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
1989                 if (err) {
1990                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1991                                    rate, err);
1992                         return err;
1993                 }
1994         }
1995
1996         msp.curr_state = MLX5_SQC_STATE_RDY;
1997         msp.next_state = MLX5_SQC_STATE_RDY;
1998         msp.rl_index   = rl_index;
1999         msp.rl_update  = true;
2000         err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
2001         if (err) {
2002                 netdev_err(dev, "Failed configuring rate %u: %d\n",
2003                            rate, err);
2004                 /* remove the rate from the table */
2005                 if (rate)
2006                         mlx5_rl_remove_rate(mdev, &rl);
2007                 return err;
2008         }
2009
2010         sq->rate_limit = rate;
2011         return 0;
2012 }
2013
2014 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
2015 {
2016         struct mlx5e_priv *priv = netdev_priv(dev);
2017         struct mlx5_core_dev *mdev = priv->mdev;
2018         struct mlx5e_txqsq *sq = priv->txq2sq[index];
2019         int err = 0;
2020
2021         if (!mlx5_rl_is_supported(mdev)) {
2022                 netdev_err(dev, "Rate limiting is not supported on this device\n");
2023                 return -EINVAL;
2024         }
2025
2026         /* rate is given in Mb/sec, HW config is in Kb/sec */
2027         rate = rate << 10;
2028
2029         /* Check whether rate in valid range, 0 is always valid */
2030         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
2031                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
2032                 return -ERANGE;
2033         }
2034
2035         mutex_lock(&priv->state_lock);
2036         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
2037                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
2038         if (!err)
2039                 priv->tx_rates[index] = rate;
2040         mutex_unlock(&priv->state_lock);
2041
2042         return err;
2043 }
2044
2045 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
2046                              struct mlx5e_rq_param *rq_params)
2047 {
2048         int err;
2049
2050         err = mlx5e_init_rxq_rq(c, params, &c->rq);
2051         if (err)
2052                 return err;
2053
2054         return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq);
2055 }
2056
2057 static int mlx5e_open_queues(struct mlx5e_channel *c,
2058                              struct mlx5e_params *params,
2059                              struct mlx5e_channel_param *cparam)
2060 {
2061         struct dim_cq_moder icocq_moder = {0, 0};
2062         struct mlx5e_create_cq_param ccp;
2063         int err;
2064
2065         mlx5e_build_create_cq_param(&ccp, c);
2066
2067         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp,
2068                             &c->async_icosq.cq);
2069         if (err)
2070                 return err;
2071
2072         err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp,
2073                             &c->icosq.cq);
2074         if (err)
2075                 goto err_close_async_icosq_cq;
2076
2077         err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
2078         if (err)
2079                 goto err_close_icosq_cq;
2080
2081         err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
2082                             &c->xdpsq.cq);
2083         if (err)
2084                 goto err_close_tx_cqs;
2085
2086         err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
2087                             &c->rq.cq);
2088         if (err)
2089                 goto err_close_xdp_tx_cqs;
2090
2091         err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
2092                                      &ccp, &c->rq_xdpsq.cq) : 0;
2093         if (err)
2094                 goto err_close_rx_cq;
2095
2096         spin_lock_init(&c->async_icosq_lock);
2097
2098         err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq,
2099                                mlx5e_async_icosq_err_cqe_work);
2100         if (err)
2101                 goto err_close_xdpsq_cq;
2102
2103         mutex_init(&c->icosq_recovery_lock);
2104
2105         err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq,
2106                                mlx5e_icosq_err_cqe_work);
2107         if (err)
2108                 goto err_close_async_icosq;
2109
2110         err = mlx5e_open_sqs(c, params, cparam);
2111         if (err)
2112                 goto err_close_icosq;
2113
2114         err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
2115         if (err)
2116                 goto err_close_sqs;
2117
2118         if (c->xdp) {
2119                 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
2120                                        &c->rq_xdpsq, false);
2121                 if (err)
2122                         goto err_close_rq;
2123         }
2124
2125         err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
2126         if (err)
2127                 goto err_close_xdp_sq;
2128
2129         return 0;
2130
2131 err_close_xdp_sq:
2132         if (c->xdp)
2133                 mlx5e_close_xdpsq(&c->rq_xdpsq);
2134
2135 err_close_rq:
2136         mlx5e_close_rq(&c->rq);
2137
2138 err_close_sqs:
2139         mlx5e_close_sqs(c);
2140
2141 err_close_icosq:
2142         mlx5e_close_icosq(&c->icosq);
2143
2144 err_close_async_icosq:
2145         mlx5e_close_icosq(&c->async_icosq);
2146
2147 err_close_xdpsq_cq:
2148         if (c->xdp)
2149                 mlx5e_close_cq(&c->rq_xdpsq.cq);
2150
2151 err_close_rx_cq:
2152         mlx5e_close_cq(&c->rq.cq);
2153
2154 err_close_xdp_tx_cqs:
2155         mlx5e_close_cq(&c->xdpsq.cq);
2156
2157 err_close_tx_cqs:
2158         mlx5e_close_tx_cqs(c);
2159
2160 err_close_icosq_cq:
2161         mlx5e_close_cq(&c->icosq.cq);
2162
2163 err_close_async_icosq_cq:
2164         mlx5e_close_cq(&c->async_icosq.cq);
2165
2166         return err;
2167 }
2168
2169 static void mlx5e_close_queues(struct mlx5e_channel *c)
2170 {
2171         mlx5e_close_xdpsq(&c->xdpsq);
2172         if (c->xdp)
2173                 mlx5e_close_xdpsq(&c->rq_xdpsq);
2174         /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */
2175         cancel_work_sync(&c->icosq.recover_work);
2176         mlx5e_close_rq(&c->rq);
2177         mlx5e_close_sqs(c);
2178         mlx5e_close_icosq(&c->icosq);
2179         mutex_destroy(&c->icosq_recovery_lock);
2180         mlx5e_close_icosq(&c->async_icosq);
2181         if (c->xdp)
2182                 mlx5e_close_cq(&c->rq_xdpsq.cq);
2183         mlx5e_close_cq(&c->rq.cq);
2184         mlx5e_close_cq(&c->xdpsq.cq);
2185         mlx5e_close_tx_cqs(c);
2186         mlx5e_close_cq(&c->icosq.cq);
2187         mlx5e_close_cq(&c->async_icosq.cq);
2188 }
2189
2190 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
2191 {
2192         u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
2193
2194         return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
2195 }
2196
2197 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
2198 {
2199         if (ix > priv->stats_nch)  {
2200                 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix,
2201                             priv->stats_nch);
2202                 return -EINVAL;
2203         }
2204
2205         if (priv->channel_stats[ix])
2206                 return 0;
2207
2208         /* Asymmetric dynamic memory allocation.
2209          * Freed in mlx5e_priv_arrays_free, not on channel closure.
2210          */
2211         mlx5e_dbg(DRV, priv, "Creating channel stats %d\n", ix);
2212         priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats),
2213                                                 GFP_KERNEL, cpu_to_node(cpu));
2214         if (!priv->channel_stats[ix])
2215                 return -ENOMEM;
2216         priv->stats_nch++;
2217
2218         return 0;
2219 }
2220
2221 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
2222                               struct mlx5e_params *params,
2223                               struct mlx5e_channel_param *cparam,
2224                               struct xsk_buff_pool *xsk_pool,
2225                               struct mlx5e_channel **cp)
2226 {
2227         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
2228         struct net_device *netdev = priv->netdev;
2229         struct mlx5e_xsk_param xsk;
2230         struct mlx5e_channel *c;
2231         unsigned int irq;
2232         int err;
2233
2234         err = mlx5_vector2irqn(priv->mdev, ix, &irq);
2235         if (err)
2236                 return err;
2237
2238         err = mlx5e_channel_stats_alloc(priv, ix, cpu);
2239         if (err)
2240                 return err;
2241
2242         c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
2243         if (!c)
2244                 return -ENOMEM;
2245
2246         c->priv     = priv;
2247         c->mdev     = priv->mdev;
2248         c->tstamp   = &priv->tstamp;
2249         c->ix       = ix;
2250         c->cpu      = cpu;
2251         c->pdev     = mlx5_core_dma_dev(priv->mdev);
2252         c->netdev   = priv->netdev;
2253         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
2254         c->num_tc   = mlx5e_get_dcb_num_tc(params);
2255         c->xdp      = !!params->xdp_prog;
2256         c->stats    = &priv->channel_stats[ix]->ch;
2257         c->aff_mask = irq_get_effective_affinity_mask(irq);
2258         c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
2259
2260         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
2261
2262         err = mlx5e_open_queues(c, params, cparam);
2263         if (unlikely(err))
2264                 goto err_napi_del;
2265
2266         if (xsk_pool) {
2267                 mlx5e_build_xsk_param(xsk_pool, &xsk);
2268                 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
2269                 if (unlikely(err))
2270                         goto err_close_queues;
2271         }
2272
2273         *cp = c;
2274
2275         return 0;
2276
2277 err_close_queues:
2278         mlx5e_close_queues(c);
2279
2280 err_napi_del:
2281         netif_napi_del(&c->napi);
2282
2283         kvfree(c);
2284
2285         return err;
2286 }
2287
2288 static void mlx5e_activate_channel(struct mlx5e_channel *c)
2289 {
2290         int tc;
2291
2292         napi_enable(&c->napi);
2293
2294         for (tc = 0; tc < c->num_tc; tc++)
2295                 mlx5e_activate_txqsq(&c->sq[tc]);
2296         mlx5e_activate_icosq(&c->icosq);
2297         mlx5e_activate_icosq(&c->async_icosq);
2298         mlx5e_activate_rq(&c->rq);
2299
2300         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2301                 mlx5e_activate_xsk(c);
2302 }
2303
2304 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2305 {
2306         int tc;
2307
2308         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2309                 mlx5e_deactivate_xsk(c);
2310
2311         mlx5e_deactivate_rq(&c->rq);
2312         mlx5e_deactivate_icosq(&c->async_icosq);
2313         mlx5e_deactivate_icosq(&c->icosq);
2314         for (tc = 0; tc < c->num_tc; tc++)
2315                 mlx5e_deactivate_txqsq(&c->sq[tc]);
2316         mlx5e_qos_deactivate_queues(c);
2317
2318         napi_disable(&c->napi);
2319 }
2320
2321 static void mlx5e_close_channel(struct mlx5e_channel *c)
2322 {
2323         if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2324                 mlx5e_close_xsk(c);
2325         mlx5e_close_queues(c);
2326         mlx5e_qos_close_queues(c);
2327         netif_napi_del(&c->napi);
2328
2329         kvfree(c);
2330 }
2331
2332 int mlx5e_open_channels(struct mlx5e_priv *priv,
2333                         struct mlx5e_channels *chs)
2334 {
2335         struct mlx5e_channel_param *cparam;
2336         int err = -ENOMEM;
2337         int i;
2338
2339         chs->num = chs->params.num_channels;
2340
2341         chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2342         cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2343         if (!chs->c || !cparam)
2344                 goto err_free;
2345
2346         err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam);
2347         if (err)
2348                 goto err_free;
2349
2350         for (i = 0; i < chs->num; i++) {
2351                 struct xsk_buff_pool *xsk_pool = NULL;
2352
2353                 if (chs->params.xdp_prog)
2354                         xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
2355
2356                 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
2357                 if (err)
2358                         goto err_close_channels;
2359         }
2360
2361         if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) {
2362                 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
2363                 if (err)
2364                         goto err_close_channels;
2365         }
2366
2367         err = mlx5e_qos_open_queues(priv, chs);
2368         if (err)
2369                 goto err_close_ptp;
2370
2371         mlx5e_health_channels_update(priv);
2372         kvfree(cparam);
2373         return 0;
2374
2375 err_close_ptp:
2376         if (chs->ptp)
2377                 mlx5e_ptp_close(chs->ptp);
2378
2379 err_close_channels:
2380         for (i--; i >= 0; i--)
2381                 mlx5e_close_channel(chs->c[i]);
2382
2383 err_free:
2384         kfree(chs->c);
2385         kvfree(cparam);
2386         chs->num = 0;
2387         return err;
2388 }
2389
2390 static void mlx5e_activate_channels(struct mlx5e_channels *chs)
2391 {
2392         int i;
2393
2394         for (i = 0; i < chs->num; i++)
2395                 mlx5e_activate_channel(chs->c[i]);
2396
2397         if (chs->ptp)
2398                 mlx5e_ptp_activate_channel(chs->ptp);
2399 }
2400
2401 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2402
2403 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2404 {
2405         int err = 0;
2406         int i;
2407
2408         for (i = 0; i < chs->num; i++) {
2409                 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2410
2411                 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
2412
2413                 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2414                  * doesn't provide any Fill Ring entries at the setup stage.
2415                  */
2416         }
2417
2418         return err ? -ETIMEDOUT : 0;
2419 }
2420
2421 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2422 {
2423         int i;
2424
2425         if (chs->ptp)
2426                 mlx5e_ptp_deactivate_channel(chs->ptp);
2427
2428         for (i = 0; i < chs->num; i++)
2429                 mlx5e_deactivate_channel(chs->c[i]);
2430 }
2431
2432 void mlx5e_close_channels(struct mlx5e_channels *chs)
2433 {
2434         int i;
2435
2436         if (chs->ptp) {
2437                 mlx5e_ptp_close(chs->ptp);
2438                 chs->ptp = NULL;
2439         }
2440         for (i = 0; i < chs->num; i++)
2441                 mlx5e_close_channel(chs->c[i]);
2442
2443         kfree(chs->c);
2444         chs->num = 0;
2445 }
2446
2447 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
2448 {
2449         struct mlx5e_rx_res *res = priv->rx_res;
2450
2451         return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge);
2452 }
2453
2454 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);
2455
2456 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2457                          struct mlx5e_params *params, u16 mtu)
2458 {
2459         u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2460         int err;
2461
2462         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2463         if (err)
2464                 return err;
2465
2466         /* Update vport context MTU */
2467         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2468         return 0;
2469 }
2470
2471 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2472                             struct mlx5e_params *params, u16 *mtu)
2473 {
2474         u16 hw_mtu = 0;
2475         int err;
2476
2477         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2478         if (err || !hw_mtu) /* fallback to port oper mtu */
2479                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2480
2481         *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2482 }
2483
2484 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2485 {
2486         struct mlx5e_params *params = &priv->channels.params;
2487         struct net_device *netdev = priv->netdev;
2488         struct mlx5_core_dev *mdev = priv->mdev;
2489         u16 mtu;
2490         int err;
2491
2492         err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2493         if (err)
2494                 return err;
2495
2496         mlx5e_query_mtu(mdev, params, &mtu);
2497         if (mtu != params->sw_mtu)
2498                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2499                             __func__, mtu, params->sw_mtu);
2500
2501         params->sw_mtu = mtu;
2502         return 0;
2503 }
2504
2505 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2506
2507 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2508 {
2509         struct mlx5e_params *params = &priv->channels.params;
2510         struct net_device *netdev   = priv->netdev;
2511         struct mlx5_core_dev *mdev  = priv->mdev;
2512         u16 max_mtu;
2513
2514         /* MTU range: 68 - hw-specific max */
2515         netdev->min_mtu = ETH_MIN_MTU;
2516
2517         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2518         netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2519                                 ETH_MAX_MTU);
2520 }
2521
2522 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc,
2523                                 struct netdev_tc_txq *tc_to_txq)
2524 {
2525         int tc, err;
2526
2527         netdev_reset_tc(netdev);
2528
2529         if (ntc == 1)
2530                 return 0;
2531
2532         err = netdev_set_num_tc(netdev, ntc);
2533         if (err) {
2534                 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc);
2535                 return err;
2536         }
2537
2538         for (tc = 0; tc < ntc; tc++) {
2539                 u16 count, offset;
2540
2541                 count = tc_to_txq[tc].count;
2542                 offset = tc_to_txq[tc].offset;
2543                 netdev_set_tc_queue(netdev, tc, count, offset);
2544         }
2545
2546         return 0;
2547 }
2548
2549 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
2550 {
2551         int qos_queues, nch, ntc, num_txqs, err;
2552
2553         qos_queues = mlx5e_qos_cur_leaf_nodes(priv);
2554
2555         nch = priv->channels.params.num_channels;
2556         ntc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2557         num_txqs = nch * ntc + qos_queues;
2558         if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
2559                 num_txqs += ntc;
2560
2561         mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs);
2562         err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
2563         if (err)
2564                 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
2565
2566         return err;
2567 }
2568
2569 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
2570 {
2571         struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq;
2572         struct net_device *netdev = priv->netdev;
2573         int old_num_txqs, old_ntc;
2574         int num_rxqs, nch, ntc;
2575         int err;
2576         int i;
2577
2578         old_num_txqs = netdev->real_num_tx_queues;
2579         old_ntc = netdev->num_tc ? : 1;
2580         for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++)
2581                 old_tc_to_txq[i] = netdev->tc_to_txq[i];
2582
2583         nch = priv->channels.params.num_channels;
2584         ntc = priv->channels.params.mqprio.num_tc;
2585         num_rxqs = nch * priv->profile->rq_groups;
2586         tc_to_txq = priv->channels.params.mqprio.tc_to_txq;
2587
2588         err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq);
2589         if (err)
2590                 goto err_out;
2591         err = mlx5e_update_tx_netdev_queues(priv);
2592         if (err)
2593                 goto err_tcs;
2594         err = netif_set_real_num_rx_queues(netdev, num_rxqs);
2595         if (err) {
2596                 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
2597                 goto err_txqs;
2598         }
2599         if (priv->mqprio_rl != priv->channels.params.mqprio.channel.rl) {
2600                 if (priv->mqprio_rl) {
2601                         mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
2602                         mlx5e_mqprio_rl_free(priv->mqprio_rl);
2603                 }
2604                 priv->mqprio_rl = priv->channels.params.mqprio.channel.rl;
2605         }
2606
2607         return 0;
2608
2609 err_txqs:
2610         /* netif_set_real_num_rx_queues could fail only when nch increased. Only
2611          * one of nch and ntc is changed in this function. That means, the call
2612          * to netif_set_real_num_tx_queues below should not fail, because it
2613          * decreases the number of TX queues.
2614          */
2615         WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs));
2616
2617 err_tcs:
2618         WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc,
2619                                           old_tc_to_txq));
2620 err_out:
2621         return err;
2622 }
2623
2624 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues);
2625
2626 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
2627                                            struct mlx5e_params *params)
2628 {
2629         struct mlx5_core_dev *mdev = priv->mdev;
2630         int num_comp_vectors, ix, irq;
2631
2632         num_comp_vectors = mlx5_comp_vectors_count(mdev);
2633
2634         for (ix = 0; ix < params->num_channels; ix++) {
2635                 cpumask_clear(priv->scratchpad.cpumask);
2636
2637                 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) {
2638                         int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq));
2639
2640                         cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
2641                 }
2642
2643                 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
2644         }
2645 }
2646
2647 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
2648 {
2649         u16 count = priv->channels.params.num_channels;
2650         int err;
2651
2652         err = mlx5e_update_netdev_queues(priv);
2653         if (err)
2654                 return err;
2655
2656         mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
2657
2658         /* This function may be called on attach, before priv->rx_res is created. */
2659         if (!netif_is_rxfh_configured(priv->netdev) && priv->rx_res)
2660                 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count);
2661
2662         return 0;
2663 }
2664
2665 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
2666
2667 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
2668 {
2669         int i, ch, tc, num_tc;
2670
2671         ch = priv->channels.num;
2672         num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
2673
2674         for (i = 0; i < ch; i++) {
2675                 for (tc = 0; tc < num_tc; tc++) {
2676                         struct mlx5e_channel *c = priv->channels.c[i];
2677                         struct mlx5e_txqsq *sq = &c->sq[tc];
2678
2679                         priv->txq2sq[sq->txq_ix] = sq;
2680                         priv->channel_tc2realtxq[i][tc] = i + tc * ch;
2681                 }
2682         }
2683
2684         if (!priv->channels.ptp)
2685                 return;
2686
2687         if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state))
2688                 return;
2689
2690         for (tc = 0; tc < num_tc; tc++) {
2691                 struct mlx5e_ptp *c = priv->channels.ptp;
2692                 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
2693
2694                 priv->txq2sq[sq->txq_ix] = sq;
2695                 priv->port_ptp_tc2realtxq[tc] = priv->num_tc_x_num_ch + tc;
2696         }
2697 }
2698
2699 static void mlx5e_update_num_tc_x_num_ch(struct mlx5e_priv *priv)
2700 {
2701         /* Sync with mlx5e_select_queue. */
2702         WRITE_ONCE(priv->num_tc_x_num_ch,
2703                    mlx5e_get_dcb_num_tc(&priv->channels.params) * priv->channels.num);
2704 }
2705
2706 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2707 {
2708         mlx5e_update_num_tc_x_num_ch(priv);
2709         mlx5e_build_txq_maps(priv);
2710         mlx5e_activate_channels(&priv->channels);
2711         mlx5e_qos_activate_queues(priv);
2712         mlx5e_xdp_tx_enable(priv);
2713         netif_tx_start_all_queues(priv->netdev);
2714
2715         if (mlx5e_is_vport_rep(priv))
2716                 mlx5e_add_sqs_fwd_rules(priv);
2717
2718         mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2719
2720         if (priv->rx_res)
2721                 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels);
2722 }
2723
2724 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2725 {
2726         if (priv->rx_res)
2727                 mlx5e_rx_res_channels_deactivate(priv->rx_res);
2728
2729         if (mlx5e_is_vport_rep(priv))
2730                 mlx5e_remove_sqs_fwd_rules(priv);
2731
2732         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2733          * polling for inactive tx queues.
2734          */
2735         netif_tx_stop_all_queues(priv->netdev);
2736         netif_tx_disable(priv->netdev);
2737         mlx5e_xdp_tx_disable(priv);
2738         mlx5e_deactivate_channels(&priv->channels);
2739 }
2740
2741 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
2742                                     struct mlx5e_params *new_params,
2743                                     mlx5e_fp_preactivate preactivate,
2744                                     void *context)
2745 {
2746         struct mlx5e_params old_params;
2747
2748         old_params = priv->channels.params;
2749         priv->channels.params = *new_params;
2750
2751         if (preactivate) {
2752                 int err;
2753
2754                 err = preactivate(priv, context);
2755                 if (err) {
2756                         priv->channels.params = old_params;
2757                         return err;
2758                 }
2759         }
2760
2761         return 0;
2762 }
2763
2764 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2765                                       struct mlx5e_channels *new_chs,
2766                                       mlx5e_fp_preactivate preactivate,
2767                                       void *context)
2768 {
2769         struct net_device *netdev = priv->netdev;
2770         struct mlx5e_channels old_chs;
2771         int carrier_ok;
2772         int err = 0;
2773
2774         carrier_ok = netif_carrier_ok(netdev);
2775         netif_carrier_off(netdev);
2776
2777         mlx5e_deactivate_priv_channels(priv);
2778
2779         old_chs = priv->channels;
2780         priv->channels = *new_chs;
2781
2782         /* New channels are ready to roll, call the preactivate hook if needed
2783          * to modify HW settings or update kernel parameters.
2784          */
2785         if (preactivate) {
2786                 err = preactivate(priv, context);
2787                 if (err) {
2788                         priv->channels = old_chs;
2789                         goto out;
2790                 }
2791         }
2792
2793         mlx5e_close_channels(&old_chs);
2794         priv->profile->update_rx(priv);
2795
2796 out:
2797         mlx5e_activate_priv_channels(priv);
2798
2799         /* return carrier back if needed */
2800         if (carrier_ok)
2801                 netif_carrier_on(netdev);
2802
2803         return err;
2804 }
2805
2806 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
2807                              struct mlx5e_params *params,
2808                              mlx5e_fp_preactivate preactivate,
2809                              void *context, bool reset)
2810 {
2811         struct mlx5e_channels new_chs = {};
2812         int err;
2813
2814         reset &= test_bit(MLX5E_STATE_OPENED, &priv->state);
2815         if (!reset)
2816                 return mlx5e_switch_priv_params(priv, params, preactivate, context);
2817
2818         new_chs.params = *params;
2819         err = mlx5e_open_channels(priv, &new_chs);
2820         if (err)
2821                 return err;
2822         err = mlx5e_switch_priv_channels(priv, &new_chs, preactivate, context);
2823         if (err)
2824                 mlx5e_close_channels(&new_chs);
2825
2826         return err;
2827 }
2828
2829 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
2830 {
2831         return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true);
2832 }
2833
2834 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
2835 {
2836         priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
2837         priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
2838 }
2839
2840 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
2841                                      enum mlx5_port_status state)
2842 {
2843         struct mlx5_eswitch *esw = mdev->priv.eswitch;
2844         int vport_admin_state;
2845
2846         mlx5_set_port_admin_status(mdev, state);
2847
2848         if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
2849             !MLX5_CAP_GEN(mdev, uplink_follow))
2850                 return;
2851
2852         if (state == MLX5_PORT_UP)
2853                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
2854         else
2855                 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
2856
2857         mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
2858 }
2859
2860 int mlx5e_open_locked(struct net_device *netdev)
2861 {
2862         struct mlx5e_priv *priv = netdev_priv(netdev);
2863         int err;
2864
2865         set_bit(MLX5E_STATE_OPENED, &priv->state);
2866
2867         err = mlx5e_open_channels(priv, &priv->channels);
2868         if (err)
2869                 goto err_clear_state_opened_flag;
2870
2871         priv->profile->update_rx(priv);
2872         mlx5e_activate_priv_channels(priv);
2873         mlx5e_apply_traps(priv, true);
2874         if (priv->profile->update_carrier)
2875                 priv->profile->update_carrier(priv);
2876
2877         mlx5e_queue_update_stats(priv);
2878         return 0;
2879
2880 err_clear_state_opened_flag:
2881         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2882         return err;
2883 }
2884
2885 int mlx5e_open(struct net_device *netdev)
2886 {
2887         struct mlx5e_priv *priv = netdev_priv(netdev);
2888         int err;
2889
2890         mutex_lock(&priv->state_lock);
2891         err = mlx5e_open_locked(netdev);
2892         if (!err)
2893                 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
2894         mutex_unlock(&priv->state_lock);
2895
2896         return err;
2897 }
2898
2899 int mlx5e_close_locked(struct net_device *netdev)
2900 {
2901         struct mlx5e_priv *priv = netdev_priv(netdev);
2902
2903         /* May already be CLOSED in case a previous configuration operation
2904          * (e.g RX/TX queue size change) that involves close&open failed.
2905          */
2906         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2907                 return 0;
2908
2909         mlx5e_apply_traps(priv, false);
2910         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2911
2912         netif_carrier_off(priv->netdev);
2913         mlx5e_deactivate_priv_channels(priv);
2914         mlx5e_close_channels(&priv->channels);
2915
2916         return 0;
2917 }
2918
2919 int mlx5e_close(struct net_device *netdev)
2920 {
2921         struct mlx5e_priv *priv = netdev_priv(netdev);
2922         int err;
2923
2924         if (!netif_device_present(netdev))
2925                 return -ENODEV;
2926
2927         mutex_lock(&priv->state_lock);
2928         mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
2929         err = mlx5e_close_locked(netdev);
2930         mutex_unlock(&priv->state_lock);
2931
2932         return err;
2933 }
2934
2935 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
2936 {
2937         mlx5_wq_destroy(&rq->wq_ctrl);
2938 }
2939
2940 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
2941                                struct mlx5e_rq *rq,
2942                                struct mlx5e_rq_param *param)
2943 {
2944         void *rqc = param->rqc;
2945         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2946         int err;
2947
2948         param->wq.db_numa_node = param->wq.buf_numa_node;
2949
2950         err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
2951                                  &rq->wq_ctrl);
2952         if (err)
2953                 return err;
2954
2955         /* Mark as unused given "Drop-RQ" packets never reach XDP */
2956         xdp_rxq_info_unused(&rq->xdp_rxq);
2957
2958         rq->mdev = mdev;
2959
2960         return 0;
2961 }
2962
2963 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
2964                                struct mlx5e_cq *cq,
2965                                struct mlx5e_cq_param *param)
2966 {
2967         struct mlx5_core_dev *mdev = priv->mdev;
2968
2969         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
2970         param->wq.db_numa_node  = dev_to_node(mlx5_core_dma_dev(mdev));
2971
2972         return mlx5e_alloc_cq_common(priv, param, cq);
2973 }
2974
2975 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
2976                        struct mlx5e_rq *drop_rq)
2977 {
2978         struct mlx5_core_dev *mdev = priv->mdev;
2979         struct mlx5e_cq_param cq_param = {};
2980         struct mlx5e_rq_param rq_param = {};
2981         struct mlx5e_cq *cq = &drop_rq->cq;
2982         int err;
2983
2984         mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param);
2985
2986         err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
2987         if (err)
2988                 return err;
2989
2990         err = mlx5e_create_cq(cq, &cq_param);
2991         if (err)
2992                 goto err_free_cq;
2993
2994         err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
2995         if (err)
2996                 goto err_destroy_cq;
2997
2998         err = mlx5e_create_rq(drop_rq, &rq_param);
2999         if (err)
3000                 goto err_free_rq;
3001
3002         err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3003         if (err)
3004                 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3005
3006         return 0;
3007
3008 err_free_rq:
3009         mlx5e_free_drop_rq(drop_rq);
3010
3011 err_destroy_cq:
3012         mlx5e_destroy_cq(cq);
3013
3014 err_free_cq:
3015         mlx5e_free_cq(cq);
3016
3017         return err;
3018 }
3019
3020 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3021 {
3022         mlx5e_destroy_rq(drop_rq);
3023         mlx5e_free_drop_rq(drop_rq);
3024         mlx5e_destroy_cq(&drop_rq->cq);
3025         mlx5e_free_cq(&drop_rq->cq);
3026 }
3027
3028 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn)
3029 {
3030         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3031
3032         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
3033
3034         if (MLX5_GET(tisc, tisc, tls_en))
3035                 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn);
3036
3037         if (mlx5_lag_is_lacp_owner(mdev))
3038                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3039
3040         return mlx5_core_create_tis(mdev, in, tisn);
3041 }
3042
3043 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3044 {
3045         mlx5_core_destroy_tis(mdev, tisn);
3046 }
3047
3048 void mlx5e_destroy_tises(struct mlx5e_priv *priv)
3049 {
3050         int tc, i;
3051
3052         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++)
3053                 for (tc = 0; tc < priv->profile->max_tc; tc++)
3054                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3055 }
3056
3057 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev)
3058 {
3059         return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1;
3060 }
3061
3062 int mlx5e_create_tises(struct mlx5e_priv *priv)
3063 {
3064         int tc, i;
3065         int err;
3066
3067         for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
3068                 for (tc = 0; tc < priv->profile->max_tc; tc++) {
3069                         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
3070                         void *tisc;
3071
3072                         tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3073
3074                         MLX5_SET(tisc, tisc, prio, tc << 1);
3075
3076                         if (mlx5e_lag_should_assign_affinity(priv->mdev))
3077                                 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1);
3078
3079                         err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]);
3080                         if (err)
3081                                 goto err_close_tises;
3082                 }
3083         }
3084
3085         return 0;
3086
3087 err_close_tises:
3088         for (; i >= 0; i--) {
3089                 for (tc--; tc >= 0; tc--)
3090                         mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3091                 tc = priv->profile->max_tc;
3092         }
3093
3094         return err;
3095 }
3096
3097 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3098 {
3099         mlx5e_destroy_tises(priv);
3100 }
3101
3102 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3103 {
3104         int err = 0;
3105         int i;
3106
3107         for (i = 0; i < chs->num; i++) {
3108                 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3109                 if (err)
3110                         return err;
3111         }
3112
3113         return 0;
3114 }
3115
3116 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3117 {
3118         int err;
3119         int i;
3120
3121         for (i = 0; i < chs->num; i++) {
3122                 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3123                 if (err)
3124                         return err;
3125         }
3126         if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state))
3127                 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd);
3128
3129         return 0;
3130 }
3131
3132 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3133                                                  int ntc, int nch)
3134 {
3135         int tc;
3136
3137         memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE);
3138
3139         /* Map netdev TCs to offset 0.
3140          * We have our own UP to TXQ mapping for DCB mode of QoS
3141          */
3142         for (tc = 0; tc < ntc; tc++) {
3143                 tc_to_txq[tc] = (struct netdev_tc_txq) {
3144                         .count = nch,
3145                         .offset = 0,
3146                 };
3147         }
3148 }
3149
3150 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3151                                          struct tc_mqprio_qopt *qopt)
3152 {
3153         int tc;
3154
3155         for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3156                 tc_to_txq[tc] = (struct netdev_tc_txq) {
3157                         .count = qopt->count[tc],
3158                         .offset = qopt->offset[tc],
3159                 };
3160         }
3161 }
3162
3163 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc)
3164 {
3165         params->mqprio.mode = TC_MQPRIO_MODE_DCB;
3166         params->mqprio.num_tc = num_tc;
3167         params->mqprio.channel.rl = NULL;
3168         mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc,
3169                                              params->num_channels);
3170 }
3171
3172 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params,
3173                                             struct tc_mqprio_qopt *qopt,
3174                                             struct mlx5e_mqprio_rl *rl)
3175 {
3176         params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL;
3177         params->mqprio.num_tc = qopt->num_tc;
3178         params->mqprio.channel.rl = rl;
3179         mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, qopt);
3180 }
3181
3182 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params)
3183 {
3184         mlx5e_params_mqprio_dcb_set(params, 1);
3185 }
3186
3187 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv,
3188                                      struct tc_mqprio_qopt *mqprio)
3189 {
3190         struct mlx5e_params new_params;
3191         u8 tc = mqprio->num_tc;
3192         int err;
3193
3194         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3195
3196         if (tc && tc != MLX5E_MAX_NUM_TC)
3197                 return -EINVAL;
3198
3199         new_params = priv->channels.params;
3200         mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1);
3201
3202         err = mlx5e_safe_switch_params(priv, &new_params,
3203                                        mlx5e_num_channels_changed_ctx, NULL, true);
3204
3205         priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3206                                     mlx5e_get_dcb_num_tc(&priv->channels.params));
3207         return err;
3208 }
3209
3210 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv,
3211                                          struct tc_mqprio_qopt_offload *mqprio)
3212 {
3213         struct net_device *netdev = priv->netdev;
3214         struct mlx5e_ptp *ptp_channel;
3215         int agg_count = 0;
3216         int i;
3217
3218         ptp_channel = priv->channels.ptp;
3219         if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) {
3220                 netdev_err(netdev,
3221                            "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n");
3222                 return -EINVAL;
3223         }
3224
3225         if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 ||
3226             mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC)
3227                 return -EINVAL;
3228
3229         for (i = 0; i < mqprio->qopt.num_tc; i++) {
3230                 if (!mqprio->qopt.count[i]) {
3231                         netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i);
3232                         return -EINVAL;
3233                 }
3234                 if (mqprio->min_rate[i]) {
3235                         netdev_err(netdev, "Min tx rate is not supported\n");
3236                         return -EINVAL;
3237                 }
3238
3239                 if (mqprio->max_rate[i]) {
3240                         int err;
3241
3242                         err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]);
3243                         if (err)
3244                                 return err;
3245                 }
3246
3247                 if (mqprio->qopt.offset[i] != agg_count) {
3248                         netdev_err(netdev, "Discontinuous queues config is not supported\n");
3249                         return -EINVAL;
3250                 }
3251                 agg_count += mqprio->qopt.count[i];
3252         }
3253
3254         if (priv->channels.params.num_channels != agg_count) {
3255                 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n",
3256                            agg_count, priv->channels.params.num_channels);
3257                 return -EINVAL;
3258         }
3259
3260         return 0;
3261 }
3262
3263 static bool mlx5e_mqprio_rate_limit(struct tc_mqprio_qopt_offload *mqprio)
3264 {
3265         int tc;
3266
3267         for (tc = 0; tc < mqprio->qopt.num_tc; tc++)
3268                 if (mqprio->max_rate[tc])
3269                         return true;
3270         return false;
3271 }
3272
3273 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv,
3274                                          struct tc_mqprio_qopt_offload *mqprio)
3275 {
3276         mlx5e_fp_preactivate preactivate;
3277         struct mlx5e_params new_params;
3278         struct mlx5e_mqprio_rl *rl;
3279         bool nch_changed;
3280         int err;
3281
3282         err = mlx5e_mqprio_channel_validate(priv, mqprio);
3283         if (err)
3284                 return err;
3285
3286         rl = NULL;
3287         if (mlx5e_mqprio_rate_limit(mqprio)) {
3288                 rl = mlx5e_mqprio_rl_alloc();
3289                 if (!rl)
3290                         return -ENOMEM;
3291                 err = mlx5e_mqprio_rl_init(rl, priv->mdev, mqprio->qopt.num_tc,
3292                                            mqprio->max_rate);
3293                 if (err) {
3294                         mlx5e_mqprio_rl_free(rl);
3295                         return err;
3296                 }
3297         }
3298
3299         new_params = priv->channels.params;
3300         mlx5e_params_mqprio_channel_set(&new_params, &mqprio->qopt, rl);
3301
3302         nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1;
3303         preactivate = nch_changed ? mlx5e_num_channels_changed_ctx :
3304                 mlx5e_update_netdev_queues_ctx;
3305         err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true);
3306         if (err && rl) {
3307                 mlx5e_mqprio_rl_cleanup(rl);
3308                 mlx5e_mqprio_rl_free(rl);
3309         }
3310
3311         return err;
3312 }
3313
3314 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3315                                  struct tc_mqprio_qopt_offload *mqprio)
3316 {
3317         /* MQPRIO is another toplevel qdisc that can't be attached
3318          * simultaneously with the offloaded HTB.
3319          */
3320         if (WARN_ON(priv->htb.maj_id))
3321                 return -EINVAL;
3322
3323         switch (mqprio->mode) {
3324         case TC_MQPRIO_MODE_DCB:
3325                 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt);
3326         case TC_MQPRIO_MODE_CHANNEL:
3327                 return mlx5e_setup_tc_mqprio_channel(priv, mqprio);
3328         default:
3329                 return -EOPNOTSUPP;
3330         }
3331 }
3332
3333 static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb)
3334 {
3335         int res;
3336
3337         switch (htb->command) {
3338         case TC_HTB_CREATE:
3339                 return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid,
3340                                           htb->extack);
3341         case TC_HTB_DESTROY:
3342                 return mlx5e_htb_root_del(priv);
3343         case TC_HTB_LEAF_ALLOC_QUEUE:
3344                 res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid,
3345                                                  htb->rate, htb->ceil, htb->extack);
3346                 if (res < 0)
3347                         return res;
3348                 htb->qid = res;
3349                 return 0;
3350         case TC_HTB_LEAF_TO_INNER:
3351                 return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid,
3352                                                htb->rate, htb->ceil, htb->extack);
3353         case TC_HTB_LEAF_DEL:
3354                 return mlx5e_htb_leaf_del(priv, &htb->classid, htb->extack);
3355         case TC_HTB_LEAF_DEL_LAST:
3356         case TC_HTB_LEAF_DEL_LAST_FORCE:
3357                 return mlx5e_htb_leaf_del_last(priv, htb->classid,
3358                                                htb->command == TC_HTB_LEAF_DEL_LAST_FORCE,
3359                                                htb->extack);
3360         case TC_HTB_NODE_MODIFY:
3361                 return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil,
3362                                              htb->extack);
3363         case TC_HTB_LEAF_QUERY_QUEUE:
3364                 res = mlx5e_get_txq_by_classid(priv, htb->classid);
3365                 if (res < 0)
3366                         return res;
3367                 htb->qid = res;
3368                 return 0;
3369         default:
3370                 return -EOPNOTSUPP;
3371         }
3372 }
3373
3374 static LIST_HEAD(mlx5e_block_cb_list);
3375
3376 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3377                           void *type_data)
3378 {
3379         struct mlx5e_priv *priv = netdev_priv(dev);
3380         bool tc_unbind = false;
3381         int err;
3382
3383         if (type == TC_SETUP_BLOCK &&
3384             ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND)
3385                 tc_unbind = true;
3386
3387         if (!netif_device_present(dev) && !tc_unbind)
3388                 return -ENODEV;
3389
3390         switch (type) {
3391         case TC_SETUP_BLOCK: {
3392                 struct flow_block_offload *f = type_data;
3393
3394                 f->unlocked_driver_cb = true;
3395                 return flow_block_cb_setup_simple(type_data,
3396                                                   &mlx5e_block_cb_list,
3397                                                   mlx5e_setup_tc_block_cb,
3398                                                   priv, priv, true);
3399         }
3400         case TC_SETUP_QDISC_MQPRIO:
3401                 mutex_lock(&priv->state_lock);
3402                 err = mlx5e_setup_tc_mqprio(priv, type_data);
3403                 mutex_unlock(&priv->state_lock);
3404                 return err;
3405         case TC_SETUP_QDISC_HTB:
3406                 mutex_lock(&priv->state_lock);
3407                 err = mlx5e_setup_tc_htb(priv, type_data);
3408                 mutex_unlock(&priv->state_lock);
3409                 return err;
3410         default:
3411                 return -EOPNOTSUPP;
3412         }
3413 }
3414
3415 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
3416 {
3417         int i;
3418
3419         for (i = 0; i < priv->stats_nch; i++) {
3420                 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
3421                 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3422                 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3423                 int j;
3424
3425                 s->rx_packets   += rq_stats->packets + xskrq_stats->packets;
3426                 s->rx_bytes     += rq_stats->bytes + xskrq_stats->bytes;
3427                 s->multicast    += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
3428
3429                 for (j = 0; j < priv->max_opened_tc; j++) {
3430                         struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3431
3432                         s->tx_packets    += sq_stats->packets;
3433                         s->tx_bytes      += sq_stats->bytes;
3434                         s->tx_dropped    += sq_stats->dropped;
3435                 }
3436         }
3437         if (priv->tx_ptp_opened) {
3438                 for (i = 0; i < priv->max_opened_tc; i++) {
3439                         struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i];
3440
3441                         s->tx_packets    += sq_stats->packets;
3442                         s->tx_bytes      += sq_stats->bytes;
3443                         s->tx_dropped    += sq_stats->dropped;
3444                 }
3445         }
3446         if (priv->rx_ptp_opened) {
3447                 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
3448
3449                 s->rx_packets   += rq_stats->packets;
3450                 s->rx_bytes     += rq_stats->bytes;
3451                 s->multicast    += rq_stats->mcast_packets;
3452         }
3453 }
3454
3455 void
3456 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3457 {
3458         struct mlx5e_priv *priv = netdev_priv(dev);
3459         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3460
3461         if (!netif_device_present(dev))
3462                 return;
3463
3464         /* In switchdev mode, monitor counters doesn't monitor
3465          * rx/tx stats of 802_3. The update stats mechanism
3466          * should keep the 802_3 layout counters updated
3467          */
3468         if (!mlx5e_monitor_counter_supported(priv) ||
3469             mlx5e_is_uplink_rep(priv)) {
3470                 /* update HW stats in background for next time */
3471                 mlx5e_queue_update_stats(priv);
3472         }
3473
3474         if (mlx5e_is_uplink_rep(priv)) {
3475                 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3476
3477                 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3478                 stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
3479                 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3480                 stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3481
3482                 /* vport multicast also counts packets that are dropped due to steering
3483                  * or rx out of buffer
3484                  */
3485                 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
3486         } else {
3487                 mlx5e_fold_sw_stats64(priv, stats);
3488         }
3489
3490         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3491
3492         stats->rx_length_errors =
3493                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3494                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3495                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
3496         stats->rx_crc_errors =
3497                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3498         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3499         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3500         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3501                            stats->rx_frame_errors;
3502         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3503 }
3504
3505 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
3506 {
3507         if (mlx5e_is_uplink_rep(priv))
3508                 return; /* no rx mode for uplink rep */
3509
3510         queue_work(priv->wq, &priv->set_rx_mode_work);
3511 }
3512
3513 static void mlx5e_set_rx_mode(struct net_device *dev)
3514 {
3515         struct mlx5e_priv *priv = netdev_priv(dev);
3516
3517         mlx5e_nic_set_rx_mode(priv);
3518 }
3519
3520 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3521 {
3522         struct mlx5e_priv *priv = netdev_priv(netdev);
3523         struct sockaddr *saddr = addr;
3524
3525         if (!is_valid_ether_addr(saddr->sa_data))
3526                 return -EADDRNOTAVAIL;
3527
3528         netif_addr_lock_bh(netdev);
3529         eth_hw_addr_set(netdev, saddr->sa_data);
3530         netif_addr_unlock_bh(netdev);
3531
3532         mlx5e_nic_set_rx_mode(priv);
3533
3534         return 0;
3535 }
3536
3537 #define MLX5E_SET_FEATURE(features, feature, enable)    \
3538         do {                                            \
3539                 if (enable)                             \
3540                         *features |= feature;           \
3541                 else                                    \
3542                         *features &= ~feature;          \
3543         } while (0)
3544
3545 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3546
3547 static int set_feature_lro(struct net_device *netdev, bool enable)
3548 {
3549         struct mlx5e_priv *priv = netdev_priv(netdev);
3550         struct mlx5_core_dev *mdev = priv->mdev;
3551         struct mlx5e_params *cur_params;
3552         struct mlx5e_params new_params;
3553         bool reset = true;
3554         int err = 0;
3555
3556         mutex_lock(&priv->state_lock);
3557
3558         if (enable && priv->xsk.refcnt) {
3559                 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
3560                             priv->xsk.refcnt);
3561                 err = -EINVAL;
3562                 goto out;
3563         }
3564
3565         cur_params = &priv->channels.params;
3566         if (enable && !MLX5E_GET_PFLAG(cur_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3567                 netdev_warn(netdev, "can't set LRO with legacy RQ\n");
3568                 err = -EINVAL;
3569                 goto out;
3570         }
3571
3572         new_params = *cur_params;
3573
3574         if (enable)
3575                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO;
3576         else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)
3577                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3578         else
3579                 goto out;
3580
3581         if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO &&
3582               new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) {
3583                 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3584                         if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
3585                             mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
3586                                 reset = false;
3587                 }
3588         }
3589
3590         err = mlx5e_safe_switch_params(priv, &new_params,
3591                                        mlx5e_modify_tirs_packet_merge_ctx, NULL, reset);
3592 out:
3593         mutex_unlock(&priv->state_lock);
3594         return err;
3595 }
3596
3597 static int set_feature_hw_gro(struct net_device *netdev, bool enable)
3598 {
3599         struct mlx5e_priv *priv = netdev_priv(netdev);
3600         struct mlx5e_params new_params;
3601         bool reset = true;
3602         int err = 0;
3603
3604         mutex_lock(&priv->state_lock);
3605         new_params = priv->channels.params;
3606
3607         if (enable) {
3608                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO;
3609                 new_params.packet_merge.shampo.match_criteria_type =
3610                         MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED;
3611                 new_params.packet_merge.shampo.alignment_granularity =
3612                         MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE;
3613         } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
3614                 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3615         } else {
3616                 goto out;
3617         }
3618
3619         err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
3620 out:
3621         mutex_unlock(&priv->state_lock);
3622         return err;
3623 }
3624
3625 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3626 {
3627         struct mlx5e_priv *priv = netdev_priv(netdev);
3628
3629         if (enable)
3630                 mlx5e_enable_cvlan_filter(priv);
3631         else
3632                 mlx5e_disable_cvlan_filter(priv);
3633
3634         return 0;
3635 }
3636
3637 static int set_feature_hw_tc(struct net_device *netdev, bool enable)
3638 {
3639         struct mlx5e_priv *priv = netdev_priv(netdev);
3640
3641 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3642         if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) {
3643                 netdev_err(netdev,
3644                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3645                 return -EINVAL;
3646         }
3647 #endif
3648
3649         if (!enable && priv->htb.maj_id) {
3650                 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
3651                 return -EINVAL;
3652         }
3653
3654         return 0;
3655 }
3656
3657 static int set_feature_rx_all(struct net_device *netdev, bool enable)
3658 {
3659         struct mlx5e_priv *priv = netdev_priv(netdev);
3660         struct mlx5_core_dev *mdev = priv->mdev;
3661
3662         return mlx5_set_port_fcs(mdev, !enable);
3663 }
3664
3665 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable)
3666 {
3667         u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {};
3668         bool supported, curr_state;
3669         int err;
3670
3671         if (!MLX5_CAP_GEN(mdev, ports_check))
3672                 return 0;
3673
3674         err = mlx5_query_ports_check(mdev, in, sizeof(in));
3675         if (err)
3676                 return err;
3677
3678         supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap);
3679         curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc);
3680
3681         if (!supported || enable == curr_state)
3682                 return 0;
3683
3684         MLX5_SET(pcmr_reg, in, local_port, 1);
3685         MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable);
3686
3687         return mlx5_set_ports_check(mdev, in, sizeof(in));
3688 }
3689
3690 static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3691 {
3692         struct mlx5e_priv *priv = netdev_priv(netdev);
3693         struct mlx5e_channels *chs = &priv->channels;
3694         struct mlx5_core_dev *mdev = priv->mdev;
3695         int err;
3696
3697         mutex_lock(&priv->state_lock);
3698
3699         if (enable) {
3700                 err = mlx5e_set_rx_port_ts(mdev, false);
3701                 if (err)
3702                         goto out;
3703
3704                 chs->params.scatter_fcs_en = true;
3705                 err = mlx5e_modify_channels_scatter_fcs(chs, true);
3706                 if (err) {
3707                         chs->params.scatter_fcs_en = false;
3708                         mlx5e_set_rx_port_ts(mdev, true);
3709                 }
3710         } else {
3711                 chs->params.scatter_fcs_en = false;
3712                 err = mlx5e_modify_channels_scatter_fcs(chs, false);
3713                 if (err) {
3714                         chs->params.scatter_fcs_en = true;
3715                         goto out;
3716                 }
3717                 err = mlx5e_set_rx_port_ts(mdev, true);
3718                 if (err) {
3719                         mlx5_core_warn(mdev, "Failed to set RX port timestamp %d\n", err);
3720                         err = 0;
3721                 }
3722         }
3723
3724 out:
3725         mutex_unlock(&priv->state_lock);
3726         return err;
3727 }
3728
3729 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3730 {
3731         struct mlx5e_priv *priv = netdev_priv(netdev);
3732         int err = 0;
3733
3734         mutex_lock(&priv->state_lock);
3735
3736         priv->channels.params.vlan_strip_disable = !enable;
3737         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3738                 goto unlock;
3739
3740         err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
3741         if (err)
3742                 priv->channels.params.vlan_strip_disable = enable;
3743
3744 unlock:
3745         mutex_unlock(&priv->state_lock);
3746
3747         return err;
3748 }
3749
3750 #ifdef CONFIG_MLX5_EN_ARFS
3751 static int set_feature_arfs(struct net_device *netdev, bool enable)
3752 {
3753         struct mlx5e_priv *priv = netdev_priv(netdev);
3754         int err;
3755
3756         if (enable)
3757                 err = mlx5e_arfs_enable(priv);
3758         else
3759                 err = mlx5e_arfs_disable(priv);
3760
3761         return err;
3762 }
3763 #endif
3764
3765 static int mlx5e_handle_feature(struct net_device *netdev,
3766                                 netdev_features_t *features,
3767                                 netdev_features_t feature,
3768                                 mlx5e_feature_handler feature_handler)
3769 {
3770         netdev_features_t changes = *features ^ netdev->features;
3771         bool enable = !!(*features & feature);
3772         int err;
3773
3774         if (!(changes & feature))
3775                 return 0;
3776
3777         err = feature_handler(netdev, enable);
3778         if (err) {
3779                 MLX5E_SET_FEATURE(features, feature, !enable);
3780                 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
3781                            enable ? "Enable" : "Disable", &feature, err);
3782                 return err;
3783         }
3784
3785         return 0;
3786 }
3787
3788 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
3789 {
3790         netdev_features_t oper_features = features;
3791         int err = 0;
3792
3793 #define MLX5E_HANDLE_FEATURE(feature, handler) \
3794         mlx5e_handle_feature(netdev, &oper_features, feature, handler)
3795
3796         err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
3797         err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro);
3798         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
3799                                     set_feature_cvlan_filter);
3800         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc);
3801         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
3802         err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
3803         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
3804 #ifdef CONFIG_MLX5_EN_ARFS
3805         err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
3806 #endif
3807         err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx);
3808
3809         if (err) {
3810                 netdev->features = oper_features;
3811                 return -EINVAL;
3812         }
3813
3814         return 0;
3815 }
3816
3817 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev,
3818                                                        netdev_features_t features)
3819 {
3820         features &= ~NETIF_F_HW_TLS_RX;
3821         if (netdev->features & NETIF_F_HW_TLS_RX)
3822                 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n");
3823
3824         features &= ~NETIF_F_HW_TLS_TX;
3825         if (netdev->features & NETIF_F_HW_TLS_TX)
3826                 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n");
3827
3828         features &= ~NETIF_F_NTUPLE;
3829         if (netdev->features & NETIF_F_NTUPLE)
3830                 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n");
3831
3832         return features;
3833 }
3834
3835 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
3836                                             netdev_features_t features)
3837 {
3838         struct mlx5e_priv *priv = netdev_priv(netdev);
3839         struct mlx5e_params *params;
3840
3841         mutex_lock(&priv->state_lock);
3842         params = &priv->channels.params;
3843         if (!priv->fs.vlan ||
3844             !bitmap_empty(mlx5e_vlan_get_active_svlans(priv->fs.vlan), VLAN_N_VID)) {
3845                 /* HW strips the outer C-tag header, this is a problem
3846                  * for S-tag traffic.
3847                  */
3848                 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
3849                 if (!params->vlan_strip_disable)
3850                         netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
3851         }
3852
3853         if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3854                 if (features & NETIF_F_LRO) {
3855                         netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
3856                         features &= ~NETIF_F_LRO;
3857                 }
3858                 if (features & NETIF_F_GRO_HW) {
3859                         netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n");
3860                         features &= ~NETIF_F_GRO_HW;
3861                 }
3862         }
3863
3864         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
3865                 features &= ~NETIF_F_RXHASH;
3866                 if (netdev->features & NETIF_F_RXHASH)
3867                         netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n");
3868
3869                 if (features & NETIF_F_GRO_HW) {
3870                         netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n");
3871                         features &= ~NETIF_F_GRO_HW;
3872                 }
3873         }
3874
3875         if (mlx5e_is_uplink_rep(priv))
3876                 features = mlx5e_fix_uplink_rep_features(netdev, features);
3877
3878         mutex_unlock(&priv->state_lock);
3879
3880         return features;
3881 }
3882
3883 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
3884                                    struct mlx5e_channels *chs,
3885                                    struct mlx5e_params *new_params,
3886                                    struct mlx5_core_dev *mdev)
3887 {
3888         u16 ix;
3889
3890         for (ix = 0; ix < chs->params.num_channels; ix++) {
3891                 struct xsk_buff_pool *xsk_pool =
3892                         mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
3893                 struct mlx5e_xsk_param xsk;
3894
3895                 if (!xsk_pool)
3896                         continue;
3897
3898                 mlx5e_build_xsk_param(xsk_pool, &xsk);
3899
3900                 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) {
3901                         u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
3902                         int max_mtu_frame, max_mtu_page, max_mtu;
3903
3904                         /* Two criteria must be met:
3905                          * 1. HW MTU + all headrooms <= XSK frame size.
3906                          * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
3907                          */
3908                         max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
3909                         max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk);
3910                         max_mtu = min(max_mtu_frame, max_mtu_page);
3911
3912                         netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n",
3913                                    new_params->sw_mtu, ix, max_mtu);
3914                         return false;
3915                 }
3916         }
3917
3918         return true;
3919 }
3920
3921 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
3922                      mlx5e_fp_preactivate preactivate)
3923 {
3924         struct mlx5e_priv *priv = netdev_priv(netdev);
3925         struct mlx5e_params new_params;
3926         struct mlx5e_params *params;
3927         bool reset = true;
3928         int err = 0;
3929
3930         mutex_lock(&priv->state_lock);
3931
3932         params = &priv->channels.params;
3933
3934         new_params = *params;
3935         new_params.sw_mtu = new_mtu;
3936         err = mlx5e_validate_params(priv->mdev, &new_params);
3937         if (err)
3938                 goto out;
3939
3940         if (params->xdp_prog &&
3941             !mlx5e_rx_is_linear_skb(&new_params, NULL)) {
3942                 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3943                            new_mtu, mlx5e_xdp_max_mtu(params, NULL));
3944                 err = -EINVAL;
3945                 goto out;
3946         }
3947
3948         if (priv->xsk.refcnt &&
3949             !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
3950                                     &new_params, priv->mdev)) {
3951                 err = -EINVAL;
3952                 goto out;
3953         }
3954
3955         if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
3956                 reset = false;
3957
3958         if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3959                 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL);
3960                 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
3961                                                                   &new_params, NULL);
3962                 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL);
3963                 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_params, NULL);
3964
3965                 /* Always reset in linear mode - hw_mtu is used in data path.
3966                  * Check that the mode was non-linear and didn't change.
3967                  * If XSK is active, XSK RQs are linear.
3968                  */
3969                 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt &&
3970                     ppw_old == ppw_new)
3971                         reset = false;
3972         }
3973
3974         err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset);
3975
3976 out:
3977         netdev->mtu = params->sw_mtu;
3978         mutex_unlock(&priv->state_lock);
3979         return err;
3980 }
3981
3982 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
3983 {
3984         return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx);
3985 }
3986
3987 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx)
3988 {
3989         bool set  = *(bool *)ctx;
3990
3991         return mlx5e_ptp_rx_manage_fs(priv, set);
3992 }
3993
3994 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter)
3995 {
3996         bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
3997         int err;
3998
3999         if (!rx_filter)
4000                 /* Reset CQE compression to Admin default */
4001                 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false);
4002
4003         if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS))
4004                 return 0;
4005
4006         /* Disable CQE compression */
4007         netdev_warn(priv->netdev, "Disabling RX cqe compression\n");
4008         err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true);
4009         if (err)
4010                 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
4011
4012         return err;
4013 }
4014
4015 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx)
4016 {
4017         struct mlx5e_params new_params;
4018
4019         if (ptp_rx == priv->channels.params.ptp_rx)
4020                 return 0;
4021
4022         new_params = priv->channels.params;
4023         new_params.ptp_rx = ptp_rx;
4024         return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx,
4025                                         &new_params.ptp_rx, true);
4026 }
4027
4028 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
4029 {
4030         struct hwtstamp_config config;
4031         bool rx_cqe_compress_def;
4032         bool ptp_rx;
4033         int err;
4034
4035         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
4036             (mlx5_clock_get_ptp_index(priv->mdev) == -1))
4037                 return -EOPNOTSUPP;
4038
4039         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
4040                 return -EFAULT;
4041
4042         /* TX HW timestamp */
4043         switch (config.tx_type) {
4044         case HWTSTAMP_TX_OFF:
4045         case HWTSTAMP_TX_ON:
4046                 break;
4047         default:
4048                 return -ERANGE;
4049         }
4050
4051         mutex_lock(&priv->state_lock);
4052         rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4053
4054         /* RX HW timestamp */
4055         switch (config.rx_filter) {
4056         case HWTSTAMP_FILTER_NONE:
4057                 ptp_rx = false;
4058                 break;
4059         case HWTSTAMP_FILTER_ALL:
4060         case HWTSTAMP_FILTER_SOME:
4061         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4062         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4063         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4064         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4065         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4066         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4067         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4068         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4069         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4070         case HWTSTAMP_FILTER_PTP_V2_EVENT:
4071         case HWTSTAMP_FILTER_PTP_V2_SYNC:
4072         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4073         case HWTSTAMP_FILTER_NTP_ALL:
4074                 config.rx_filter = HWTSTAMP_FILTER_ALL;
4075                 /* ptp_rx is set if both HW TS is set and CQE
4076                  * compression is set
4077                  */
4078                 ptp_rx = rx_cqe_compress_def;
4079                 break;
4080         default:
4081                 err = -ERANGE;
4082                 goto err_unlock;
4083         }
4084
4085         if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
4086                 err = mlx5e_hwstamp_config_no_ptp_rx(priv,
4087                                                      config.rx_filter != HWTSTAMP_FILTER_NONE);
4088         else
4089                 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx);
4090         if (err)
4091                 goto err_unlock;
4092
4093         memcpy(&priv->tstamp, &config, sizeof(config));
4094         mutex_unlock(&priv->state_lock);
4095
4096         /* might need to fix some features */
4097         netdev_update_features(priv->netdev);
4098
4099         return copy_to_user(ifr->ifr_data, &config,
4100                             sizeof(config)) ? -EFAULT : 0;
4101 err_unlock:
4102         mutex_unlock(&priv->state_lock);
4103         return err;
4104 }
4105
4106 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
4107 {
4108         struct hwtstamp_config *cfg = &priv->tstamp;
4109
4110         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
4111                 return -EOPNOTSUPP;
4112
4113         return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
4114 }
4115
4116 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4117 {
4118         struct mlx5e_priv *priv = netdev_priv(dev);
4119
4120         switch (cmd) {
4121         case SIOCSHWTSTAMP:
4122                 return mlx5e_hwstamp_set(priv, ifr);
4123         case SIOCGHWTSTAMP:
4124                 return mlx5e_hwstamp_get(priv, ifr);
4125         default:
4126                 return -EOPNOTSUPP;
4127         }
4128 }
4129
4130 #ifdef CONFIG_MLX5_ESWITCH
4131 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
4132 {
4133         struct mlx5e_priv *priv = netdev_priv(dev);
4134         struct mlx5_core_dev *mdev = priv->mdev;
4135
4136         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
4137 }
4138
4139 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
4140                              __be16 vlan_proto)
4141 {
4142         struct mlx5e_priv *priv = netdev_priv(dev);
4143         struct mlx5_core_dev *mdev = priv->mdev;
4144
4145         if (vlan_proto != htons(ETH_P_8021Q))
4146                 return -EPROTONOSUPPORT;
4147
4148         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
4149                                            vlan, qos);
4150 }
4151
4152 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
4153 {
4154         struct mlx5e_priv *priv = netdev_priv(dev);
4155         struct mlx5_core_dev *mdev = priv->mdev;
4156
4157         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
4158 }
4159
4160 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
4161 {
4162         struct mlx5e_priv *priv = netdev_priv(dev);
4163         struct mlx5_core_dev *mdev = priv->mdev;
4164
4165         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
4166 }
4167
4168 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
4169                       int max_tx_rate)
4170 {
4171         struct mlx5e_priv *priv = netdev_priv(dev);
4172         struct mlx5_core_dev *mdev = priv->mdev;
4173
4174         return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
4175                                            max_tx_rate, min_tx_rate);
4176 }
4177
4178 static int mlx5_vport_link2ifla(u8 esw_link)
4179 {
4180         switch (esw_link) {
4181         case MLX5_VPORT_ADMIN_STATE_DOWN:
4182                 return IFLA_VF_LINK_STATE_DISABLE;
4183         case MLX5_VPORT_ADMIN_STATE_UP:
4184                 return IFLA_VF_LINK_STATE_ENABLE;
4185         }
4186         return IFLA_VF_LINK_STATE_AUTO;
4187 }
4188
4189 static int mlx5_ifla_link2vport(u8 ifla_link)
4190 {
4191         switch (ifla_link) {
4192         case IFLA_VF_LINK_STATE_DISABLE:
4193                 return MLX5_VPORT_ADMIN_STATE_DOWN;
4194         case IFLA_VF_LINK_STATE_ENABLE:
4195                 return MLX5_VPORT_ADMIN_STATE_UP;
4196         }
4197         return MLX5_VPORT_ADMIN_STATE_AUTO;
4198 }
4199
4200 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
4201                                    int link_state)
4202 {
4203         struct mlx5e_priv *priv = netdev_priv(dev);
4204         struct mlx5_core_dev *mdev = priv->mdev;
4205
4206         if (mlx5e_is_uplink_rep(priv))
4207                 return -EOPNOTSUPP;
4208
4209         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
4210                                             mlx5_ifla_link2vport(link_state));
4211 }
4212
4213 int mlx5e_get_vf_config(struct net_device *dev,
4214                         int vf, struct ifla_vf_info *ivi)
4215 {
4216         struct mlx5e_priv *priv = netdev_priv(dev);
4217         struct mlx5_core_dev *mdev = priv->mdev;
4218         int err;
4219
4220         if (!netif_device_present(dev))
4221                 return -EOPNOTSUPP;
4222
4223         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
4224         if (err)
4225                 return err;
4226         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
4227         return 0;
4228 }
4229
4230 int mlx5e_get_vf_stats(struct net_device *dev,
4231                        int vf, struct ifla_vf_stats *vf_stats)
4232 {
4233         struct mlx5e_priv *priv = netdev_priv(dev);
4234         struct mlx5_core_dev *mdev = priv->mdev;
4235
4236         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
4237                                             vf_stats);
4238 }
4239
4240 static bool
4241 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id)
4242 {
4243         struct mlx5e_priv *priv = netdev_priv(dev);
4244
4245         if (!netif_device_present(dev))
4246                 return false;
4247
4248         if (!mlx5e_is_uplink_rep(priv))
4249                 return false;
4250
4251         return mlx5e_rep_has_offload_stats(dev, attr_id);
4252 }
4253
4254 static int
4255 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev,
4256                         void *sp)
4257 {
4258         struct mlx5e_priv *priv = netdev_priv(dev);
4259
4260         if (!mlx5e_is_uplink_rep(priv))
4261                 return -EOPNOTSUPP;
4262
4263         return mlx5e_rep_get_offload_stats(attr_id, dev, sp);
4264 }
4265 #endif
4266
4267 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
4268 {
4269         switch (proto_type) {
4270         case IPPROTO_GRE:
4271                 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
4272         case IPPROTO_IPIP:
4273         case IPPROTO_IPV6:
4274                 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
4275                         MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx));
4276         default:
4277                 return false;
4278         }
4279 }
4280
4281 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
4282                                                            struct sk_buff *skb)
4283 {
4284         switch (skb->inner_protocol) {
4285         case htons(ETH_P_IP):
4286         case htons(ETH_P_IPV6):
4287         case htons(ETH_P_TEB):
4288                 return true;
4289         case htons(ETH_P_MPLS_UC):
4290         case htons(ETH_P_MPLS_MC):
4291                 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
4292         }
4293         return false;
4294 }
4295
4296 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4297                                                      struct sk_buff *skb,
4298                                                      netdev_features_t features)
4299 {
4300         unsigned int offset = 0;
4301         struct udphdr *udph;
4302         u8 proto;
4303         u16 port;
4304
4305         switch (vlan_get_protocol(skb)) {
4306         case htons(ETH_P_IP):
4307                 proto = ip_hdr(skb)->protocol;
4308                 break;
4309         case htons(ETH_P_IPV6):
4310                 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
4311                 break;
4312         default:
4313                 goto out;
4314         }
4315
4316         switch (proto) {
4317         case IPPROTO_GRE:
4318                 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
4319                         return features;
4320                 break;
4321         case IPPROTO_IPIP:
4322         case IPPROTO_IPV6:
4323                 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP))
4324                         return features;
4325                 break;
4326         case IPPROTO_UDP:
4327                 udph = udp_hdr(skb);
4328                 port = be16_to_cpu(udph->dest);
4329
4330                 /* Verify if UDP port is being offloaded by HW */
4331                 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
4332                         return features;
4333
4334 #if IS_ENABLED(CONFIG_GENEVE)
4335                 /* Support Geneve offload for default UDP port */
4336                 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev))
4337                         return features;
4338 #endif
4339                 break;
4340 #ifdef CONFIG_MLX5_EN_IPSEC
4341         case IPPROTO_ESP:
4342                 return mlx5e_ipsec_feature_check(skb, features);
4343 #endif
4344         }
4345
4346 out:
4347         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4348         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4349 }
4350
4351 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4352                                        struct net_device *netdev,
4353                                        netdev_features_t features)
4354 {
4355         struct mlx5e_priv *priv = netdev_priv(netdev);
4356
4357         features = vlan_features_check(skb, features);
4358         features = vxlan_features_check(skb, features);
4359
4360         /* Validate if the tunneled packet is being offloaded by HW */
4361         if (skb->encapsulation &&
4362             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
4363                 return mlx5e_tunnel_features_check(priv, skb, features);
4364
4365         return features;
4366 }
4367
4368 static void mlx5e_tx_timeout_work(struct work_struct *work)
4369 {
4370         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4371                                                tx_timeout_work);
4372         struct net_device *netdev = priv->netdev;
4373         int i;
4374
4375         rtnl_lock();
4376         mutex_lock(&priv->state_lock);
4377
4378         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4379                 goto unlock;
4380
4381         for (i = 0; i < netdev->real_num_tx_queues; i++) {
4382                 struct netdev_queue *dev_queue =
4383                         netdev_get_tx_queue(netdev, i);
4384                 struct mlx5e_txqsq *sq = priv->txq2sq[i];
4385
4386                 if (!netif_xmit_stopped(dev_queue))
4387                         continue;
4388
4389                 if (mlx5e_reporter_tx_timeout(sq))
4390                 /* break if tried to reopened channels */
4391                         break;
4392         }
4393
4394 unlock:
4395         mutex_unlock(&priv->state_lock);
4396         rtnl_unlock();
4397 }
4398
4399 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
4400 {
4401         struct mlx5e_priv *priv = netdev_priv(dev);
4402
4403         netdev_err(dev, "TX timeout detected\n");
4404         queue_work(priv->wq, &priv->tx_timeout_work);
4405 }
4406
4407 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
4408 {
4409         struct net_device *netdev = priv->netdev;
4410         struct mlx5e_params new_params;
4411
4412         if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
4413                 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
4414                 return -EINVAL;
4415         }
4416
4417         if (mlx5_fpga_is_ipsec_device(priv->mdev)) {
4418                 netdev_warn(netdev,
4419                             "XDP is not available on Innova cards with IPsec support\n");
4420                 return -EINVAL;
4421         }
4422
4423         new_params = priv->channels.params;
4424         new_params.xdp_prog = prog;
4425
4426         /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4427          * the XDP program.
4428          */
4429         if (!mlx5e_rx_is_linear_skb(&new_params, NULL)) {
4430                 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4431                             new_params.sw_mtu,
4432                             mlx5e_xdp_max_mtu(&new_params, NULL));
4433                 return -EINVAL;
4434         }
4435
4436         return 0;
4437 }
4438
4439 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
4440 {
4441         struct bpf_prog *old_prog;
4442
4443         old_prog = rcu_replace_pointer(rq->xdp_prog, prog,
4444                                        lockdep_is_held(&rq->priv->state_lock));
4445         if (old_prog)
4446                 bpf_prog_put(old_prog);
4447 }
4448
4449 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4450 {
4451         struct mlx5e_priv *priv = netdev_priv(netdev);
4452         struct mlx5e_params new_params;
4453         struct bpf_prog *old_prog;
4454         int err = 0;
4455         bool reset;
4456         int i;
4457
4458         mutex_lock(&priv->state_lock);
4459
4460         if (prog) {
4461                 err = mlx5e_xdp_allowed(priv, prog);
4462                 if (err)
4463                         goto unlock;
4464         }
4465
4466         /* no need for full reset when exchanging programs */
4467         reset = (!priv->channels.params.xdp_prog || !prog);
4468
4469         new_params = priv->channels.params;
4470         new_params.xdp_prog = prog;
4471         if (reset)
4472                 mlx5e_set_rq_type(priv->mdev, &new_params);
4473         old_prog = priv->channels.params.xdp_prog;
4474
4475         err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
4476         if (err)
4477                 goto unlock;
4478
4479         if (old_prog)
4480                 bpf_prog_put(old_prog);
4481
4482         if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4483                 goto unlock;
4484
4485         /* exchanging programs w/o reset, we update ref counts on behalf
4486          * of the channels RQs here.
4487          */
4488         bpf_prog_add(prog, priv->channels.num);
4489         for (i = 0; i < priv->channels.num; i++) {
4490                 struct mlx5e_channel *c = priv->channels.c[i];
4491
4492                 mlx5e_rq_replace_xdp_prog(&c->rq, prog);
4493                 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) {
4494                         bpf_prog_inc(prog);
4495                         mlx5e_rq_replace_xdp_prog(&c->xskrq, prog);
4496                 }
4497         }
4498
4499 unlock:
4500         mutex_unlock(&priv->state_lock);
4501         return err;
4502 }
4503
4504 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4505 {
4506         switch (xdp->command) {
4507         case XDP_SETUP_PROG:
4508                 return mlx5e_xdp_set(dev, xdp->prog);
4509         case XDP_SETUP_XSK_POOL:
4510                 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
4511                                             xdp->xsk.queue_id);
4512         default:
4513                 return -EINVAL;
4514         }
4515 }
4516
4517 #ifdef CONFIG_MLX5_ESWITCH
4518 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4519                                 struct net_device *dev, u32 filter_mask,
4520                                 int nlflags)
4521 {
4522         struct mlx5e_priv *priv = netdev_priv(dev);
4523         struct mlx5_core_dev *mdev = priv->mdev;
4524         u8 mode, setting;
4525         int err;
4526
4527         err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting);
4528         if (err)
4529                 return err;
4530         mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
4531         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4532                                        mode,
4533                                        0, 0, nlflags, filter_mask, NULL);
4534 }
4535
4536 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4537                                 u16 flags, struct netlink_ext_ack *extack)
4538 {
4539         struct mlx5e_priv *priv = netdev_priv(dev);
4540         struct mlx5_core_dev *mdev = priv->mdev;
4541         struct nlattr *attr, *br_spec;
4542         u16 mode = BRIDGE_MODE_UNDEF;
4543         u8 setting;
4544         int rem;
4545
4546         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4547         if (!br_spec)
4548                 return -EINVAL;
4549
4550         nla_for_each_nested(attr, br_spec, rem) {
4551                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4552                         continue;
4553
4554                 if (nla_len(attr) < sizeof(mode))
4555                         return -EINVAL;
4556
4557                 mode = nla_get_u16(attr);
4558                 if (mode > BRIDGE_MODE_VEPA)
4559                         return -EINVAL;
4560
4561                 break;
4562         }
4563
4564         if (mode == BRIDGE_MODE_UNDEF)
4565                 return -EINVAL;
4566
4567         setting = (mode == BRIDGE_MODE_VEPA) ?  1 : 0;
4568         return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting);
4569 }
4570 #endif
4571
4572 const struct net_device_ops mlx5e_netdev_ops = {
4573         .ndo_open                = mlx5e_open,
4574         .ndo_stop                = mlx5e_close,
4575         .ndo_start_xmit          = mlx5e_xmit,
4576         .ndo_setup_tc            = mlx5e_setup_tc,
4577         .ndo_select_queue        = mlx5e_select_queue,
4578         .ndo_get_stats64         = mlx5e_get_stats,
4579         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
4580         .ndo_set_mac_address     = mlx5e_set_mac,
4581         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
4582         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
4583         .ndo_set_features        = mlx5e_set_features,
4584         .ndo_fix_features        = mlx5e_fix_features,
4585         .ndo_change_mtu          = mlx5e_change_nic_mtu,
4586         .ndo_eth_ioctl            = mlx5e_ioctl,
4587         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
4588         .ndo_features_check      = mlx5e_features_check,
4589         .ndo_tx_timeout          = mlx5e_tx_timeout,
4590         .ndo_bpf                 = mlx5e_xdp,
4591         .ndo_xdp_xmit            = mlx5e_xdp_xmit,
4592         .ndo_xsk_wakeup          = mlx5e_xsk_wakeup,
4593 #ifdef CONFIG_MLX5_EN_ARFS
4594         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
4595 #endif
4596 #ifdef CONFIG_MLX5_ESWITCH
4597         .ndo_bridge_setlink      = mlx5e_bridge_setlink,
4598         .ndo_bridge_getlink      = mlx5e_bridge_getlink,
4599
4600         /* SRIOV E-Switch NDOs */
4601         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
4602         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
4603         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
4604         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
4605         .ndo_set_vf_rate         = mlx5e_set_vf_rate,
4606         .ndo_get_vf_config       = mlx5e_get_vf_config,
4607         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
4608         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
4609         .ndo_has_offload_stats   = mlx5e_has_offload_stats,
4610         .ndo_get_offload_stats   = mlx5e_get_offload_stats,
4611 #endif
4612         .ndo_get_devlink_port    = mlx5e_get_devlink_port,
4613 };
4614
4615 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
4616 {
4617         int i;
4618
4619         /* The supported periods are organized in ascending order */
4620         for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4621                 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4622                         break;
4623
4624         return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4625 }
4626
4627 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
4628 {
4629         struct mlx5e_params *params = &priv->channels.params;
4630         struct mlx5_core_dev *mdev = priv->mdev;
4631         u8 rx_cq_period_mode;
4632
4633         params->sw_mtu = mtu;
4634         params->hard_mtu = MLX5E_ETH_HARD_MTU;
4635         params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2,
4636                                      priv->max_nch);
4637         mlx5e_params_mqprio_reset(params);
4638
4639         /* Set an initial non-zero value, so that mlx5e_select_queue won't
4640          * divide by zero if called before first activating channels.
4641          */
4642         priv->num_tc_x_num_ch = params->num_channels * params->mqprio.num_tc;
4643
4644         /* SQ */
4645         params->log_sq_size = is_kdump_kernel() ?
4646                 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4647                 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
4648         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4649
4650         /* XDP SQ */
4651         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
4652
4653         /* set CQE compression */
4654         params->rx_cqe_compress_def = false;
4655         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
4656             MLX5_CAP_GEN(mdev, vport_group_manager))
4657                 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
4658
4659         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
4660         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false);
4661
4662         /* RQ */
4663         mlx5e_build_rq_params(mdev, params);
4664
4665         /* HW LRO */
4666         if (MLX5_CAP_ETH(mdev, lro_cap) &&
4667             params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
4668                 /* No XSK params: checking the availability of striding RQ in general. */
4669                 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
4670                         params->packet_merge.type = slow_pci_heuristic(mdev) ?
4671                                 MLX5E_PACKET_MERGE_NONE : MLX5E_PACKET_MERGE_LRO;
4672         }
4673         params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4674
4675         /* CQ moderation params */
4676         rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
4677                         MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
4678                         MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
4679         params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4680         params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4681         mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
4682         mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
4683
4684         /* TX inline */
4685         mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
4686
4687         params->tunneled_offload_en = mlx5_tunnel_inner_ft_supported(mdev);
4688
4689         /* AF_XDP */
4690         params->xsk = xsk;
4691
4692         /* Do not update netdev->features directly in here
4693          * on mlx5e_attach_netdev() we will call mlx5e_update_features()
4694          * To update netdev->features please modify mlx5e_fix_features()
4695          */
4696 }
4697
4698 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
4699 {
4700         struct mlx5e_priv *priv = netdev_priv(netdev);
4701         u8 addr[ETH_ALEN];
4702
4703         mlx5_query_mac_address(priv->mdev, addr);
4704         if (is_zero_ether_addr(addr) &&
4705             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
4706                 eth_hw_addr_random(netdev);
4707                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
4708                 return;
4709         }
4710
4711         eth_hw_addr_set(netdev, addr);
4712 }
4713
4714 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
4715                                 unsigned int entry, struct udp_tunnel_info *ti)
4716 {
4717         struct mlx5e_priv *priv = netdev_priv(netdev);
4718
4719         return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port));
4720 }
4721
4722 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
4723                                   unsigned int entry, struct udp_tunnel_info *ti)
4724 {
4725         struct mlx5e_priv *priv = netdev_priv(netdev);
4726
4727         return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port));
4728 }
4729
4730 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
4731 {
4732         if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
4733                 return;
4734
4735         priv->nic_info.set_port = mlx5e_vxlan_set_port;
4736         priv->nic_info.unset_port = mlx5e_vxlan_unset_port;
4737         priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
4738                                 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN;
4739         priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN;
4740         /* Don't count the space hard-coded to the IANA port */
4741         priv->nic_info.tables[0].n_entries =
4742                 mlx5_vxlan_max_udp_ports(priv->mdev) - 1;
4743
4744         priv->netdev->udp_tunnel_nic_info = &priv->nic_info;
4745 }
4746
4747 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
4748 {
4749         int tt;
4750
4751         for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
4752                 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt)))
4753                         return true;
4754         }
4755         return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev));
4756 }
4757
4758 static void mlx5e_build_nic_netdev(struct net_device *netdev)
4759 {
4760         struct mlx5e_priv *priv = netdev_priv(netdev);
4761         struct mlx5_core_dev *mdev = priv->mdev;
4762         bool fcs_supported;
4763         bool fcs_enabled;
4764
4765         SET_NETDEV_DEV(netdev, mdev->device);
4766
4767         netdev->netdev_ops = &mlx5e_netdev_ops;
4768
4769         mlx5e_dcbnl_build_netdev(netdev);
4770
4771         netdev->watchdog_timeo    = 15 * HZ;
4772
4773         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
4774
4775         netdev->vlan_features    |= NETIF_F_SG;
4776         netdev->vlan_features    |= NETIF_F_HW_CSUM;
4777         netdev->vlan_features    |= NETIF_F_GRO;
4778         netdev->vlan_features    |= NETIF_F_TSO;
4779         netdev->vlan_features    |= NETIF_F_TSO6;
4780         netdev->vlan_features    |= NETIF_F_RXCSUM;
4781         netdev->vlan_features    |= NETIF_F_RXHASH;
4782
4783         netdev->mpls_features    |= NETIF_F_SG;
4784         netdev->mpls_features    |= NETIF_F_HW_CSUM;
4785         netdev->mpls_features    |= NETIF_F_TSO;
4786         netdev->mpls_features    |= NETIF_F_TSO6;
4787
4788         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_TX;
4789         netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_RX;
4790
4791         /* Tunneled LRO is not supported in the driver, and the same RQs are
4792          * shared between inner and outer TIRs, so the driver can't disable LRO
4793          * for inner TIRs while having it enabled for outer TIRs. Due to this,
4794          * block LRO altogether if the firmware declares tunneled LRO support.
4795          */
4796         if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
4797             !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
4798             !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
4799             mlx5e_check_fragmented_striding_rq_cap(mdev))
4800                 netdev->vlan_features    |= NETIF_F_LRO;
4801
4802         netdev->hw_features       = netdev->vlan_features;
4803         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
4804         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
4805         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
4806         netdev->hw_features      |= NETIF_F_HW_VLAN_STAG_TX;
4807
4808         if (!!MLX5_CAP_GEN(mdev, shampo) &&
4809             mlx5e_check_fragmented_striding_rq_cap(mdev))
4810                 netdev->hw_features    |= NETIF_F_GRO_HW;
4811
4812         if (mlx5e_tunnel_any_tx_proto_supported(mdev)) {
4813                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
4814                 netdev->hw_enc_features |= NETIF_F_TSO;
4815                 netdev->hw_enc_features |= NETIF_F_TSO6;
4816                 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
4817         }
4818
4819         if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
4820                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
4821                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
4822                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
4823                                            NETIF_F_GSO_UDP_TUNNEL_CSUM;
4824                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
4825                 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
4826                                          NETIF_F_GSO_UDP_TUNNEL_CSUM;
4827         }
4828
4829         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
4830                 netdev->hw_features     |= NETIF_F_GSO_GRE |
4831                                            NETIF_F_GSO_GRE_CSUM;
4832                 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
4833                                            NETIF_F_GSO_GRE_CSUM;
4834                 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
4835                                                 NETIF_F_GSO_GRE_CSUM;
4836         }
4837
4838         if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
4839                 netdev->hw_features |= NETIF_F_GSO_IPXIP4 |
4840                                        NETIF_F_GSO_IPXIP6;
4841                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 |
4842                                            NETIF_F_GSO_IPXIP6;
4843                 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 |
4844                                                 NETIF_F_GSO_IPXIP6;
4845         }
4846
4847         netdev->hw_features                      |= NETIF_F_GSO_PARTIAL;
4848         netdev->gso_partial_features             |= NETIF_F_GSO_UDP_L4;
4849         netdev->hw_features                      |= NETIF_F_GSO_UDP_L4;
4850         netdev->features                         |= NETIF_F_GSO_UDP_L4;
4851
4852         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
4853
4854         if (fcs_supported)
4855                 netdev->hw_features |= NETIF_F_RXALL;
4856
4857         if (MLX5_CAP_ETH(mdev, scatter_fcs))
4858                 netdev->hw_features |= NETIF_F_RXFCS;
4859
4860         if (mlx5_qos_is_supported(mdev))
4861                 netdev->hw_features |= NETIF_F_HW_TC;
4862
4863         netdev->features          = netdev->hw_features;
4864
4865         /* Defaults */
4866         if (fcs_enabled)
4867                 netdev->features  &= ~NETIF_F_RXALL;
4868         netdev->features  &= ~NETIF_F_LRO;
4869         netdev->features  &= ~NETIF_F_GRO_HW;
4870         netdev->features  &= ~NETIF_F_RXFCS;
4871
4872 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
4873         if (FT_CAP(flow_modify_en) &&
4874             FT_CAP(modify_root) &&
4875             FT_CAP(identified_miss_table_mode) &&
4876             FT_CAP(flow_table_modify)) {
4877 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
4878                 netdev->hw_features      |= NETIF_F_HW_TC;
4879 #endif
4880 #ifdef CONFIG_MLX5_EN_ARFS
4881                 netdev->hw_features      |= NETIF_F_NTUPLE;
4882 #endif
4883         }
4884
4885         netdev->features         |= NETIF_F_HIGHDMA;
4886         netdev->features         |= NETIF_F_HW_VLAN_STAG_FILTER;
4887
4888         netdev->priv_flags       |= IFF_UNICAST_FLT;
4889
4890         mlx5e_set_netdev_dev_addr(netdev);
4891         mlx5e_ipsec_build_netdev(priv);
4892         mlx5e_tls_build_netdev(priv);
4893 }
4894
4895 void mlx5e_create_q_counters(struct mlx5e_priv *priv)
4896 {
4897         u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
4898         u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
4899         struct mlx5_core_dev *mdev = priv->mdev;
4900         int err;
4901
4902         MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
4903         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
4904         if (!err)
4905                 priv->q_counter =
4906                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
4907
4908         err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
4909         if (!err)
4910                 priv->drop_rq_q_counter =
4911                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
4912 }
4913
4914 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
4915 {
4916         u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
4917
4918         MLX5_SET(dealloc_q_counter_in, in, opcode,
4919                  MLX5_CMD_OP_DEALLOC_Q_COUNTER);
4920         if (priv->q_counter) {
4921                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
4922                          priv->q_counter);
4923                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
4924         }
4925
4926         if (priv->drop_rq_q_counter) {
4927                 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
4928                          priv->drop_rq_q_counter);
4929                 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
4930         }
4931 }
4932
4933 static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
4934                           struct net_device *netdev)
4935 {
4936         struct mlx5e_priv *priv = netdev_priv(netdev);
4937         int err;
4938
4939         mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu);
4940         mlx5e_vxlan_set_netdev_info(priv);
4941
4942         mlx5e_timestamp_init(priv);
4943
4944         err = mlx5e_fs_init(priv);
4945         if (err) {
4946                 mlx5_core_err(mdev, "FS initialization failed, %d\n", err);
4947                 return err;
4948         }
4949
4950         err = mlx5e_ipsec_init(priv);
4951         if (err)
4952                 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
4953
4954         err = mlx5e_tls_init(priv);
4955         if (err)
4956                 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
4957
4958         mlx5e_health_create_reporters(priv);
4959         return 0;
4960 }
4961
4962 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
4963 {
4964         mlx5e_health_destroy_reporters(priv);
4965         mlx5e_tls_cleanup(priv);
4966         mlx5e_ipsec_cleanup(priv);
4967         mlx5e_fs_cleanup(priv);
4968 }
4969
4970 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
4971 {
4972         struct mlx5_core_dev *mdev = priv->mdev;
4973         enum mlx5e_rx_res_features features;
4974         int err;
4975
4976         priv->rx_res = mlx5e_rx_res_alloc();
4977         if (!priv->rx_res)
4978                 return -ENOMEM;
4979
4980         mlx5e_create_q_counters(priv);
4981
4982         err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
4983         if (err) {
4984                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
4985                 goto err_destroy_q_counters;
4986         }
4987
4988         features = MLX5E_RX_RES_FEATURE_XSK | MLX5E_RX_RES_FEATURE_PTP;
4989         if (priv->channels.params.tunneled_offload_en)
4990                 features |= MLX5E_RX_RES_FEATURE_INNER_FT;
4991         err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features,
4992                                 priv->max_nch, priv->drop_rq.rqn,
4993                                 &priv->channels.params.packet_merge,
4994                                 priv->channels.params.num_channels);
4995         if (err)
4996                 goto err_close_drop_rq;
4997
4998         err = mlx5e_create_flow_steering(priv);
4999         if (err) {
5000                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
5001                 goto err_destroy_rx_res;
5002         }
5003
5004         err = mlx5e_tc_nic_init(priv);
5005         if (err)
5006                 goto err_destroy_flow_steering;
5007
5008         err = mlx5e_accel_init_rx(priv);
5009         if (err)
5010                 goto err_tc_nic_cleanup;
5011
5012 #ifdef CONFIG_MLX5_EN_ARFS
5013         priv->netdev->rx_cpu_rmap =  mlx5_eq_table_get_rmap(priv->mdev);
5014 #endif
5015
5016         return 0;
5017
5018 err_tc_nic_cleanup:
5019         mlx5e_tc_nic_cleanup(priv);
5020 err_destroy_flow_steering:
5021         mlx5e_destroy_flow_steering(priv);
5022 err_destroy_rx_res:
5023         mlx5e_rx_res_destroy(priv->rx_res);
5024 err_close_drop_rq:
5025         mlx5e_close_drop_rq(&priv->drop_rq);
5026 err_destroy_q_counters:
5027         mlx5e_destroy_q_counters(priv);
5028         mlx5e_rx_res_free(priv->rx_res);
5029         priv->rx_res = NULL;
5030         return err;
5031 }
5032
5033 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
5034 {
5035         mlx5e_accel_cleanup_rx(priv);
5036         mlx5e_tc_nic_cleanup(priv);
5037         mlx5e_destroy_flow_steering(priv);
5038         mlx5e_rx_res_destroy(priv->rx_res);
5039         mlx5e_close_drop_rq(&priv->drop_rq);
5040         mlx5e_destroy_q_counters(priv);
5041         mlx5e_rx_res_free(priv->rx_res);
5042         priv->rx_res = NULL;
5043 }
5044
5045 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
5046 {
5047         int err;
5048
5049         err = mlx5e_create_tises(priv);
5050         if (err) {
5051                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
5052                 return err;
5053         }
5054
5055         mlx5e_dcbnl_initialize(priv);
5056         return 0;
5057 }
5058
5059 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
5060 {
5061         struct net_device *netdev = priv->netdev;
5062         struct mlx5_core_dev *mdev = priv->mdev;
5063
5064         mlx5e_init_l2_addr(priv);
5065
5066         /* Marking the link as currently not needed by the Driver */
5067         if (!netif_running(netdev))
5068                 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
5069
5070         mlx5e_set_netdev_mtu_boundaries(priv);
5071         mlx5e_set_dev_port_mtu(priv);
5072
5073         mlx5_lag_add_netdev(mdev, netdev);
5074
5075         mlx5e_enable_async_events(priv);
5076         mlx5e_enable_blocking_events(priv);
5077         if (mlx5e_monitor_counter_supported(priv))
5078                 mlx5e_monitor_counter_init(priv);
5079
5080         mlx5e_hv_vhca_stats_create(priv);
5081         if (netdev->reg_state != NETREG_REGISTERED)
5082                 return;
5083         mlx5e_dcbnl_init_app(priv);
5084
5085         mlx5e_nic_set_rx_mode(priv);
5086
5087         rtnl_lock();
5088         if (netif_running(netdev))
5089                 mlx5e_open(netdev);
5090         udp_tunnel_nic_reset_ntf(priv->netdev);
5091         netif_device_attach(netdev);
5092         rtnl_unlock();
5093 }
5094
5095 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
5096 {
5097         struct mlx5_core_dev *mdev = priv->mdev;
5098
5099         if (priv->netdev->reg_state == NETREG_REGISTERED)
5100                 mlx5e_dcbnl_delete_app(priv);
5101
5102         rtnl_lock();
5103         if (netif_running(priv->netdev))
5104                 mlx5e_close(priv->netdev);
5105         netif_device_detach(priv->netdev);
5106         rtnl_unlock();
5107
5108         mlx5e_nic_set_rx_mode(priv);
5109
5110         mlx5e_hv_vhca_stats_destroy(priv);
5111         if (mlx5e_monitor_counter_supported(priv))
5112                 mlx5e_monitor_counter_cleanup(priv);
5113
5114         mlx5e_disable_blocking_events(priv);
5115         if (priv->en_trap) {
5116                 mlx5e_deactivate_trap(priv);
5117                 mlx5e_close_trap(priv->en_trap);
5118                 priv->en_trap = NULL;
5119         }
5120         mlx5e_disable_async_events(priv);
5121         mlx5_lag_remove_netdev(mdev, priv->netdev);
5122         mlx5_vxlan_reset_to_default(mdev->vxlan);
5123 }
5124
5125 int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
5126 {
5127         return mlx5e_refresh_tirs(priv, false, false);
5128 }
5129
5130 static const struct mlx5e_profile mlx5e_nic_profile = {
5131         .init              = mlx5e_nic_init,
5132         .cleanup           = mlx5e_nic_cleanup,
5133         .init_rx           = mlx5e_init_nic_rx,
5134         .cleanup_rx        = mlx5e_cleanup_nic_rx,
5135         .init_tx           = mlx5e_init_nic_tx,
5136         .cleanup_tx        = mlx5e_cleanup_nic_tx,
5137         .enable            = mlx5e_nic_enable,
5138         .disable           = mlx5e_nic_disable,
5139         .update_rx         = mlx5e_update_nic_rx,
5140         .update_stats      = mlx5e_stats_update_ndo_stats,
5141         .update_carrier    = mlx5e_update_carrier,
5142         .rx_handlers       = &mlx5e_rx_handlers_nic,
5143         .max_tc            = MLX5E_MAX_NUM_TC,
5144         .rq_groups         = MLX5E_NUM_RQ_GROUPS(XSK),
5145         .stats_grps        = mlx5e_nic_stats_grps,
5146         .stats_grps_num    = mlx5e_nic_stats_grps_num,
5147         .features          = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) |
5148                 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) |
5149                 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB),
5150 };
5151
5152 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev,
5153                                           const struct mlx5e_profile *profile)
5154 {
5155         int nch;
5156
5157         nch = mlx5e_get_max_num_channels(mdev);
5158
5159         if (profile->max_nch_limit)
5160                 nch = min_t(int, nch, profile->max_nch_limit(mdev));
5161         return nch;
5162 }
5163
5164 static unsigned int
5165 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
5166                    const struct mlx5e_profile *profile)
5167
5168 {
5169         unsigned int max_nch, tmp;
5170
5171         /* core resources */
5172         max_nch = mlx5e_profile_max_num_channels(mdev, profile);
5173
5174         /* netdev rx queues */
5175         tmp = netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1);
5176         max_nch = min_t(unsigned int, max_nch, tmp);
5177
5178         /* netdev tx queues */
5179         tmp = netdev->num_tx_queues;
5180         if (mlx5_qos_is_supported(mdev))
5181                 tmp -= mlx5e_qos_max_leaf_nodes(mdev);
5182         if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn))
5183                 tmp -= profile->max_tc;
5184         tmp = tmp / profile->max_tc;
5185         max_nch = min_t(unsigned int, max_nch, tmp);
5186
5187         return max_nch;
5188 }
5189
5190 /* mlx5e generic netdev management API (move to en_common.c) */
5191 int mlx5e_priv_init(struct mlx5e_priv *priv,
5192                     const struct mlx5e_profile *profile,
5193                     struct net_device *netdev,
5194                     struct mlx5_core_dev *mdev)
5195 {
5196         int nch, num_txqs, node, i;
5197
5198         num_txqs = netdev->num_tx_queues;
5199         nch = mlx5e_calc_max_nch(mdev, netdev, profile);
5200         node = dev_to_node(mlx5_core_dma_dev(mdev));
5201
5202         /* priv init */
5203         priv->mdev        = mdev;
5204         priv->netdev      = netdev;
5205         priv->msglevel    = MLX5E_MSG_LEVEL;
5206         priv->max_nch     = nch;
5207         priv->max_opened_tc = 1;
5208
5209         if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL))
5210                 return -ENOMEM;
5211
5212         mutex_init(&priv->state_lock);
5213         hash_init(priv->htb.qos_tc2node);
5214         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
5215         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
5216         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
5217         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
5218
5219         priv->wq = create_singlethread_workqueue("mlx5e");
5220         if (!priv->wq)
5221                 goto err_free_cpumask;
5222
5223         priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node);
5224         if (!priv->txq2sq)
5225                 goto err_destroy_workqueue;
5226
5227         priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node);
5228         if (!priv->tx_rates)
5229                 goto err_free_txq2sq;
5230
5231         priv->channel_tc2realtxq =
5232                 kcalloc_node(nch, sizeof(*priv->channel_tc2realtxq), GFP_KERNEL, node);
5233         if (!priv->channel_tc2realtxq)
5234                 goto err_free_tx_rates;
5235
5236         for (i = 0; i < nch; i++) {
5237                 priv->channel_tc2realtxq[i] =
5238                         kcalloc_node(profile->max_tc, sizeof(**priv->channel_tc2realtxq),
5239                                      GFP_KERNEL, node);
5240                 if (!priv->channel_tc2realtxq[i])
5241                         goto err_free_channel_tc2realtxq;
5242         }
5243
5244         priv->channel_stats =
5245                 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node);
5246         if (!priv->channel_stats)
5247                 goto err_free_channel_tc2realtxq;
5248
5249         return 0;
5250
5251 err_free_channel_tc2realtxq:
5252         while (--i >= 0)
5253                 kfree(priv->channel_tc2realtxq[i]);
5254         kfree(priv->channel_tc2realtxq);
5255 err_free_tx_rates:
5256         kfree(priv->tx_rates);
5257 err_free_txq2sq:
5258         kfree(priv->txq2sq);
5259 err_destroy_workqueue:
5260         destroy_workqueue(priv->wq);
5261 err_free_cpumask:
5262         free_cpumask_var(priv->scratchpad.cpumask);
5263         return -ENOMEM;
5264 }
5265
5266 void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
5267 {
5268         int i;
5269
5270         /* bail if change profile failed and also rollback failed */
5271         if (!priv->mdev)
5272                 return;
5273
5274         for (i = 0; i < priv->stats_nch; i++)
5275                 kvfree(priv->channel_stats[i]);
5276         kfree(priv->channel_stats);
5277         for (i = 0; i < priv->max_nch; i++)
5278                 kfree(priv->channel_tc2realtxq[i]);
5279         kfree(priv->channel_tc2realtxq);
5280         kfree(priv->tx_rates);
5281         kfree(priv->txq2sq);
5282         destroy_workqueue(priv->wq);
5283         free_cpumask_var(priv->scratchpad.cpumask);
5284
5285         for (i = 0; i < priv->htb.max_qos_sqs; i++)
5286                 kfree(priv->htb.qos_sq_stats[i]);
5287         kvfree(priv->htb.qos_sq_stats);
5288
5289         if (priv->mqprio_rl) {
5290                 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
5291                 mlx5e_mqprio_rl_free(priv->mqprio_rl);
5292         }
5293
5294         memset(priv, 0, sizeof(*priv));
5295 }
5296
5297 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
5298                                            const struct mlx5e_profile *profile)
5299 {
5300         unsigned int nch, ptp_txqs, qos_txqs;
5301
5302         nch = mlx5e_profile_max_num_channels(mdev, profile);
5303
5304         ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) &&
5305                 mlx5e_profile_feature_cap(profile, PTP_TX) ?
5306                 profile->max_tc : 0;
5307
5308         qos_txqs = mlx5_qos_is_supported(mdev) &&
5309                 mlx5e_profile_feature_cap(profile, QOS_HTB) ?
5310                 mlx5e_qos_max_leaf_nodes(mdev) : 0;
5311
5312         return nch * profile->max_tc + ptp_txqs + qos_txqs;
5313 }
5314
5315 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
5316                                            const struct mlx5e_profile *profile)
5317 {
5318         unsigned int nch;
5319
5320         nch = mlx5e_profile_max_num_channels(mdev, profile);
5321
5322         return nch * profile->rq_groups;
5323 }
5324
5325 struct net_device *
5326 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile)
5327 {
5328         struct net_device *netdev;
5329         unsigned int txqs, rxqs;
5330         int err;
5331
5332         txqs = mlx5e_get_max_num_txqs(mdev, profile);
5333         rxqs = mlx5e_get_max_num_rxqs(mdev, profile);
5334
5335         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs);
5336         if (!netdev) {
5337                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
5338                 return NULL;
5339         }
5340
5341         err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev);
5342         if (err) {
5343                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5344                 goto err_free_netdev;
5345         }
5346
5347         netif_carrier_off(netdev);
5348         dev_net_set(netdev, mlx5_core_net(mdev));
5349
5350         return netdev;
5351
5352 err_free_netdev:
5353         free_netdev(netdev);
5354
5355         return NULL;
5356 }
5357
5358 static void mlx5e_update_features(struct net_device *netdev)
5359 {
5360         if (netdev->reg_state != NETREG_REGISTERED)
5361                 return; /* features will be updated on netdev registration */
5362
5363         rtnl_lock();
5364         netdev_update_features(netdev);
5365         rtnl_unlock();
5366 }
5367
5368 static void mlx5e_reset_channels(struct net_device *netdev)
5369 {
5370         netdev_reset_tc(netdev);
5371 }
5372
5373 int mlx5e_attach_netdev(struct mlx5e_priv *priv)
5374 {
5375         const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED;
5376         const struct mlx5e_profile *profile = priv->profile;
5377         int max_nch;
5378         int err;
5379
5380         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
5381
5382         /* max number of channels may have changed */
5383         max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile);
5384         if (priv->channels.params.num_channels > max_nch) {
5385                 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch);
5386                 /* Reducing the number of channels - RXFH has to be reset, and
5387                  * mlx5e_num_channels_changed below will build the RQT.
5388                  */
5389                 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED;
5390                 priv->channels.params.num_channels = max_nch;
5391                 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) {
5392                         mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n");
5393                         mlx5e_params_mqprio_reset(&priv->channels.params);
5394                 }
5395         }
5396         if (max_nch != priv->max_nch) {
5397                 mlx5_core_warn(priv->mdev,
5398                                "MLX5E: Updating max number of channels from %u to %u\n",
5399                                priv->max_nch, max_nch);
5400                 priv->max_nch = max_nch;
5401         }
5402
5403         /* 1. Set the real number of queues in the kernel the first time.
5404          * 2. Set our default XPS cpumask.
5405          * 3. Build the RQT.
5406          *
5407          * rtnl_lock is required by netif_set_real_num_*_queues in case the
5408          * netdev has been registered by this point (if this function was called
5409          * in the reload or resume flow).
5410          */
5411         if (take_rtnl)
5412                 rtnl_lock();
5413         err = mlx5e_num_channels_changed(priv);
5414         if (take_rtnl)
5415                 rtnl_unlock();
5416         if (err)
5417                 goto out;
5418
5419         err = profile->init_tx(priv);
5420         if (err)
5421                 goto out;
5422
5423         err = profile->init_rx(priv);
5424         if (err)
5425                 goto err_cleanup_tx;
5426
5427         if (profile->enable)
5428                 profile->enable(priv);
5429
5430         mlx5e_update_features(priv->netdev);
5431
5432         return 0;
5433
5434 err_cleanup_tx:
5435         profile->cleanup_tx(priv);
5436
5437 out:
5438         mlx5e_reset_channels(priv->netdev);
5439         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5440         cancel_work_sync(&priv->update_stats_work);
5441         return err;
5442 }
5443
5444 void mlx5e_detach_netdev(struct mlx5e_priv *priv)
5445 {
5446         const struct mlx5e_profile *profile = priv->profile;
5447
5448         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5449
5450         if (profile->disable)
5451                 profile->disable(priv);
5452         flush_workqueue(priv->wq);
5453
5454         profile->cleanup_rx(priv);
5455         profile->cleanup_tx(priv);
5456         mlx5e_reset_channels(priv->netdev);
5457         cancel_work_sync(&priv->update_stats_work);
5458 }
5459
5460 static int
5461 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
5462                             const struct mlx5e_profile *new_profile, void *new_ppriv)
5463 {
5464         struct mlx5e_priv *priv = netdev_priv(netdev);
5465         int err;
5466
5467         err = mlx5e_priv_init(priv, new_profile, netdev, mdev);
5468         if (err) {
5469                 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
5470                 return err;
5471         }
5472         netif_carrier_off(netdev);
5473         priv->profile = new_profile;
5474         priv->ppriv = new_ppriv;
5475         err = new_profile->init(priv->mdev, priv->netdev);
5476         if (err)
5477                 goto priv_cleanup;
5478         err = mlx5e_attach_netdev(priv);
5479         if (err)
5480                 goto profile_cleanup;
5481         return err;
5482
5483 profile_cleanup:
5484         new_profile->cleanup(priv);
5485 priv_cleanup:
5486         mlx5e_priv_cleanup(priv);
5487         return err;
5488 }
5489
5490 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
5491                                 const struct mlx5e_profile *new_profile, void *new_ppriv)
5492 {
5493         const struct mlx5e_profile *orig_profile = priv->profile;
5494         struct net_device *netdev = priv->netdev;
5495         struct mlx5_core_dev *mdev = priv->mdev;
5496         void *orig_ppriv = priv->ppriv;
5497         int err, rollback_err;
5498
5499         /* cleanup old profile */
5500         mlx5e_detach_netdev(priv);
5501         priv->profile->cleanup(priv);
5502         mlx5e_priv_cleanup(priv);
5503
5504         err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv);
5505         if (err) { /* roll back to original profile */
5506                 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err);
5507                 goto rollback;
5508         }
5509
5510         return 0;
5511
5512 rollback:
5513         rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv);
5514         if (rollback_err)
5515                 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n",
5516                            __func__, rollback_err);
5517         return err;
5518 }
5519
5520 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv)
5521 {
5522         mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL);
5523 }
5524
5525 void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
5526 {
5527         struct net_device *netdev = priv->netdev;
5528
5529         mlx5e_priv_cleanup(priv);
5530         free_netdev(netdev);
5531 }
5532
5533 static int mlx5e_resume(struct auxiliary_device *adev)
5534 {
5535         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5536         struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5537         struct net_device *netdev = priv->netdev;
5538         struct mlx5_core_dev *mdev = edev->mdev;
5539         int err;
5540
5541         if (netif_device_present(netdev))
5542                 return 0;
5543
5544         err = mlx5e_create_mdev_resources(mdev);
5545         if (err)
5546                 return err;
5547
5548         err = mlx5e_attach_netdev(priv);
5549         if (err) {
5550                 mlx5e_destroy_mdev_resources(mdev);
5551                 return err;
5552         }
5553
5554         return 0;
5555 }
5556
5557 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
5558 {
5559         struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5560         struct net_device *netdev = priv->netdev;
5561         struct mlx5_core_dev *mdev = priv->mdev;
5562
5563         if (!netif_device_present(netdev))
5564                 return -ENODEV;
5565
5566         mlx5e_detach_netdev(priv);
5567         mlx5e_destroy_mdev_resources(mdev);
5568         return 0;
5569 }
5570
5571 static int mlx5e_probe(struct auxiliary_device *adev,
5572                        const struct auxiliary_device_id *id)
5573 {
5574         struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
5575         const struct mlx5e_profile *profile = &mlx5e_nic_profile;
5576         struct mlx5_core_dev *mdev = edev->mdev;
5577         struct net_device *netdev;
5578         pm_message_t state = {};
5579         struct mlx5e_priv *priv;
5580         int err;
5581
5582         netdev = mlx5e_create_netdev(mdev, profile);
5583         if (!netdev) {
5584                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
5585                 return -ENOMEM;
5586         }
5587
5588         mlx5e_build_nic_netdev(netdev);
5589
5590         priv = netdev_priv(netdev);
5591         auxiliary_set_drvdata(adev, priv);
5592
5593         priv->profile = profile;
5594         priv->ppriv = NULL;
5595
5596         err = mlx5e_devlink_port_register(priv);
5597         if (err) {
5598                 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err);
5599                 goto err_destroy_netdev;
5600         }
5601
5602         err = profile->init(mdev, netdev);
5603         if (err) {
5604                 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err);
5605                 goto err_devlink_cleanup;
5606         }
5607
5608         err = mlx5e_resume(adev);
5609         if (err) {
5610                 mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err);
5611                 goto err_profile_cleanup;
5612         }
5613
5614         err = register_netdev(netdev);
5615         if (err) {
5616                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
5617                 goto err_resume;
5618         }
5619
5620         mlx5e_devlink_port_type_eth_set(priv);
5621
5622         mlx5e_dcbnl_init_app(priv);
5623         mlx5_uplink_netdev_set(mdev, netdev);
5624         return 0;
5625
5626 err_resume:
5627         mlx5e_suspend(adev, state);
5628 err_profile_cleanup:
5629         profile->cleanup(priv);
5630 err_devlink_cleanup:
5631         mlx5e_devlink_port_unregister(priv);
5632 err_destroy_netdev:
5633         mlx5e_destroy_netdev(priv);
5634         return err;
5635 }
5636
5637 static void mlx5e_remove(struct auxiliary_device *adev)
5638 {
5639         struct mlx5e_priv *priv = auxiliary_get_drvdata(adev);
5640         pm_message_t state = {};
5641
5642         mlx5e_dcbnl_delete_app(priv);
5643         unregister_netdev(priv->netdev);
5644         mlx5e_suspend(adev, state);
5645         priv->profile->cleanup(priv);
5646         mlx5e_devlink_port_unregister(priv);
5647         mlx5e_destroy_netdev(priv);
5648 }
5649
5650 static const struct auxiliary_device_id mlx5e_id_table[] = {
5651         { .name = MLX5_ADEV_NAME ".eth", },
5652         {},
5653 };
5654
5655 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);
5656
5657 static struct auxiliary_driver mlx5e_driver = {
5658         .name = "eth",
5659         .probe = mlx5e_probe,
5660         .remove = mlx5e_remove,
5661         .suspend = mlx5e_suspend,
5662         .resume = mlx5e_resume,
5663         .id_table = mlx5e_id_table,
5664 };
5665
5666 int mlx5e_init(void)
5667 {
5668         int ret;
5669
5670         mlx5e_ipsec_build_inverse_table();
5671         mlx5e_build_ptys2ethtool_map();
5672         ret = auxiliary_driver_register(&mlx5e_driver);
5673         if (ret)
5674                 return ret;
5675
5676         ret = mlx5e_rep_init();
5677         if (ret)
5678                 auxiliary_driver_unregister(&mlx5e_driver);
5679         return ret;
5680 }
5681
5682 void mlx5e_cleanup(void)
5683 {
5684         mlx5e_rep_cleanup();
5685         auxiliary_driver_unregister(&mlx5e_driver);
5686 }