drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2020 Mellanox Technologies. */
   3
   4 #include <net/dst_metadata.h>
   5 #include <linux/netdevice.h>
   6 #include <linux/list.h>
   7 #include <linux/rculist.h>
   8 #include <linux/rtnetlink.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/spinlock.h>
  11 #include "tc.h"
  12 #include "neigh.h"
  13 #include "en_rep.h"
  14 #include "eswitch.h"
  15 #include "lib/fs_chains.h"
  16 #include "en/tc_ct.h"
  17 #include "en/mapping.h"
  18 #include "en/tc_tun.h"
  19 #include "lib/port_tun.h"
  20 #include "en/tc/sample.h"
  21
  22 struct mlx5e_rep_indr_block_priv {
  23         struct net_device *netdev;
  24         struct mlx5e_rep_priv *rpriv;
  25
  26         struct list_head list;
  27 };
  28
  29 int mlx5e_rep_encap_entry_attach(struct mlx5e_priv *priv,
  30                                  struct mlx5e_encap_entry *e,
  31                                  struct mlx5e_neigh *m_neigh,
  32                                  struct net_device *neigh_dev)
  33 {
  34         struct mlx5e_rep_priv *rpriv = priv->ppriv;
  35         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
  36         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
  37         struct mlx5e_neigh_hash_entry *nhe;
  38         int err;
  39
  40         err = mlx5_tun_entropy_refcount_inc(tun_entropy, e->reformat_type);
  41         if (err)
  42                 return err;
  43
  44         mutex_lock(&rpriv->neigh_update.encap_lock);
  45         nhe = mlx5e_rep_neigh_entry_lookup(priv, m_neigh);
  46         if (!nhe) {
  47                 err = mlx5e_rep_neigh_entry_create(priv, m_neigh, neigh_dev, &nhe);
  48                 if (err) {
  49                         mutex_unlock(&rpriv->neigh_update.encap_lock);
  50                         mlx5_tun_entropy_refcount_dec(tun_entropy,
  51                                                       e->reformat_type);
  52                         return err;
  53                 }
  54         }
  55
  56         e->nhe = nhe;
  57         spin_lock(&nhe->encap_list_lock);
  58         list_add_rcu(&e->encap_list, &nhe->encap_list);
  59         spin_unlock(&nhe->encap_list_lock);
  60
  61         mutex_unlock(&rpriv->neigh_update.encap_lock);
  62
  63         return 0;
  64 }
  65
  66 void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv,
  67                                   struct mlx5e_encap_entry *e)
  68 {
  69         struct mlx5e_rep_priv *rpriv = priv->ppriv;
  70         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
  71         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
  72
  73         if (!e->nhe)
  74                 return;
  75
  76         spin_lock(&e->nhe->encap_list_lock);
  77         list_del_rcu(&e->encap_list);
  78         spin_unlock(&e->nhe->encap_list_lock);
  79
  80         mlx5e_rep_neigh_entry_release(e->nhe);
  81         e->nhe = NULL;
  82         mlx5_tun_entropy_refcount_dec(tun_entropy, e->reformat_type);
  83 }
  84
  85 void mlx5e_rep_update_flows(struct mlx5e_priv *priv,
  86                             struct mlx5e_encap_entry *e,
  87                             bool neigh_connected,
  88                             unsigned char ha[ETH_ALEN])
  89 {
  90         struct ethhdr *eth = (struct ethhdr *)e->encap_header;
  91         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
  92         bool encap_connected;
  93         LIST_HEAD(flow_list);
  94
  95         ASSERT_RTNL();
  96
  97         mutex_lock(&esw->offloads.encap_tbl_lock);
  98         encap_connected = !!(e->flags & MLX5_ENCAP_ENTRY_VALID);
  99         if (encap_connected == neigh_connected && ether_addr_equal(e->h_dest, ha))
 100                 goto unlock;
 101
 102         mlx5e_take_all_encap_flows(e, &flow_list);
 103
 104         if ((e->flags & MLX5_ENCAP_ENTRY_VALID) &&
 105             (!neigh_connected || !ether_addr_equal(e->h_dest, ha)))
 106                 mlx5e_tc_encap_flows_del(priv, e, &flow_list);
 107
 108         if (neigh_connected && !(e->flags & MLX5_ENCAP_ENTRY_VALID)) {
 109                 struct net_device *route_dev;
 110
 111                 ether_addr_copy(e->h_dest, ha);
 112                 ether_addr_copy(eth->h_dest, ha);
 113                 /* Update the encap source mac, in case that we delete
 114                  * the flows when encap source mac changed.
 115                  */
 116                 route_dev = __dev_get_by_index(dev_net(priv->netdev), e->route_dev_ifindex);
 117                 if (route_dev)
 118                         ether_addr_copy(eth->h_source, route_dev->dev_addr);
 119
 120                 mlx5e_tc_encap_flows_add(priv, e, &flow_list);
 121         }
 122 unlock:
 123         mutex_unlock(&esw->offloads.encap_tbl_lock);
 124         mlx5e_put_flow_list(priv, &flow_list);
 125 }
 126
 127 static int
 128 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
 129                               struct flow_cls_offload *cls_flower, int flags)
 130 {
 131         switch (cls_flower->command) {
 132         case FLOW_CLS_REPLACE:
 133                 return mlx5e_configure_flower(priv->netdev, priv, cls_flower,
 134                                               flags);
 135         case FLOW_CLS_DESTROY:
 136                 return mlx5e_delete_flower(priv->netdev, priv, cls_flower,
 137                                            flags);
 138         case FLOW_CLS_STATS:
 139                 return mlx5e_stats_flower(priv->netdev, priv, cls_flower,
 140                                           flags);
 141         default:
 142                 return -EOPNOTSUPP;
 143         }
 144 }
 145
 146 static
 147 int mlx5e_rep_setup_tc_cls_matchall(struct mlx5e_priv *priv,
 148                                     struct tc_cls_matchall_offload *ma)
 149 {
 150         switch (ma->command) {
 151         case TC_CLSMATCHALL_REPLACE:
 152                 return mlx5e_tc_configure_matchall(priv, ma);
 153         case TC_CLSMATCHALL_DESTROY:
 154                 return mlx5e_tc_delete_matchall(priv, ma);
 155         case TC_CLSMATCHALL_STATS:
 156                 mlx5e_tc_stats_matchall(priv, ma);
 157                 return 0;
 158         default:
 159                 return -EOPNOTSUPP;
 160         }
 161 }
 162
 163 static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data,
 164                                  void *cb_priv)
 165 {
 166         unsigned long flags = MLX5_TC_FLAG(INGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
 167         struct mlx5e_priv *priv = cb_priv;
 168
 169         if (!priv->netdev || !netif_device_present(priv->netdev))
 170                 return -EOPNOTSUPP;
 171
 172         switch (type) {
 173         case TC_SETUP_CLSFLOWER:
 174                 return mlx5e_rep_setup_tc_cls_flower(priv, type_data, flags);
 175         case TC_SETUP_CLSMATCHALL:
 176                 return mlx5e_rep_setup_tc_cls_matchall(priv, type_data);
 177         default:
 178                 return -EOPNOTSUPP;
 179         }
 180 }
 181
 182 static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data,
 183                                  void *cb_priv)
 184 {
 185         struct flow_cls_offload tmp, *f = type_data;
 186         struct mlx5e_priv *priv = cb_priv;
 187         struct mlx5_eswitch *esw;
 188         unsigned long flags;
 189         int err;
 190
 191         flags = MLX5_TC_FLAG(INGRESS) |
 192                 MLX5_TC_FLAG(ESW_OFFLOAD) |
 193                 MLX5_TC_FLAG(FT_OFFLOAD);
 194         esw = priv->mdev->priv.eswitch;
 195
 196         switch (type) {
 197         case TC_SETUP_CLSFLOWER:
 198                 memcpy(&tmp, f, sizeof(*f));
 199
 200                 if (!mlx5_chains_prios_supported(esw_chains(esw)))
 201                         return -EOPNOTSUPP;
 202
 203                 /* Re-use tc offload path by moving the ft flow to the
 204                  * reserved ft chain.
 205                  *
 206                  * FT offload can use prio range [0, INT_MAX], so we normalize
 207                  * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
 208                  * as with tc, where prio 0 isn't supported.
 209                  *
 210                  * We only support chain 0 of FT offload.
 211                  */
 212                 if (tmp.common.prio >= mlx5_chains_get_prio_range(esw_chains(esw)))
 213                         return -EOPNOTSUPP;
 214                 if (tmp.common.chain_index != 0)
 215                         return -EOPNOTSUPP;
 216
 217                 tmp.common.chain_index = mlx5_chains_get_nf_ft_chain(esw_chains(esw));
 218                 tmp.common.prio++;
 219                 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);
 220                 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
 221                 return err;
 222         default:
 223                 return -EOPNOTSUPP;
 224         }
 225 }
 226
 227 static LIST_HEAD(mlx5e_rep_block_tc_cb_list);
 228 static LIST_HEAD(mlx5e_rep_block_ft_cb_list);
 229 int mlx5e_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
 230                        void *type_data)
 231 {
 232         struct mlx5e_priv *priv = netdev_priv(dev);
 233         struct flow_block_offload *f = type_data;
 234
 235         f->unlocked_driver_cb = true;
 236
 237         switch (type) {
 238         case TC_SETUP_BLOCK:
 239                 return flow_block_cb_setup_simple(type_data,
 240                                                   &mlx5e_rep_block_tc_cb_list,
 241                                                   mlx5e_rep_setup_tc_cb,
 242                                                   priv, priv, true);
 243         case TC_SETUP_FT:
 244                 return flow_block_cb_setup_simple(type_data,
 245                                                   &mlx5e_rep_block_ft_cb_list,
 246                                                   mlx5e_rep_setup_ft_cb,
 247                                                   priv, priv, true);
 248         default:
 249                 return -EOPNOTSUPP;
 250         }
 251 }
 252
 253 int mlx5e_rep_tc_init(struct mlx5e_rep_priv *rpriv)
 254 {
 255         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
 256         int err;
 257
 258         mutex_init(&uplink_priv->unready_flows_lock);
 259         INIT_LIST_HEAD(&uplink_priv->unready_flows);
 260
 261         /* init shared tc flow table */
 262         err = mlx5e_tc_esw_init(&uplink_priv->tc_ht);
 263         return err;
 264 }
 265
 266 void mlx5e_rep_tc_cleanup(struct mlx5e_rep_priv *rpriv)
 267 {
 268         /* delete shared tc flow table */
 269         mlx5e_tc_esw_cleanup(&rpriv->uplink_priv.tc_ht);
 270         mutex_destroy(&rpriv->uplink_priv.unready_flows_lock);
 271 }
 272
 273 void mlx5e_rep_tc_enable(struct mlx5e_priv *priv)
 274 {
 275         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 276
 277         INIT_WORK(&rpriv->uplink_priv.reoffload_flows_work,
 278                   mlx5e_tc_reoffload_flows_work);
 279 }
 280
 281 void mlx5e_rep_tc_disable(struct mlx5e_priv *priv)
 282 {
 283         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 284
 285         cancel_work_sync(&rpriv->uplink_priv.reoffload_flows_work);
 286 }
 287
 288 int mlx5e_rep_tc_event_port_affinity(struct mlx5e_priv *priv)
 289 {
 290         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 291
 292         queue_work(priv->wq, &rpriv->uplink_priv.reoffload_flows_work);
 293
 294         return NOTIFY_OK;
 295 }
 296
 297 static struct mlx5e_rep_indr_block_priv *
 298 mlx5e_rep_indr_block_priv_lookup(struct mlx5e_rep_priv *rpriv,
 299                                  struct net_device *netdev)
 300 {
 301         struct mlx5e_rep_indr_block_priv *cb_priv;
 302
 303         list_for_each_entry(cb_priv,
 304                             &rpriv->uplink_priv.tc_indr_block_priv_list,
 305                             list)
 306                 if (cb_priv->netdev == netdev)
 307                         return cb_priv;
 308
 309         return NULL;
 310 }
 311
 312 static int
 313 mlx5e_rep_indr_offload(struct net_device *netdev,
 314                        struct flow_cls_offload *flower,
 315                        struct mlx5e_rep_indr_block_priv *indr_priv,
 316                        unsigned long flags)
 317 {
 318         struct mlx5e_priv *priv = netdev_priv(indr_priv->rpriv->netdev);
 319         int err = 0;
 320
 321         if (!netif_device_present(indr_priv->rpriv->netdev))
 322                 return -EOPNOTSUPP;
 323
 324         switch (flower->command) {
 325         case FLOW_CLS_REPLACE:
 326                 err = mlx5e_configure_flower(netdev, priv, flower, flags);
 327                 break;
 328         case FLOW_CLS_DESTROY:
 329                 err = mlx5e_delete_flower(netdev, priv, flower, flags);
 330                 break;
 331         case FLOW_CLS_STATS:
 332                 err = mlx5e_stats_flower(netdev, priv, flower, flags);
 333                 break;
 334         default:
 335                 err = -EOPNOTSUPP;
 336         }
 337
 338         return err;
 339 }
 340
 341 static int mlx5e_rep_indr_setup_tc_cb(enum tc_setup_type type,
 342                                       void *type_data, void *indr_priv)
 343 {
 344         unsigned long flags = MLX5_TC_FLAG(EGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
 345         struct mlx5e_rep_indr_block_priv *priv = indr_priv;
 346
 347         switch (type) {
 348         case TC_SETUP_CLSFLOWER:
 349                 return mlx5e_rep_indr_offload(priv->netdev, type_data, priv,
 350                                               flags);
 351         default:
 352                 return -EOPNOTSUPP;
 353         }
 354 }
 355
 356 static int mlx5e_rep_indr_setup_ft_cb(enum tc_setup_type type,
 357                                       void *type_data, void *indr_priv)
 358 {
 359         struct mlx5e_rep_indr_block_priv *priv = indr_priv;
 360         struct flow_cls_offload *f = type_data;
 361         struct flow_cls_offload tmp;
 362         struct mlx5e_priv *mpriv;
 363         struct mlx5_eswitch *esw;
 364         unsigned long flags;
 365         int err;
 366
 367         mpriv = netdev_priv(priv->rpriv->netdev);
 368         esw = mpriv->mdev->priv.eswitch;
 369
 370         flags = MLX5_TC_FLAG(EGRESS) |
 371                 MLX5_TC_FLAG(ESW_OFFLOAD) |
 372                 MLX5_TC_FLAG(FT_OFFLOAD);
 373
 374         switch (type) {
 375         case TC_SETUP_CLSFLOWER:
 376                 memcpy(&tmp, f, sizeof(*f));
 377
 378                 /* Re-use tc offload path by moving the ft flow to the
 379                  * reserved ft chain.
 380                  *
 381                  * FT offload can use prio range [0, INT_MAX], so we normalize
 382                  * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
 383                  * as with tc, where prio 0 isn't supported.
 384                  *
 385                  * We only support chain 0 of FT offload.
 386                  */
 387                 if (!mlx5_chains_prios_supported(esw_chains(esw)) ||
 388                     tmp.common.prio >= mlx5_chains_get_prio_range(esw_chains(esw)) ||
 389                     tmp.common.chain_index)
 390                         return -EOPNOTSUPP;
 391
 392                 tmp.common.chain_index = mlx5_chains_get_nf_ft_chain(esw_chains(esw));
 393                 tmp.common.prio++;
 394                 err = mlx5e_rep_indr_offload(priv->netdev, &tmp, priv, flags);
 395                 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
 396                 return err;
 397         default:
 398                 return -EOPNOTSUPP;
 399         }
 400 }
 401
 402 static void mlx5e_rep_indr_block_unbind(void *cb_priv)
 403 {
 404         struct mlx5e_rep_indr_block_priv *indr_priv = cb_priv;
 405
 406         list_del(&indr_priv->list);
 407         kfree(indr_priv);
 408 }
 409
 410 static LIST_HEAD(mlx5e_block_cb_list);
 411
 412 static int
 413 mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch,
 414                            struct mlx5e_rep_priv *rpriv,
 415                            struct flow_block_offload *f,
 416                            flow_setup_cb_t *setup_cb,
 417                            void *data,
 418                            void (*cleanup)(struct flow_block_cb *block_cb))
 419 {
 420         struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 421         struct mlx5e_rep_indr_block_priv *indr_priv;
 422         struct flow_block_cb *block_cb;
 423
 424         if (!mlx5e_tc_tun_device_to_offload(priv, netdev) &&
 425             !(is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev))
 426                 return -EOPNOTSUPP;
 427
 428         if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
 429                 return -EOPNOTSUPP;
 430
 431         f->unlocked_driver_cb = true;
 432         f->driver_block_list = &mlx5e_block_cb_list;
 433
 434         switch (f->command) {
 435         case FLOW_BLOCK_BIND:
 436                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 437                 if (indr_priv)
 438                         return -EEXIST;
 439
 440                 indr_priv = kmalloc(sizeof(*indr_priv), GFP_KERNEL);
 441                 if (!indr_priv)
 442                         return -ENOMEM;
 443
 444                 indr_priv->netdev = netdev;
 445                 indr_priv->rpriv = rpriv;
 446                 list_add(&indr_priv->list,
 447                          &rpriv->uplink_priv.tc_indr_block_priv_list);
 448
 449                 block_cb = flow_indr_block_cb_alloc(setup_cb, indr_priv, indr_priv,
 450                                                     mlx5e_rep_indr_block_unbind,
 451                                                     f, netdev, sch, data, rpriv,
 452                                                     cleanup);
 453                 if (IS_ERR(block_cb)) {
 454                         list_del(&indr_priv->list);
 455                         kfree(indr_priv);
 456                         return PTR_ERR(block_cb);
 457                 }
 458                 flow_block_cb_add(block_cb, f);
 459                 list_add_tail(&block_cb->driver_list, &mlx5e_block_cb_list);
 460
 461                 return 0;
 462         case FLOW_BLOCK_UNBIND:
 463                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 464                 if (!indr_priv)
 465                         return -ENOENT;
 466
 467                 block_cb = flow_block_cb_lookup(f->block, setup_cb, indr_priv);
 468                 if (!block_cb)
 469                         return -ENOENT;
 470
 471                 flow_indr_block_cb_remove(block_cb, f);
 472                 list_del(&block_cb->driver_list);
 473                 return 0;
 474         default:
 475                 return -EOPNOTSUPP;
 476         }
 477         return 0;
 478 }
 479
 480 static
 481 int mlx5e_rep_indr_setup_cb(struct net_device *netdev, struct Qdisc *sch, void *cb_priv,
 482                             enum tc_setup_type type, void *type_data,
 483                             void *data,
 484                             void (*cleanup)(struct flow_block_cb *block_cb))
 485 {
 486         switch (type) {
 487         case TC_SETUP_BLOCK:
 488                 return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data,
 489                                                   mlx5e_rep_indr_setup_tc_cb,
 490                                                   data, cleanup);
 491         case TC_SETUP_FT:
 492                 return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data,
 493                                                   mlx5e_rep_indr_setup_ft_cb,
 494                                                   data, cleanup);
 495         default:
 496                 return -EOPNOTSUPP;
 497         }
 498 }
 499
 500 int mlx5e_rep_tc_netdevice_event_register(struct mlx5e_rep_priv *rpriv)
 501 {
 502         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
 503
 504         /* init indirect block notifications */
 505         INIT_LIST_HEAD(&uplink_priv->tc_indr_block_priv_list);
 506
 507         return flow_indr_dev_register(mlx5e_rep_indr_setup_cb, rpriv);
 508 }
 509
 510 void mlx5e_rep_tc_netdevice_event_unregister(struct mlx5e_rep_priv *rpriv)
 511 {
 512         flow_indr_dev_unregister(mlx5e_rep_indr_setup_cb, rpriv,
 513                                  mlx5e_rep_indr_block_unbind);
 514 }
 515
 516 static bool mlx5e_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb,
 517                                  struct mlx5e_tc_update_priv *tc_priv,
 518                                  u32 tunnel_id)
 519 {
 520         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 521         struct tunnel_match_enc_opts enc_opts = {};
 522         struct mlx5_rep_uplink_priv *uplink_priv;
 523         struct mlx5e_rep_priv *uplink_rpriv;
 524         struct metadata_dst *tun_dst;
 525         struct tunnel_match_key key;
 526         u32 tun_id, enc_opts_id;
 527         struct net_device *dev;
 528         int err;
 529
 530         enc_opts_id = tunnel_id & ENC_OPTS_BITS_MASK;
 531         tun_id = tunnel_id >> ENC_OPTS_BITS;
 532
 533         if (!tun_id)
 534                 return true;
 535
 536         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 537         uplink_priv = &uplink_rpriv->uplink_priv;
 538
 539         err = mapping_find(uplink_priv->tunnel_mapping, tun_id, &key);
 540         if (err) {
 541                 WARN_ON_ONCE(true);
 542                 netdev_dbg(priv->netdev,
 543                            "Couldn't find tunnel for tun_id: %d, err: %d\n",
 544                            tun_id, err);
 545                 return false;
 546         }
 547
 548         if (enc_opts_id) {
 549                 err = mapping_find(uplink_priv->tunnel_enc_opts_mapping,
 550                                    enc_opts_id, &enc_opts);
 551                 if (err) {
 552                         netdev_dbg(priv->netdev,
 553                                    "Couldn't find tunnel (opts) for tun_id: %d, err: %d\n",
 554                                    enc_opts_id, err);
 555                         return false;
 556                 }
 557         }
 558
 559         if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 560                 tun_dst = __ip_tun_set_dst(key.enc_ipv4.src, key.enc_ipv4.dst,
 561                                            key.enc_ip.tos, key.enc_ip.ttl,
 562                                            key.enc_tp.dst, TUNNEL_KEY,
 563                                            key32_to_tunnel_id(key.enc_key_id.keyid),
 564                                            enc_opts.key.len);
 565         } else if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 566                 tun_dst = __ipv6_tun_set_dst(&key.enc_ipv6.src, &key.enc_ipv6.dst,
 567                                              key.enc_ip.tos, key.enc_ip.ttl,
 568                                              key.enc_tp.dst, 0, TUNNEL_KEY,
 569                                              key32_to_tunnel_id(key.enc_key_id.keyid),
 570                                              enc_opts.key.len);
 571         } else {
 572                 netdev_dbg(priv->netdev,
 573                            "Couldn't restore tunnel, unsupported addr_type: %d\n",
 574                            key.enc_control.addr_type);
 575                 return false;
 576         }
 577
 578         if (!tun_dst) {
 579                 netdev_dbg(priv->netdev, "Couldn't restore tunnel, no tun_dst\n");
 580                 return false;
 581         }
 582
 583         tun_dst->u.tun_info.key.tp_src = key.enc_tp.src;
 584
 585         if (enc_opts.key.len)
 586                 ip_tunnel_info_opts_set(&tun_dst->u.tun_info,
 587                                         enc_opts.key.data,
 588                                         enc_opts.key.len,
 589                                         enc_opts.key.dst_opt_type);
 590
 591         skb_dst_set(skb, (struct dst_entry *)tun_dst);
 592         dev = dev_get_by_index(&init_net, key.filter_ifindex);
 593         if (!dev) {
 594                 netdev_dbg(priv->netdev,
 595                            "Couldn't find tunnel device with ifindex: %d\n",
 596                            key.filter_ifindex);
 597                 return false;
 598         }
 599
 600         /* Set tun_dev so we do dev_put() after datapath */
 601         tc_priv->tun_dev = dev;
 602
 603         skb->dev = dev;
 604
 605         return true;
 606 }
 607
 608 static bool mlx5e_restore_skb_chain(struct sk_buff *skb, u32 chain, u32 reg_c1,
 609                                     struct mlx5e_tc_update_priv *tc_priv)
 610 {
 611         struct mlx5e_priv *priv = netdev_priv(skb->dev);
 612         u32 tunnel_id = (reg_c1 >> ESW_TUN_OFFSET) & TUNNEL_ID_MASK;
 613
 614 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 615         if (chain) {
 616                 struct mlx5_rep_uplink_priv *uplink_priv;
 617                 struct mlx5e_rep_priv *uplink_rpriv;
 618                 struct tc_skb_ext *tc_skb_ext;
 619                 struct mlx5_eswitch *esw;
 620                 u32 zone_restore_id;
 621
 622                 tc_skb_ext = tc_skb_ext_alloc(skb);
 623                 if (!tc_skb_ext) {
 624                         WARN_ON(1);
 625                         return false;
 626                 }
 627                 tc_skb_ext->chain = chain;
 628                 zone_restore_id = reg_c1 & ESW_ZONE_ID_MASK;
 629                 esw = priv->mdev->priv.eswitch;
 630                 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 631                 uplink_priv = &uplink_rpriv->uplink_priv;
 632                 if (!mlx5e_tc_ct_restore_flow(uplink_priv->ct_priv, skb,
 633                                               zone_restore_id))
 634                         return false;
 635         }
 636 #endif /* CONFIG_NET_TC_SKB_EXT */
 637
 638         return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id);
 639 }
 640
 641 static void mlx5e_restore_skb_sample(struct mlx5e_priv *priv, struct sk_buff *skb,
 642                                      struct mlx5_mapped_obj *mapped_obj,
 643                                      struct mlx5e_tc_update_priv *tc_priv)
 644 {
 645         if (!mlx5e_restore_tunnel(priv, skb, tc_priv, mapped_obj->sample.tunnel_id)) {
 646                 netdev_dbg(priv->netdev,
 647                            "Failed to restore tunnel info for sampled packet\n");
 648                 return;
 649         }
 650 #if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
 651         mlx5e_tc_sample_skb(skb, mapped_obj);
 652 #endif /* CONFIG_MLX5_TC_SAMPLE */
 653         mlx5_rep_tc_post_napi_receive(tc_priv);
 654 }
 655
 656 bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe,
 657                              struct sk_buff *skb,
 658                              struct mlx5e_tc_update_priv *tc_priv)
 659 {
 660         struct mlx5_mapped_obj mapped_obj;
 661         struct mlx5_eswitch *esw;
 662         struct mlx5e_priv *priv;
 663         u32 reg_c0;
 664         int err;
 665
 666         reg_c0 = (be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK);
 667         if (!reg_c0 || reg_c0 == MLX5_FS_DEFAULT_FLOW_TAG)
 668                 return true;
 669
 670         /* If reg_c0 is not equal to the default flow tag then skb->mark
 671          * is not supported and must be reset back to 0.
 672          */
 673         skb->mark = 0;
 674
 675         priv = netdev_priv(skb->dev);
 676         esw = priv->mdev->priv.eswitch;
 677         err = mapping_find(esw->offloads.reg_c0_obj_pool, reg_c0, &mapped_obj);
 678         if (err) {
 679                 netdev_dbg(priv->netdev,
 680                            "Couldn't find mapped object for reg_c0: %d, err: %d\n",
 681                            reg_c0, err);
 682                 return false;
 683         }
 684
 685         if (mapped_obj.type == MLX5_MAPPED_OBJ_CHAIN) {
 686                 u32 reg_c1 = be32_to_cpu(cqe->ft_metadata);
 687
 688                 return mlx5e_restore_skb_chain(skb, mapped_obj.chain, reg_c1, tc_priv);
 689         } else if (mapped_obj.type == MLX5_MAPPED_OBJ_SAMPLE) {
 690                 mlx5e_restore_skb_sample(priv, skb, &mapped_obj, tc_priv);
 691                 return false;
 692         } else {
 693                 netdev_dbg(priv->netdev, "Invalid mapped object type: %d\n", mapped_obj.type);
 694                 return false;
 695         }
 696
 697         return true;
 698 }
 699
 700 void mlx5_rep_tc_post_napi_receive(struct mlx5e_tc_update_priv *tc_priv)
 701 {
 702         if (tc_priv->tun_dev)
 703                 dev_put(tc_priv->tun_dev);
 704 }