drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/if_macvlan.h>
  17 #include <linux/refcount.h>
  18 #include <linux/jhash.h>
  19 #include <linux/net_namespace.h>
  20 #include <linux/mutex.h>
  21 #include <linux/genalloc.h>
  22 #include <net/netevent.h>
  23 #include <net/neighbour.h>
  24 #include <net/arp.h>
  25 #include <net/inet_dscp.h>
  26 #include <net/ip_fib.h>
  27 #include <net/ip6_fib.h>
  28 #include <net/nexthop.h>
  29 #include <net/fib_rules.h>
  30 #include <net/ip_tunnels.h>
  31 #include <net/l3mdev.h>
  32 #include <net/addrconf.h>
  33 #include <net/ndisc.h>
  34 #include <net/ipv6.h>
  35 #include <net/fib_notifier.h>
  36 #include <net/switchdev.h>
  37
  38 #include "spectrum.h"
  39 #include "core.h"
  40 #include "reg.h"
  41 #include "spectrum_cnt.h"
  42 #include "spectrum_dpipe.h"
  43 #include "spectrum_ipip.h"
  44 #include "spectrum_mr.h"
  45 #include "spectrum_mr_tcam.h"
  46 #include "spectrum_router.h"
  47 #include "spectrum_span.h"
  48
  49 struct mlxsw_sp_fib;
  50 struct mlxsw_sp_vr;
  51 struct mlxsw_sp_lpm_tree;
  52 struct mlxsw_sp_rif_ops;
  53
  54 struct mlxsw_sp_crif_key {
  55         struct net_device *dev;
  56 };
  57
  58 struct mlxsw_sp_crif {
  59         struct mlxsw_sp_crif_key key;
  60         struct rhash_head ht_node;
  61         bool can_destroy;
  62         struct list_head nexthop_list;
  63         struct mlxsw_sp_rif *rif;
  64 };
  65
  66 static const struct rhashtable_params mlxsw_sp_crif_ht_params = {
  67         .key_offset = offsetof(struct mlxsw_sp_crif, key),
  68         .key_len = sizeof_field(struct mlxsw_sp_crif, key),
  69         .head_offset = offsetof(struct mlxsw_sp_crif, ht_node),
  70 };
  71
  72 struct mlxsw_sp_rif {
  73         struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */
  74         netdevice_tracker dev_tracker;
  75         struct list_head neigh_list;
  76         struct mlxsw_sp_fid *fid;
  77         unsigned char addr[ETH_ALEN];
  78         int mtu;
  79         u16 rif_index;
  80         u8 mac_profile_id;
  81         u8 rif_entries;
  82         u16 vr_id;
  83         const struct mlxsw_sp_rif_ops *ops;
  84         struct mlxsw_sp *mlxsw_sp;
  85
  86         unsigned int counter_ingress;
  87         bool counter_ingress_valid;
  88         unsigned int counter_egress;
  89         bool counter_egress_valid;
  90 };
  91
  92 static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
  93 {
  94         if (!rif->crif)
  95                 return NULL;
  96         return rif->crif->key.dev;
  97 }
  98
  99 struct mlxsw_sp_rif_params {
 100         struct net_device *dev;
 101         union {
 102                 u16 system_port;
 103                 u16 lag_id;
 104         };
 105         u16 vid;
 106         bool lag;
 107         bool double_entry;
 108 };
 109
 110 struct mlxsw_sp_rif_subport {
 111         struct mlxsw_sp_rif common;
 112         refcount_t ref_count;
 113         union {
 114                 u16 system_port;
 115                 u16 lag_id;
 116         };
 117         u16 vid;
 118         bool lag;
 119 };
 120
 121 struct mlxsw_sp_rif_ipip_lb {
 122         struct mlxsw_sp_rif common;
 123         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 124         u16 ul_vr_id;   /* Spectrum-1. */
 125         u16 ul_rif_id;  /* Spectrum-2+. */
 126 };
 127
 128 struct mlxsw_sp_rif_params_ipip_lb {
 129         struct mlxsw_sp_rif_params common;
 130         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 131 };
 132
 133 struct mlxsw_sp_rif_ops {
 134         enum mlxsw_sp_rif_type type;
 135         size_t rif_size;
 136
 137         void (*setup)(struct mlxsw_sp_rif *rif,
 138                       const struct mlxsw_sp_rif_params *params);
 139         int (*configure)(struct mlxsw_sp_rif *rif,
 140                          struct netlink_ext_ack *extack);
 141         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 142         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 143                                          const struct mlxsw_sp_rif_params *params,
 144                                          struct netlink_ext_ack *extack);
 145         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 146 };
 147
 148 struct mlxsw_sp_rif_mac_profile {
 149         unsigned char mac_prefix[ETH_ALEN];
 150         refcount_t ref_count;
 151         u8 id;
 152 };
 153
 154 struct mlxsw_sp_router_ops {
 155         int (*init)(struct mlxsw_sp *mlxsw_sp);
 156         int (*ipips_init)(struct mlxsw_sp *mlxsw_sp);
 157 };
 158
 159 static struct mlxsw_sp_rif *
 160 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 161                          const struct net_device *dev);
 162 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 163 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 164 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 165                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 166 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 167                                      const struct mlxsw_sp_fib *fib,
 168                                      u8 tree_id);
 169 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 170                                        const struct mlxsw_sp_fib *fib);
 171
 172 static unsigned int *
 173 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 174                            enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return &rif->counter_egress;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return &rif->counter_ingress;
 181         }
 182         return NULL;
 183 }
 184
 185 static bool
 186 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir)
 188 {
 189         switch (dir) {
 190         case MLXSW_SP_RIF_COUNTER_EGRESS:
 191                 return rif->counter_egress_valid;
 192         case MLXSW_SP_RIF_COUNTER_INGRESS:
 193                 return rif->counter_ingress_valid;
 194         }
 195         return false;
 196 }
 197
 198 static void
 199 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 200                                enum mlxsw_sp_rif_counter_dir dir,
 201                                bool valid)
 202 {
 203         switch (dir) {
 204         case MLXSW_SP_RIF_COUNTER_EGRESS:
 205                 rif->counter_egress_valid = valid;
 206                 break;
 207         case MLXSW_SP_RIF_COUNTER_INGRESS:
 208                 rif->counter_ingress_valid = valid;
 209                 break;
 210         }
 211 }
 212
 213 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 214                                      unsigned int counter_index, bool enable,
 215                                      enum mlxsw_sp_rif_counter_dir dir)
 216 {
 217         char ritr_pl[MLXSW_REG_RITR_LEN];
 218         bool is_egress = false;
 219         int err;
 220
 221         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 222                 is_egress = true;
 223         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 224         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 225         if (err)
 226                 return err;
 227
 228         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 229                                     is_egress);
 230         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 231 }
 232
 233 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 234                                    struct mlxsw_sp_rif *rif,
 235                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 236 {
 237         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 238         unsigned int *p_counter_index;
 239         bool valid;
 240         int err;
 241
 242         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 243         if (!valid)
 244                 return -EINVAL;
 245
 246         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 247         if (!p_counter_index)
 248                 return -EINVAL;
 249         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 250                              MLXSW_REG_RICNT_OPCODE_NOP);
 251         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 252         if (err)
 253                 return err;
 254         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 255         return 0;
 256 }
 257
 258 struct mlxsw_sp_rif_counter_set_basic {
 259         u64 good_unicast_packets;
 260         u64 good_multicast_packets;
 261         u64 good_broadcast_packets;
 262         u64 good_unicast_bytes;
 263         u64 good_multicast_bytes;
 264         u64 good_broadcast_bytes;
 265         u64 error_packets;
 266         u64 discard_packets;
 267         u64 error_bytes;
 268         u64 discard_bytes;
 269 };
 270
 271 static int
 272 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif,
 273                                  enum mlxsw_sp_rif_counter_dir dir,
 274                                  struct mlxsw_sp_rif_counter_set_basic *set)
 275 {
 276         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 277         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 278         unsigned int *p_counter_index;
 279         int err;
 280
 281         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 282                 return -EINVAL;
 283
 284         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 285         if (!p_counter_index)
 286                 return -EINVAL;
 287
 288         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 289                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 290         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 291         if (err)
 292                 return err;
 293
 294         if (!set)
 295                 return 0;
 296
 297 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME)                              \
 298                 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl))
 299
 300         MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets);
 301         MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets);
 302         MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets);
 303         MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes);
 304         MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes);
 305         MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes);
 306         MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets);
 307         MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets);
 308         MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes);
 309         MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes);
 310
 311 #undef MLXSW_SP_RIF_COUNTER_EXTRACT
 312
 313         return 0;
 314 }
 315
 316 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 317                                       unsigned int counter_index)
 318 {
 319         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 320
 321         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 322                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 323         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 324 }
 325
 326 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif,
 327                                enum mlxsw_sp_rif_counter_dir dir)
 328 {
 329         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 330         unsigned int *p_counter_index;
 331         int err;
 332
 333         if (mlxsw_sp_rif_counter_valid_get(rif, dir))
 334                 return 0;
 335
 336         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 337         if (!p_counter_index)
 338                 return -EINVAL;
 339
 340         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 341                                      p_counter_index);
 342         if (err)
 343                 return err;
 344
 345         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 346         if (err)
 347                 goto err_counter_clear;
 348
 349         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 350                                         *p_counter_index, true, dir);
 351         if (err)
 352                 goto err_counter_edit;
 353         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 354         return 0;
 355
 356 err_counter_edit:
 357 err_counter_clear:
 358         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 359                               *p_counter_index);
 360         return err;
 361 }
 362
 363 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif,
 364                                enum mlxsw_sp_rif_counter_dir dir)
 365 {
 366         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 367         unsigned int *p_counter_index;
 368
 369         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 370                 return;
 371
 372         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 373         if (WARN_ON(!p_counter_index))
 374                 return;
 375         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 376                                   *p_counter_index, false, dir);
 377         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 378                               *p_counter_index);
 379         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 380 }
 381
 382 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 383 {
 384         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 385         struct devlink *devlink;
 386
 387         devlink = priv_to_devlink(mlxsw_sp->core);
 388         if (!devlink_dpipe_table_counter_enabled(devlink,
 389                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 390                 return;
 391         mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 392 }
 393
 394 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 395 {
 396         mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 397 }
 398
 399 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 400
 401 struct mlxsw_sp_prefix_usage {
 402         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 403 };
 404
 405 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 406         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 407
 408 static bool
 409 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 410                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 411 {
 412         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 413 }
 414
 415 static void
 416 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 417                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 418 {
 419         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 420 }
 421
 422 static void
 423 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 424                           unsigned char prefix_len)
 425 {
 426         set_bit(prefix_len, prefix_usage->b);
 427 }
 428
 429 static void
 430 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 431                             unsigned char prefix_len)
 432 {
 433         clear_bit(prefix_len, prefix_usage->b);
 434 }
 435
 436 struct mlxsw_sp_fib_key {
 437         unsigned char addr[sizeof(struct in6_addr)];
 438         unsigned char prefix_len;
 439 };
 440
 441 enum mlxsw_sp_fib_entry_type {
 442         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 443         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 444         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 445         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 446         MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 447
 448         /* This is a special case of local delivery, where a packet should be
 449          * decapsulated on reception. Note that there is no corresponding ENCAP,
 450          * because that's a type of next hop, not of FIB entry. (There can be
 451          * several next hops in a REMOTE entry, and some of them may be
 452          * encapsulating entries.)
 453          */
 454         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 455         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 456 };
 457
 458 struct mlxsw_sp_nexthop_group_info;
 459 struct mlxsw_sp_nexthop_group;
 460 struct mlxsw_sp_fib_entry;
 461
 462 struct mlxsw_sp_fib_node {
 463         struct mlxsw_sp_fib_entry *fib_entry;
 464         struct list_head list;
 465         struct rhash_head ht_node;
 466         struct mlxsw_sp_fib *fib;
 467         struct mlxsw_sp_fib_key key;
 468 };
 469
 470 struct mlxsw_sp_fib_entry_decap {
 471         struct mlxsw_sp_ipip_entry *ipip_entry;
 472         u32 tunnel_index;
 473 };
 474
 475 struct mlxsw_sp_fib_entry {
 476         struct mlxsw_sp_fib_node *fib_node;
 477         enum mlxsw_sp_fib_entry_type type;
 478         struct list_head nexthop_group_node;
 479         struct mlxsw_sp_nexthop_group *nh_group;
 480         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 481 };
 482
 483 struct mlxsw_sp_fib4_entry {
 484         struct mlxsw_sp_fib_entry common;
 485         struct fib_info *fi;
 486         u32 tb_id;
 487         dscp_t dscp;
 488         u8 type;
 489 };
 490
 491 struct mlxsw_sp_fib6_entry {
 492         struct mlxsw_sp_fib_entry common;
 493         struct list_head rt6_list;
 494         unsigned int nrt6;
 495 };
 496
 497 struct mlxsw_sp_rt6 {
 498         struct list_head list;
 499         struct fib6_info *rt;
 500 };
 501
 502 struct mlxsw_sp_lpm_tree {
 503         u8 id; /* tree ID */
 504         unsigned int ref_count;
 505         enum mlxsw_sp_l3proto proto;
 506         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 507         struct mlxsw_sp_prefix_usage prefix_usage;
 508 };
 509
 510 struct mlxsw_sp_fib {
 511         struct rhashtable ht;
 512         struct list_head node_list;
 513         struct mlxsw_sp_vr *vr;
 514         struct mlxsw_sp_lpm_tree *lpm_tree;
 515         enum mlxsw_sp_l3proto proto;
 516 };
 517
 518 struct mlxsw_sp_vr {
 519         u16 id; /* virtual router ID */
 520         u32 tb_id; /* kernel fib table id */
 521         unsigned int rif_count;
 522         struct mlxsw_sp_fib *fib4;
 523         struct mlxsw_sp_fib *fib6;
 524         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 525         struct mlxsw_sp_rif *ul_rif;
 526         refcount_t ul_rif_refcnt;
 527 };
 528
 529 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 530
 531 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 532                                                 struct mlxsw_sp_vr *vr,
 533                                                 enum mlxsw_sp_l3proto proto)
 534 {
 535         struct mlxsw_sp_lpm_tree *lpm_tree;
 536         struct mlxsw_sp_fib *fib;
 537         int err;
 538
 539         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 540         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 541         if (!fib)
 542                 return ERR_PTR(-ENOMEM);
 543         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 544         if (err)
 545                 goto err_rhashtable_init;
 546         INIT_LIST_HEAD(&fib->node_list);
 547         fib->proto = proto;
 548         fib->vr = vr;
 549         fib->lpm_tree = lpm_tree;
 550         mlxsw_sp_lpm_tree_hold(lpm_tree);
 551         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 552         if (err)
 553                 goto err_lpm_tree_bind;
 554         return fib;
 555
 556 err_lpm_tree_bind:
 557         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 558 err_rhashtable_init:
 559         kfree(fib);
 560         return ERR_PTR(err);
 561 }
 562
 563 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 564                                  struct mlxsw_sp_fib *fib)
 565 {
 566         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 567         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 568         WARN_ON(!list_empty(&fib->node_list));
 569         rhashtable_destroy(&fib->ht);
 570         kfree(fib);
 571 }
 572
 573 static struct mlxsw_sp_lpm_tree *
 574 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 575 {
 576         static struct mlxsw_sp_lpm_tree *lpm_tree;
 577         int i;
 578
 579         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 580                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 581                 if (lpm_tree->ref_count == 0)
 582                         return lpm_tree;
 583         }
 584         return NULL;
 585 }
 586
 587 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 588                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 589 {
 590         char ralta_pl[MLXSW_REG_RALTA_LEN];
 591
 592         mlxsw_reg_ralta_pack(ralta_pl, true,
 593                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 594                              lpm_tree->id);
 595         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 596 }
 597
 598 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 599                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 600 {
 601         char ralta_pl[MLXSW_REG_RALTA_LEN];
 602
 603         mlxsw_reg_ralta_pack(ralta_pl, false,
 604                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 605                              lpm_tree->id);
 606         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 607 }
 608
 609 static int
 610 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 611                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 612                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 613 {
 614         char ralst_pl[MLXSW_REG_RALST_LEN];
 615         u8 root_bin = 0;
 616         u8 prefix;
 617         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 618
 619         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 620                 root_bin = prefix;
 621
 622         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 623         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 624                 if (prefix == 0)
 625                         continue;
 626                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 627                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 628                 last_prefix = prefix;
 629         }
 630         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 631 }
 632
 633 static struct mlxsw_sp_lpm_tree *
 634 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 635                          struct mlxsw_sp_prefix_usage *prefix_usage,
 636                          enum mlxsw_sp_l3proto proto)
 637 {
 638         struct mlxsw_sp_lpm_tree *lpm_tree;
 639         int err;
 640
 641         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 642         if (!lpm_tree)
 643                 return ERR_PTR(-EBUSY);
 644         lpm_tree->proto = proto;
 645         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 646         if (err)
 647                 return ERR_PTR(err);
 648
 649         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 650                                                 lpm_tree);
 651         if (err)
 652                 goto err_left_struct_set;
 653         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 654                sizeof(lpm_tree->prefix_usage));
 655         memset(&lpm_tree->prefix_ref_count, 0,
 656                sizeof(lpm_tree->prefix_ref_count));
 657         lpm_tree->ref_count = 1;
 658         return lpm_tree;
 659
 660 err_left_struct_set:
 661         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 662         return ERR_PTR(err);
 663 }
 664
 665 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 666                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 667 {
 668         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 669 }
 670
 671 static struct mlxsw_sp_lpm_tree *
 672 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 673                       struct mlxsw_sp_prefix_usage *prefix_usage,
 674                       enum mlxsw_sp_l3proto proto)
 675 {
 676         struct mlxsw_sp_lpm_tree *lpm_tree;
 677         int i;
 678
 679         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 680                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 681                 if (lpm_tree->ref_count != 0 &&
 682                     lpm_tree->proto == proto &&
 683                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 684                                              prefix_usage)) {
 685                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 686                         return lpm_tree;
 687                 }
 688         }
 689         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 690 }
 691
 692 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 693 {
 694         lpm_tree->ref_count++;
 695 }
 696
 697 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 698                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 699 {
 700         if (--lpm_tree->ref_count == 0)
 701                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 702 }
 703
 704 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 705
 706 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 707 {
 708         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 709         struct mlxsw_sp_lpm_tree *lpm_tree;
 710         u64 max_trees;
 711         int err, i;
 712
 713         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 714                 return -EIO;
 715
 716         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 717         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 718         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 719                                              sizeof(struct mlxsw_sp_lpm_tree),
 720                                              GFP_KERNEL);
 721         if (!mlxsw_sp->router->lpm.trees)
 722                 return -ENOMEM;
 723
 724         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 725                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 726                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 727         }
 728
 729         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 730                                          MLXSW_SP_L3_PROTO_IPV4);
 731         if (IS_ERR(lpm_tree)) {
 732                 err = PTR_ERR(lpm_tree);
 733                 goto err_ipv4_tree_get;
 734         }
 735         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 736
 737         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 738                                          MLXSW_SP_L3_PROTO_IPV6);
 739         if (IS_ERR(lpm_tree)) {
 740                 err = PTR_ERR(lpm_tree);
 741                 goto err_ipv6_tree_get;
 742         }
 743         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 744
 745         return 0;
 746
 747 err_ipv6_tree_get:
 748         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 749         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 750 err_ipv4_tree_get:
 751         kfree(mlxsw_sp->router->lpm.trees);
 752         return err;
 753 }
 754
 755 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 756 {
 757         struct mlxsw_sp_lpm_tree *lpm_tree;
 758
 759         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 760         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 761
 762         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 763         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 764
 765         kfree(mlxsw_sp->router->lpm.trees);
 766 }
 767
 768 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 769 {
 770         return !!vr->fib4 || !!vr->fib6 ||
 771                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 772                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 773 }
 774
 775 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 776 {
 777         int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 778         struct mlxsw_sp_vr *vr;
 779         int i;
 780
 781         for (i = 0; i < max_vrs; i++) {
 782                 vr = &mlxsw_sp->router->vrs[i];
 783                 if (!mlxsw_sp_vr_is_used(vr))
 784                         return vr;
 785         }
 786         return NULL;
 787 }
 788
 789 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 790                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 791 {
 792         char raltb_pl[MLXSW_REG_RALTB_LEN];
 793
 794         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 795                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 796                              tree_id);
 797         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 798 }
 799
 800 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 801                                        const struct mlxsw_sp_fib *fib)
 802 {
 803         char raltb_pl[MLXSW_REG_RALTB_LEN];
 804
 805         /* Bind to tree 0 which is default */
 806         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 807                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 808         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 809 }
 810
 811 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 812 {
 813         /* For our purpose, squash main, default and local tables into one */
 814         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 815                 tb_id = RT_TABLE_MAIN;
 816         return tb_id;
 817 }
 818
 819 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 820                                             u32 tb_id)
 821 {
 822         int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 823         struct mlxsw_sp_vr *vr;
 824         int i;
 825
 826         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 827
 828         for (i = 0; i < max_vrs; i++) {
 829                 vr = &mlxsw_sp->router->vrs[i];
 830                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 831                         return vr;
 832         }
 833         return NULL;
 834 }
 835
 836 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 837                                 u16 *vr_id)
 838 {
 839         struct mlxsw_sp_vr *vr;
 840         int err = 0;
 841
 842         mutex_lock(&mlxsw_sp->router->lock);
 843         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 844         if (!vr) {
 845                 err = -ESRCH;
 846                 goto out;
 847         }
 848         *vr_id = vr->id;
 849 out:
 850         mutex_unlock(&mlxsw_sp->router->lock);
 851         return err;
 852 }
 853
 854 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 855                                             enum mlxsw_sp_l3proto proto)
 856 {
 857         switch (proto) {
 858         case MLXSW_SP_L3_PROTO_IPV4:
 859                 return vr->fib4;
 860         case MLXSW_SP_L3_PROTO_IPV6:
 861                 return vr->fib6;
 862         }
 863         return NULL;
 864 }
 865
 866 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 867                                               u32 tb_id,
 868                                               struct netlink_ext_ack *extack)
 869 {
 870         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 871         struct mlxsw_sp_fib *fib4;
 872         struct mlxsw_sp_fib *fib6;
 873         struct mlxsw_sp_vr *vr;
 874         int err;
 875
 876         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 877         if (!vr) {
 878                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 879                 return ERR_PTR(-EBUSY);
 880         }
 881         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 882         if (IS_ERR(fib4))
 883                 return ERR_CAST(fib4);
 884         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 885         if (IS_ERR(fib6)) {
 886                 err = PTR_ERR(fib6);
 887                 goto err_fib6_create;
 888         }
 889         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 890                                              MLXSW_SP_L3_PROTO_IPV4);
 891         if (IS_ERR(mr4_table)) {
 892                 err = PTR_ERR(mr4_table);
 893                 goto err_mr4_table_create;
 894         }
 895         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 896                                              MLXSW_SP_L3_PROTO_IPV6);
 897         if (IS_ERR(mr6_table)) {
 898                 err = PTR_ERR(mr6_table);
 899                 goto err_mr6_table_create;
 900         }
 901
 902         vr->fib4 = fib4;
 903         vr->fib6 = fib6;
 904         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 905         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 906         vr->tb_id = tb_id;
 907         return vr;
 908
 909 err_mr6_table_create:
 910         mlxsw_sp_mr_table_destroy(mr4_table);
 911 err_mr4_table_create:
 912         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 913 err_fib6_create:
 914         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 915         return ERR_PTR(err);
 916 }
 917
 918 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 919                                 struct mlxsw_sp_vr *vr)
 920 {
 921         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 922         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 923         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 924         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 925         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 926         vr->fib6 = NULL;
 927         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 928         vr->fib4 = NULL;
 929 }
 930
 931 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 932                                            struct netlink_ext_ack *extack)
 933 {
 934         struct mlxsw_sp_vr *vr;
 935
 936         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 937         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 938         if (!vr)
 939                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 940         return vr;
 941 }
 942
 943 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 944 {
 945         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 946             list_empty(&vr->fib6->node_list) &&
 947             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 948             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 949                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 950 }
 951
 952 static bool
 953 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 954                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 955 {
 956         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 957
 958         if (!mlxsw_sp_vr_is_used(vr))
 959                 return false;
 960         if (fib->lpm_tree->id == tree_id)
 961                 return true;
 962         return false;
 963 }
 964
 965 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 966                                         struct mlxsw_sp_fib *fib,
 967                                         struct mlxsw_sp_lpm_tree *new_tree)
 968 {
 969         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 970         int err;
 971
 972         fib->lpm_tree = new_tree;
 973         mlxsw_sp_lpm_tree_hold(new_tree);
 974         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 975         if (err)
 976                 goto err_tree_bind;
 977         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 978         return 0;
 979
 980 err_tree_bind:
 981         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 982         fib->lpm_tree = old_tree;
 983         return err;
 984 }
 985
 986 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 987                                          struct mlxsw_sp_fib *fib,
 988                                          struct mlxsw_sp_lpm_tree *new_tree)
 989 {
 990         int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 991         enum mlxsw_sp_l3proto proto = fib->proto;
 992         struct mlxsw_sp_lpm_tree *old_tree;
 993         u8 old_id, new_id = new_tree->id;
 994         struct mlxsw_sp_vr *vr;
 995         int i, err;
 996
 997         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 998         old_id = old_tree->id;
 999
1000         for (i = 0; i < max_vrs; i++) {
1001                 vr = &mlxsw_sp->router->vrs[i];
1002                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
1003                         continue;
1004                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1005                                                    mlxsw_sp_vr_fib(vr, proto),
1006                                                    new_tree);
1007                 if (err)
1008                         goto err_tree_replace;
1009         }
1010
1011         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
1012                sizeof(new_tree->prefix_ref_count));
1013         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
1014         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
1015
1016         return 0;
1017
1018 err_tree_replace:
1019         for (i--; i >= 0; i--) {
1020                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
1021                         continue;
1022                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1023                                              mlxsw_sp_vr_fib(vr, proto),
1024                                              old_tree);
1025         }
1026         return err;
1027 }
1028
1029 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1030 {
1031         struct mlxsw_sp_vr *vr;
1032         u64 max_vrs;
1033         int i;
1034
1035         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1036                 return -EIO;
1037
1038         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1039         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1040                                         GFP_KERNEL);
1041         if (!mlxsw_sp->router->vrs)
1042                 return -ENOMEM;
1043
1044         for (i = 0; i < max_vrs; i++) {
1045                 vr = &mlxsw_sp->router->vrs[i];
1046                 vr->id = i;
1047         }
1048
1049         return 0;
1050 }
1051
1052 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1053
1054 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1055 {
1056         /* At this stage we're guaranteed not to have new incoming
1057          * FIB notifications and the work queue is free from FIBs
1058          * sitting on top of mlxsw netdevs. However, we can still
1059          * have other FIBs queued. Flush the queue before flushing
1060          * the device's tables. No need for locks, as we're the only
1061          * writer.
1062          */
1063         mlxsw_core_flush_owq();
1064         mlxsw_sp_router_fib_flush(mlxsw_sp);
1065         kfree(mlxsw_sp->router->vrs);
1066 }
1067
1068 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1069 {
1070         struct net_device *d;
1071         u32 tb_id;
1072
1073         rcu_read_lock();
1074         d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1075         if (d)
1076                 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1077         else
1078                 tb_id = RT_TABLE_MAIN;
1079         rcu_read_unlock();
1080
1081         return tb_id;
1082 }
1083
1084 static void
1085 mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev)
1086 {
1087         crif->key.dev = dev;
1088         INIT_LIST_HEAD(&crif->nexthop_list);
1089 }
1090
1091 static struct mlxsw_sp_crif *
1092 mlxsw_sp_crif_alloc(struct net_device *dev)
1093 {
1094         struct mlxsw_sp_crif *crif;
1095
1096         crif = kzalloc(sizeof(*crif), GFP_KERNEL);
1097         if (!crif)
1098                 return NULL;
1099
1100         mlxsw_sp_crif_init(crif, dev);
1101         return crif;
1102 }
1103
1104 static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif)
1105 {
1106         if (WARN_ON(crif->rif))
1107                 return;
1108
1109         WARN_ON(!list_empty(&crif->nexthop_list));
1110         kfree(crif);
1111 }
1112
1113 static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router,
1114                                 struct mlxsw_sp_crif *crif)
1115 {
1116         return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node,
1117                                       mlxsw_sp_crif_ht_params);
1118 }
1119
1120 static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router,
1121                                  struct mlxsw_sp_crif *crif)
1122 {
1123         rhashtable_remove_fast(&router->crif_ht, &crif->ht_node,
1124                                mlxsw_sp_crif_ht_params);
1125 }
1126
1127 static struct mlxsw_sp_crif *
1128 mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router,
1129                      const struct net_device *dev)
1130 {
1131         struct mlxsw_sp_crif_key key = {
1132                 .dev = (struct net_device *)dev,
1133         };
1134
1135         return rhashtable_lookup_fast(&router->crif_ht, &key,
1136                                       mlxsw_sp_crif_ht_params);
1137 }
1138
1139 static struct mlxsw_sp_rif *
1140 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1141                     const struct mlxsw_sp_rif_params *params,
1142                     struct netlink_ext_ack *extack);
1143
1144 static struct mlxsw_sp_rif_ipip_lb *
1145 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1146                                 enum mlxsw_sp_ipip_type ipipt,
1147                                 struct net_device *ol_dev,
1148                                 struct netlink_ext_ack *extack)
1149 {
1150         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1151         const struct mlxsw_sp_ipip_ops *ipip_ops;
1152         struct mlxsw_sp_rif *rif;
1153
1154         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1155         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1156                 .common.dev = ol_dev,
1157                 .common.lag = false,
1158                 .common.double_entry = ipip_ops->double_rif_entry,
1159                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1160         };
1161
1162         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1163         if (IS_ERR(rif))
1164                 return ERR_CAST(rif);
1165         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1166 }
1167
1168 static struct mlxsw_sp_ipip_entry *
1169 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1170                           enum mlxsw_sp_ipip_type ipipt,
1171                           struct net_device *ol_dev)
1172 {
1173         const struct mlxsw_sp_ipip_ops *ipip_ops;
1174         struct mlxsw_sp_ipip_entry *ipip_entry;
1175         struct mlxsw_sp_ipip_entry *ret = NULL;
1176         int err;
1177
1178         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1179         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1180         if (!ipip_entry)
1181                 return ERR_PTR(-ENOMEM);
1182
1183         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1184                                                             ol_dev, NULL);
1185         if (IS_ERR(ipip_entry->ol_lb)) {
1186                 ret = ERR_CAST(ipip_entry->ol_lb);
1187                 goto err_ol_ipip_lb_create;
1188         }
1189
1190         ipip_entry->ipipt = ipipt;
1191         ipip_entry->ol_dev = ol_dev;
1192         ipip_entry->parms = ipip_ops->parms_init(ol_dev);
1193
1194         err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry);
1195         if (err) {
1196                 ret = ERR_PTR(err);
1197                 goto err_rem_ip_addr_set;
1198         }
1199
1200         return ipip_entry;
1201
1202 err_rem_ip_addr_set:
1203         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1204 err_ol_ipip_lb_create:
1205         kfree(ipip_entry);
1206         return ret;
1207 }
1208
1209 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp,
1210                                         struct mlxsw_sp_ipip_entry *ipip_entry)
1211 {
1212         const struct mlxsw_sp_ipip_ops *ipip_ops =
1213                 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1214
1215         ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry);
1216         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1217         kfree(ipip_entry);
1218 }
1219
1220 static bool
1221 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1222                                   const enum mlxsw_sp_l3proto ul_proto,
1223                                   union mlxsw_sp_l3addr saddr,
1224                                   u32 ul_tb_id,
1225                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1226 {
1227         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1228         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1229         union mlxsw_sp_l3addr tun_saddr;
1230
1231         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1232                 return false;
1233
1234         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1235         return tun_ul_tb_id == ul_tb_id &&
1236                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1237 }
1238
1239 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp,
1240                                                  enum mlxsw_sp_ipip_type ipipt)
1241 {
1242         const struct mlxsw_sp_ipip_ops *ipip_ops;
1243
1244         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1245
1246         /* Not all tunnels require to increase the default pasing depth
1247          * (96 bytes).
1248          */
1249         if (ipip_ops->inc_parsing_depth)
1250                 return mlxsw_sp_parsing_depth_inc(mlxsw_sp);
1251
1252         return 0;
1253 }
1254
1255 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp,
1256                                                   enum mlxsw_sp_ipip_type ipipt)
1257 {
1258         const struct mlxsw_sp_ipip_ops *ipip_ops =
1259                 mlxsw_sp->router->ipip_ops_arr[ipipt];
1260
1261         if (ipip_ops->inc_parsing_depth)
1262                 mlxsw_sp_parsing_depth_dec(mlxsw_sp);
1263 }
1264
1265 static int
1266 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1267                               struct mlxsw_sp_fib_entry *fib_entry,
1268                               struct mlxsw_sp_ipip_entry *ipip_entry)
1269 {
1270         u32 tunnel_index;
1271         int err;
1272
1273         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1274                                   1, &tunnel_index);
1275         if (err)
1276                 return err;
1277
1278         err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp,
1279                                                     ipip_entry->ipipt);
1280         if (err)
1281                 goto err_parsing_depth_inc;
1282
1283         ipip_entry->decap_fib_entry = fib_entry;
1284         fib_entry->decap.ipip_entry = ipip_entry;
1285         fib_entry->decap.tunnel_index = tunnel_index;
1286
1287         return 0;
1288
1289 err_parsing_depth_inc:
1290         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
1291                            fib_entry->decap.tunnel_index);
1292         return err;
1293 }
1294
1295 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1296                                           struct mlxsw_sp_fib_entry *fib_entry)
1297 {
1298         enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt;
1299
1300         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1301         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1302         fib_entry->decap.ipip_entry = NULL;
1303         mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt);
1304         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1305                            1, fib_entry->decap.tunnel_index);
1306 }
1307
1308 static struct mlxsw_sp_fib_node *
1309 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1310                          size_t addr_len, unsigned char prefix_len);
1311 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1312                                      struct mlxsw_sp_fib_entry *fib_entry);
1313
1314 static void
1315 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1316                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1317 {
1318         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1319
1320         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1321         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1322
1323         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1324 }
1325
1326 static void
1327 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1328                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1329                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1330 {
1331         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1332                                           ipip_entry))
1333                 return;
1334         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1335
1336         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1337                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1338 }
1339
1340 static struct mlxsw_sp_fib_entry *
1341 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1342                                      enum mlxsw_sp_l3proto proto,
1343                                      const union mlxsw_sp_l3addr *addr,
1344                                      enum mlxsw_sp_fib_entry_type type)
1345 {
1346         struct mlxsw_sp_fib_node *fib_node;
1347         unsigned char addr_prefix_len;
1348         struct mlxsw_sp_fib *fib;
1349         struct mlxsw_sp_vr *vr;
1350         const void *addrp;
1351         size_t addr_len;
1352         u32 addr4;
1353
1354         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1355         if (!vr)
1356                 return NULL;
1357         fib = mlxsw_sp_vr_fib(vr, proto);
1358
1359         switch (proto) {
1360         case MLXSW_SP_L3_PROTO_IPV4:
1361                 addr4 = be32_to_cpu(addr->addr4);
1362                 addrp = &addr4;
1363                 addr_len = 4;
1364                 addr_prefix_len = 32;
1365                 break;
1366         case MLXSW_SP_L3_PROTO_IPV6:
1367                 addrp = &addr->addr6;
1368                 addr_len = 16;
1369                 addr_prefix_len = 128;
1370                 break;
1371         default:
1372                 WARN_ON(1);
1373                 return NULL;
1374         }
1375
1376         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1377                                             addr_prefix_len);
1378         if (!fib_node || fib_node->fib_entry->type != type)
1379                 return NULL;
1380
1381         return fib_node->fib_entry;
1382 }
1383
1384 /* Given an IPIP entry, find the corresponding decap route. */
1385 static struct mlxsw_sp_fib_entry *
1386 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1387                                struct mlxsw_sp_ipip_entry *ipip_entry)
1388 {
1389         static struct mlxsw_sp_fib_node *fib_node;
1390         const struct mlxsw_sp_ipip_ops *ipip_ops;
1391         unsigned char saddr_prefix_len;
1392         union mlxsw_sp_l3addr saddr;
1393         struct mlxsw_sp_fib *ul_fib;
1394         struct mlxsw_sp_vr *ul_vr;
1395         const void *saddrp;
1396         size_t saddr_len;
1397         u32 ul_tb_id;
1398         u32 saddr4;
1399
1400         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1401
1402         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1403         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1404         if (!ul_vr)
1405                 return NULL;
1406
1407         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1408         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1409                                            ipip_entry->ol_dev);
1410
1411         switch (ipip_ops->ul_proto) {
1412         case MLXSW_SP_L3_PROTO_IPV4:
1413                 saddr4 = be32_to_cpu(saddr.addr4);
1414                 saddrp = &saddr4;
1415                 saddr_len = 4;
1416                 saddr_prefix_len = 32;
1417                 break;
1418         case MLXSW_SP_L3_PROTO_IPV6:
1419                 saddrp = &saddr.addr6;
1420                 saddr_len = 16;
1421                 saddr_prefix_len = 128;
1422                 break;
1423         default:
1424                 WARN_ON(1);
1425                 return NULL;
1426         }
1427
1428         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1429                                             saddr_prefix_len);
1430         if (!fib_node ||
1431             fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1432                 return NULL;
1433
1434         return fib_node->fib_entry;
1435 }
1436
1437 static struct mlxsw_sp_ipip_entry *
1438 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1439                            enum mlxsw_sp_ipip_type ipipt,
1440                            struct net_device *ol_dev)
1441 {
1442         struct mlxsw_sp_ipip_entry *ipip_entry;
1443
1444         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1445         if (IS_ERR(ipip_entry))
1446                 return ipip_entry;
1447
1448         list_add_tail(&ipip_entry->ipip_list_node,
1449                       &mlxsw_sp->router->ipip_list);
1450
1451         return ipip_entry;
1452 }
1453
1454 static void
1455 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1456                             struct mlxsw_sp_ipip_entry *ipip_entry)
1457 {
1458         list_del(&ipip_entry->ipip_list_node);
1459         mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry);
1460 }
1461
1462 static bool
1463 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1464                                   const struct net_device *ul_dev,
1465                                   enum mlxsw_sp_l3proto ul_proto,
1466                                   union mlxsw_sp_l3addr ul_dip,
1467                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1468 {
1469         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1470         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1471
1472         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1473                 return false;
1474
1475         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1476                                                  ul_tb_id, ipip_entry);
1477 }
1478
1479 /* Given decap parameters, find the corresponding IPIP entry. */
1480 static struct mlxsw_sp_ipip_entry *
1481 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1482                                   enum mlxsw_sp_l3proto ul_proto,
1483                                   union mlxsw_sp_l3addr ul_dip)
1484 {
1485         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1486         struct net_device *ul_dev;
1487
1488         rcu_read_lock();
1489
1490         ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1491         if (!ul_dev)
1492                 goto out_unlock;
1493
1494         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1495                             ipip_list_node)
1496                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1497                                                       ul_proto, ul_dip,
1498                                                       ipip_entry))
1499                         goto out_unlock;
1500
1501         rcu_read_unlock();
1502
1503         return NULL;
1504
1505 out_unlock:
1506         rcu_read_unlock();
1507         return ipip_entry;
1508 }
1509
1510 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1511                                       const struct net_device *dev,
1512                                       enum mlxsw_sp_ipip_type *p_type)
1513 {
1514         struct mlxsw_sp_router *router = mlxsw_sp->router;
1515         const struct mlxsw_sp_ipip_ops *ipip_ops;
1516         enum mlxsw_sp_ipip_type ipipt;
1517
1518         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1519                 ipip_ops = router->ipip_ops_arr[ipipt];
1520                 if (dev->type == ipip_ops->dev_type) {
1521                         if (p_type)
1522                                 *p_type = ipipt;
1523                         return true;
1524                 }
1525         }
1526         return false;
1527 }
1528
1529 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1530                                        const struct net_device *dev)
1531 {
1532         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1533 }
1534
1535 static struct mlxsw_sp_ipip_entry *
1536 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1537                                    const struct net_device *ol_dev)
1538 {
1539         struct mlxsw_sp_ipip_entry *ipip_entry;
1540
1541         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1542                             ipip_list_node)
1543                 if (ipip_entry->ol_dev == ol_dev)
1544                         return ipip_entry;
1545
1546         return NULL;
1547 }
1548
1549 static struct mlxsw_sp_ipip_entry *
1550 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1551                                    const struct net_device *ul_dev,
1552                                    struct mlxsw_sp_ipip_entry *start)
1553 {
1554         struct mlxsw_sp_ipip_entry *ipip_entry;
1555
1556         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1557                                         ipip_list_node);
1558         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1559                                      ipip_list_node) {
1560                 struct net_device *ol_dev = ipip_entry->ol_dev;
1561                 struct net_device *ipip_ul_dev;
1562
1563                 rcu_read_lock();
1564                 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1565                 rcu_read_unlock();
1566
1567                 if (ipip_ul_dev == ul_dev)
1568                         return ipip_entry;
1569         }
1570
1571         return NULL;
1572 }
1573
1574 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1575                                        const struct net_device *dev)
1576 {
1577         return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1578 }
1579
1580 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1581                                                 const struct net_device *ol_dev,
1582                                                 enum mlxsw_sp_ipip_type ipipt)
1583 {
1584         const struct mlxsw_sp_ipip_ops *ops
1585                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1586
1587         return ops->can_offload(mlxsw_sp, ol_dev);
1588 }
1589
1590 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1591                                                 struct net_device *ol_dev)
1592 {
1593         enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1594         struct mlxsw_sp_ipip_entry *ipip_entry;
1595         enum mlxsw_sp_l3proto ul_proto;
1596         union mlxsw_sp_l3addr saddr;
1597         u32 ul_tb_id;
1598
1599         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1600         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1601                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1602                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1603                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1604                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1605                                                           saddr, ul_tb_id,
1606                                                           NULL)) {
1607                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1608                                                                 ol_dev);
1609                         if (IS_ERR(ipip_entry))
1610                                 return PTR_ERR(ipip_entry);
1611                 }
1612         }
1613
1614         return 0;
1615 }
1616
1617 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1618                                                    struct net_device *ol_dev)
1619 {
1620         struct mlxsw_sp_ipip_entry *ipip_entry;
1621
1622         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1623         if (ipip_entry)
1624                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1625 }
1626
1627 static void
1628 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1629                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1630 {
1631         struct mlxsw_sp_fib_entry *decap_fib_entry;
1632
1633         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1634         if (decap_fib_entry)
1635                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1636                                                   decap_fib_entry);
1637 }
1638
1639 static int
1640 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1641                         u16 ul_rif_id, bool enable)
1642 {
1643         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1644         struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common);
1645         enum mlxsw_reg_ritr_loopback_ipip_options ipip_options;
1646         struct mlxsw_sp_rif *rif = &lb_rif->common;
1647         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1648         char ritr_pl[MLXSW_REG_RITR_LEN];
1649         struct in6_addr *saddr6;
1650         u32 saddr4;
1651
1652         ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET;
1653         switch (lb_cf.ul_protocol) {
1654         case MLXSW_SP_L3_PROTO_IPV4:
1655                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1656                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1657                                     rif->rif_index, rif->vr_id, dev->mtu);
1658                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1659                                                    ipip_options, ul_vr_id,
1660                                                    ul_rif_id, saddr4,
1661                                                    lb_cf.okey);
1662                 break;
1663
1664         case MLXSW_SP_L3_PROTO_IPV6:
1665                 saddr6 = &lb_cf.saddr.addr6;
1666                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1667                                     rif->rif_index, rif->vr_id, dev->mtu);
1668                 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt,
1669                                                    ipip_options, ul_vr_id,
1670                                                    ul_rif_id, saddr6,
1671                                                    lb_cf.okey);
1672                 break;
1673         }
1674
1675         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1676 }
1677
1678 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1679                                                  struct net_device *ol_dev)
1680 {
1681         struct mlxsw_sp_ipip_entry *ipip_entry;
1682         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1683         int err = 0;
1684
1685         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1686         if (ipip_entry) {
1687                 lb_rif = ipip_entry->ol_lb;
1688                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1689                                               lb_rif->ul_rif_id, true);
1690                 if (err)
1691                         goto out;
1692                 lb_rif->common.mtu = ol_dev->mtu;
1693         }
1694
1695 out:
1696         return err;
1697 }
1698
1699 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1700                                                 struct net_device *ol_dev)
1701 {
1702         struct mlxsw_sp_ipip_entry *ipip_entry;
1703
1704         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1705         if (ipip_entry)
1706                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1707 }
1708
1709 static void
1710 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1711                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1712 {
1713         if (ipip_entry->decap_fib_entry)
1714                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1715 }
1716
1717 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1718                                                   struct net_device *ol_dev)
1719 {
1720         struct mlxsw_sp_ipip_entry *ipip_entry;
1721
1722         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1723         if (ipip_entry)
1724                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1725 }
1726
1727 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1728                                         struct mlxsw_sp_rif *rif);
1729
1730 static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp,
1731                                          struct mlxsw_sp_rif *old_rif,
1732                                          struct mlxsw_sp_rif *new_rif,
1733                                          bool migrate_nhs)
1734 {
1735         struct mlxsw_sp_crif *crif = old_rif->crif;
1736         struct mlxsw_sp_crif mock_crif = {};
1737
1738         if (migrate_nhs)
1739                 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
1740
1741         /* Plant a mock CRIF so that destroying the old RIF doesn't unoffload
1742          * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link.
1743          */
1744         mlxsw_sp_crif_init(&mock_crif, crif->key.dev);
1745         old_rif->crif = &mock_crif;
1746         mock_crif.rif = old_rif;
1747         mlxsw_sp_rif_destroy(old_rif);
1748 }
1749
1750 static int
1751 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1752                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1753                                  bool keep_encap,
1754                                  struct netlink_ext_ack *extack)
1755 {
1756         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1757         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1758
1759         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1760                                                      ipip_entry->ipipt,
1761                                                      ipip_entry->ol_dev,
1762                                                      extack);
1763         if (IS_ERR(new_lb_rif))
1764                 return PTR_ERR(new_lb_rif);
1765         ipip_entry->ol_lb = new_lb_rif;
1766
1767         mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common,
1768                                      &new_lb_rif->common, keep_encap);
1769         return 0;
1770 }
1771
1772 /**
1773  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1774  * @mlxsw_sp: mlxsw_sp.
1775  * @ipip_entry: IPIP entry.
1776  * @recreate_loopback: Recreates the associated loopback RIF.
1777  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1778  *              relevant when recreate_loopback is true.
1779  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1780  *                   is only relevant when recreate_loopback is false.
1781  * @extack: extack.
1782  *
1783  * Return: Non-zero value on failure.
1784  */
1785 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1786                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1787                                         bool recreate_loopback,
1788                                         bool keep_encap,
1789                                         bool update_nexthops,
1790                                         struct netlink_ext_ack *extack)
1791 {
1792         int err;
1793
1794         /* RIFs can't be edited, so to update loopback, we need to destroy and
1795          * recreate it. That creates a window of opportunity where RALUE and
1796          * RATR registers end up referencing a RIF that's already gone. RATRs
1797          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1798          * of RALUE, demote the decap route back.
1799          */
1800         if (ipip_entry->decap_fib_entry)
1801                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1802
1803         if (recreate_loopback) {
1804                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1805                                                        keep_encap, extack);
1806                 if (err)
1807                         return err;
1808         } else if (update_nexthops) {
1809                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1810                                             &ipip_entry->ol_lb->common);
1811         }
1812
1813         if (ipip_entry->ol_dev->flags & IFF_UP)
1814                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1815
1816         return 0;
1817 }
1818
1819 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1820                                                 struct net_device *ol_dev,
1821                                                 struct netlink_ext_ack *extack)
1822 {
1823         struct mlxsw_sp_ipip_entry *ipip_entry =
1824                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1825
1826         if (!ipip_entry)
1827                 return 0;
1828
1829         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1830                                                    true, false, false, extack);
1831 }
1832
1833 static int
1834 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1835                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1836                                      struct net_device *ul_dev,
1837                                      bool *demote_this,
1838                                      struct netlink_ext_ack *extack)
1839 {
1840         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1841         enum mlxsw_sp_l3proto ul_proto;
1842         union mlxsw_sp_l3addr saddr;
1843
1844         /* Moving underlay to a different VRF might cause local address
1845          * conflict, and the conflicting tunnels need to be demoted.
1846          */
1847         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1848         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1849         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1850                                                  saddr, ul_tb_id,
1851                                                  ipip_entry)) {
1852                 *demote_this = true;
1853                 return 0;
1854         }
1855
1856         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1857                                                    true, true, false, extack);
1858 }
1859
1860 static int
1861 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1862                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1863                                     struct net_device *ul_dev)
1864 {
1865         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1866                                                    false, false, true, NULL);
1867 }
1868
1869 static int
1870 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1871                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1872                                       struct net_device *ul_dev)
1873 {
1874         /* A down underlay device causes encapsulated packets to not be
1875          * forwarded, but decap still works. So refresh next hops without
1876          * touching anything else.
1877          */
1878         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1879                                                    false, false, true, NULL);
1880 }
1881
1882 static int
1883 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1884                                         struct net_device *ol_dev,
1885                                         struct netlink_ext_ack *extack)
1886 {
1887         const struct mlxsw_sp_ipip_ops *ipip_ops;
1888         struct mlxsw_sp_ipip_entry *ipip_entry;
1889         int err;
1890
1891         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1892         if (!ipip_entry)
1893                 /* A change might make a tunnel eligible for offloading, but
1894                  * that is currently not implemented. What falls to slow path
1895                  * stays there.
1896                  */
1897                 return 0;
1898
1899         /* A change might make a tunnel not eligible for offloading. */
1900         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1901                                                  ipip_entry->ipipt)) {
1902                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1903                 return 0;
1904         }
1905
1906         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1907         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1908         return err;
1909 }
1910
1911 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1912                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1913 {
1914         struct net_device *ol_dev = ipip_entry->ol_dev;
1915
1916         if (ol_dev->flags & IFF_UP)
1917                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1918         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1919 }
1920
1921 /* The configuration where several tunnels have the same local address in the
1922  * same underlay table needs special treatment in the HW. That is currently not
1923  * implemented in the driver. This function finds and demotes the first tunnel
1924  * with a given source address, except the one passed in the argument
1925  * `except'.
1926  */
1927 bool
1928 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1929                                      enum mlxsw_sp_l3proto ul_proto,
1930                                      union mlxsw_sp_l3addr saddr,
1931                                      u32 ul_tb_id,
1932                                      const struct mlxsw_sp_ipip_entry *except)
1933 {
1934         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1935
1936         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1937                                  ipip_list_node) {
1938                 if (ipip_entry != except &&
1939                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1940                                                       ul_tb_id, ipip_entry)) {
1941                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1942                         return true;
1943                 }
1944         }
1945
1946         return false;
1947 }
1948
1949 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1950                                                      struct net_device *ul_dev)
1951 {
1952         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1953
1954         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1955                                  ipip_list_node) {
1956                 struct net_device *ol_dev = ipip_entry->ol_dev;
1957                 struct net_device *ipip_ul_dev;
1958
1959                 rcu_read_lock();
1960                 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1961                 rcu_read_unlock();
1962                 if (ipip_ul_dev == ul_dev)
1963                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1964         }
1965 }
1966
1967 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1968                                             struct net_device *ol_dev,
1969                                             unsigned long event,
1970                                             struct netdev_notifier_info *info)
1971 {
1972         struct netdev_notifier_changeupper_info *chup;
1973         struct netlink_ext_ack *extack;
1974         int err = 0;
1975
1976         switch (event) {
1977         case NETDEV_REGISTER:
1978                 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1979                 break;
1980         case NETDEV_UNREGISTER:
1981                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1982                 break;
1983         case NETDEV_UP:
1984                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1985                 break;
1986         case NETDEV_DOWN:
1987                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1988                 break;
1989         case NETDEV_CHANGEUPPER:
1990                 chup = container_of(info, typeof(*chup), info);
1991                 extack = info->extack;
1992                 if (netif_is_l3_master(chup->upper_dev))
1993                         err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1994                                                                    ol_dev,
1995                                                                    extack);
1996                 break;
1997         case NETDEV_CHANGE:
1998                 extack = info->extack;
1999                 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
2000                                                               ol_dev, extack);
2001                 break;
2002         case NETDEV_CHANGEMTU:
2003                 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
2004                 break;
2005         }
2006         return err;
2007 }
2008
2009 static int
2010 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2011                                    struct mlxsw_sp_ipip_entry *ipip_entry,
2012                                    struct net_device *ul_dev,
2013                                    bool *demote_this,
2014                                    unsigned long event,
2015                                    struct netdev_notifier_info *info)
2016 {
2017         struct netdev_notifier_changeupper_info *chup;
2018         struct netlink_ext_ack *extack;
2019
2020         switch (event) {
2021         case NETDEV_CHANGEUPPER:
2022                 chup = container_of(info, typeof(*chup), info);
2023                 extack = info->extack;
2024                 if (netif_is_l3_master(chup->upper_dev))
2025                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
2026                                                                     ipip_entry,
2027                                                                     ul_dev,
2028                                                                     demote_this,
2029                                                                     extack);
2030                 break;
2031
2032         case NETDEV_UP:
2033                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
2034                                                            ul_dev);
2035         case NETDEV_DOWN:
2036                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
2037                                                              ipip_entry,
2038                                                              ul_dev);
2039         }
2040         return 0;
2041 }
2042
2043 static int
2044 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2045                                  struct net_device *ul_dev,
2046                                  unsigned long event,
2047                                  struct netdev_notifier_info *info)
2048 {
2049         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
2050         int err;
2051
2052         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
2053                                                                 ul_dev,
2054                                                                 ipip_entry))) {
2055                 struct mlxsw_sp_ipip_entry *prev;
2056                 bool demote_this = false;
2057
2058                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
2059                                                          ul_dev, &demote_this,
2060                                                          event, info);
2061                 if (err) {
2062                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
2063                                                                  ul_dev);
2064                         return err;
2065                 }
2066
2067                 if (demote_this) {
2068                         if (list_is_first(&ipip_entry->ipip_list_node,
2069                                           &mlxsw_sp->router->ipip_list))
2070                                 prev = NULL;
2071                         else
2072                                 /* This can't be cached from previous iteration,
2073                                  * because that entry could be gone now.
2074                                  */
2075                                 prev = list_prev_entry(ipip_entry,
2076                                                        ipip_list_node);
2077                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
2078                         ipip_entry = prev;
2079                 }
2080         }
2081
2082         return 0;
2083 }
2084
2085 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2086                                       enum mlxsw_sp_l3proto ul_proto,
2087                                       const union mlxsw_sp_l3addr *ul_sip,
2088                                       u32 tunnel_index)
2089 {
2090         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2091         struct mlxsw_sp_router *router = mlxsw_sp->router;
2092         struct mlxsw_sp_fib_entry *fib_entry;
2093         int err = 0;
2094
2095         mutex_lock(&mlxsw_sp->router->lock);
2096
2097         if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
2098                 err = -EINVAL;
2099                 goto out;
2100         }
2101
2102         router->nve_decap_config.ul_tb_id = ul_tb_id;
2103         router->nve_decap_config.tunnel_index = tunnel_index;
2104         router->nve_decap_config.ul_proto = ul_proto;
2105         router->nve_decap_config.ul_sip = *ul_sip;
2106         router->nve_decap_config.valid = true;
2107
2108         /* It is valid to create a tunnel with a local IP and only later
2109          * assign this IP address to a local interface
2110          */
2111         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2112                                                          ul_proto, ul_sip,
2113                                                          type);
2114         if (!fib_entry)
2115                 goto out;
2116
2117         fib_entry->decap.tunnel_index = tunnel_index;
2118         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2119
2120         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2121         if (err)
2122                 goto err_fib_entry_update;
2123
2124         goto out;
2125
2126 err_fib_entry_update:
2127         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2128         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2129 out:
2130         mutex_unlock(&mlxsw_sp->router->lock);
2131         return err;
2132 }
2133
2134 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2135                                       enum mlxsw_sp_l3proto ul_proto,
2136                                       const union mlxsw_sp_l3addr *ul_sip)
2137 {
2138         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2139         struct mlxsw_sp_router *router = mlxsw_sp->router;
2140         struct mlxsw_sp_fib_entry *fib_entry;
2141
2142         mutex_lock(&mlxsw_sp->router->lock);
2143
2144         if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2145                 goto out;
2146
2147         router->nve_decap_config.valid = false;
2148
2149         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2150                                                          ul_proto, ul_sip,
2151                                                          type);
2152         if (!fib_entry)
2153                 goto out;
2154
2155         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2156         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2157 out:
2158         mutex_unlock(&mlxsw_sp->router->lock);
2159 }
2160
2161 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2162                                          u32 ul_tb_id,
2163                                          enum mlxsw_sp_l3proto ul_proto,
2164                                          const union mlxsw_sp_l3addr *ul_sip)
2165 {
2166         struct mlxsw_sp_router *router = mlxsw_sp->router;
2167
2168         return router->nve_decap_config.valid &&
2169                router->nve_decap_config.ul_tb_id == ul_tb_id &&
2170                router->nve_decap_config.ul_proto == ul_proto &&
2171                !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2172                        sizeof(*ul_sip));
2173 }
2174
2175 struct mlxsw_sp_neigh_key {
2176         struct neighbour *n;
2177 };
2178
2179 struct mlxsw_sp_neigh_entry {
2180         struct list_head rif_list_node;
2181         struct rhash_head ht_node;
2182         struct mlxsw_sp_neigh_key key;
2183         u16 rif;
2184         bool connected;
2185         unsigned char ha[ETH_ALEN];
2186         struct list_head nexthop_list; /* list of nexthops using
2187                                         * this neigh entry
2188                                         */
2189         struct list_head nexthop_neighs_list_node;
2190         unsigned int counter_index;
2191         bool counter_valid;
2192 };
2193
2194 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2195         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2196         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2197         .key_len = sizeof(struct mlxsw_sp_neigh_key),
2198 };
2199
2200 struct mlxsw_sp_neigh_entry *
2201 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2202                         struct mlxsw_sp_neigh_entry *neigh_entry)
2203 {
2204         if (!neigh_entry) {
2205                 if (list_empty(&rif->neigh_list))
2206                         return NULL;
2207                 else
2208                         return list_first_entry(&rif->neigh_list,
2209                                                 typeof(*neigh_entry),
2210                                                 rif_list_node);
2211         }
2212         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2213                 return NULL;
2214         return list_next_entry(neigh_entry, rif_list_node);
2215 }
2216
2217 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2218 {
2219         return neigh_entry->key.n->tbl->family;
2220 }
2221
2222 unsigned char *
2223 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2224 {
2225         return neigh_entry->ha;
2226 }
2227
2228 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2229 {
2230         struct neighbour *n;
2231
2232         n = neigh_entry->key.n;
2233         return ntohl(*((__be32 *) n->primary_key));
2234 }
2235
2236 struct in6_addr *
2237 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2238 {
2239         struct neighbour *n;
2240
2241         n = neigh_entry->key.n;
2242         return (struct in6_addr *) &n->primary_key;
2243 }
2244
2245 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2246                                struct mlxsw_sp_neigh_entry *neigh_entry,
2247                                u64 *p_counter)
2248 {
2249         if (!neigh_entry->counter_valid)
2250                 return -EINVAL;
2251
2252         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2253                                          p_counter, NULL);
2254 }
2255
2256 static struct mlxsw_sp_neigh_entry *
2257 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2258                            u16 rif)
2259 {
2260         struct mlxsw_sp_neigh_entry *neigh_entry;
2261
2262         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2263         if (!neigh_entry)
2264                 return NULL;
2265
2266         neigh_entry->key.n = n;
2267         neigh_entry->rif = rif;
2268         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2269
2270         return neigh_entry;
2271 }
2272
2273 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2274 {
2275         kfree(neigh_entry);
2276 }
2277
2278 static int
2279 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2280                             struct mlxsw_sp_neigh_entry *neigh_entry)
2281 {
2282         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2283                                       &neigh_entry->ht_node,
2284                                       mlxsw_sp_neigh_ht_params);
2285 }
2286
2287 static void
2288 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2289                             struct mlxsw_sp_neigh_entry *neigh_entry)
2290 {
2291         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2292                                &neigh_entry->ht_node,
2293                                mlxsw_sp_neigh_ht_params);
2294 }
2295
2296 static bool
2297 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2298                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2299 {
2300         struct devlink *devlink;
2301         const char *table_name;
2302
2303         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2304         case AF_INET:
2305                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2306                 break;
2307         case AF_INET6:
2308                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2309                 break;
2310         default:
2311                 WARN_ON(1);
2312                 return false;
2313         }
2314
2315         devlink = priv_to_devlink(mlxsw_sp->core);
2316         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2317 }
2318
2319 static void
2320 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2321                              struct mlxsw_sp_neigh_entry *neigh_entry)
2322 {
2323         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2324                 return;
2325
2326         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2327                 return;
2328
2329         neigh_entry->counter_valid = true;
2330 }
2331
2332 static void
2333 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2334                             struct mlxsw_sp_neigh_entry *neigh_entry)
2335 {
2336         if (!neigh_entry->counter_valid)
2337                 return;
2338         mlxsw_sp_flow_counter_free(mlxsw_sp,
2339                                    neigh_entry->counter_index);
2340         neigh_entry->counter_valid = false;
2341 }
2342
2343 static struct mlxsw_sp_neigh_entry *
2344 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2345 {
2346         struct mlxsw_sp_neigh_entry *neigh_entry;
2347         struct mlxsw_sp_rif *rif;
2348         int err;
2349
2350         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2351         if (!rif)
2352                 return ERR_PTR(-EINVAL);
2353
2354         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2355         if (!neigh_entry)
2356                 return ERR_PTR(-ENOMEM);
2357
2358         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2359         if (err)
2360                 goto err_neigh_entry_insert;
2361
2362         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2363         atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count);
2364         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2365
2366         return neigh_entry;
2367
2368 err_neigh_entry_insert:
2369         mlxsw_sp_neigh_entry_free(neigh_entry);
2370         return ERR_PTR(err);
2371 }
2372
2373 static void
2374 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2375                              struct mlxsw_sp_neigh_entry *neigh_entry)
2376 {
2377         list_del(&neigh_entry->rif_list_node);
2378         atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count);
2379         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2380         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2381         mlxsw_sp_neigh_entry_free(neigh_entry);
2382 }
2383
2384 static struct mlxsw_sp_neigh_entry *
2385 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2386 {
2387         struct mlxsw_sp_neigh_key key;
2388
2389         key.n = n;
2390         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2391                                       &key, mlxsw_sp_neigh_ht_params);
2392 }
2393
2394 static void
2395 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2396 {
2397         unsigned long interval;
2398
2399 #if IS_ENABLED(CONFIG_IPV6)
2400         interval = min_t(unsigned long,
2401                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2402                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2403 #else
2404         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2405 #endif
2406         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2407 }
2408
2409 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2410                                                    char *rauhtd_pl,
2411                                                    int ent_index)
2412 {
2413         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2414         struct net_device *dev;
2415         struct neighbour *n;
2416         __be32 dipn;
2417         u32 dip;
2418         u16 rif;
2419
2420         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2421
2422         if (WARN_ON_ONCE(rif >= max_rifs))
2423                 return;
2424         if (!mlxsw_sp->router->rifs[rif]) {
2425                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2426                 return;
2427         }
2428
2429         dipn = htonl(dip);
2430         dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
2431         n = neigh_lookup(&arp_tbl, &dipn, dev);
2432         if (!n)
2433                 return;
2434
2435         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2436         neigh_event_send(n, NULL);
2437         neigh_release(n);
2438 }
2439
2440 #if IS_ENABLED(CONFIG_IPV6)
2441 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2442                                                    char *rauhtd_pl,
2443                                                    int rec_index)
2444 {
2445         struct net_device *dev;
2446         struct neighbour *n;
2447         struct in6_addr dip;
2448         u16 rif;
2449
2450         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2451                                          (char *) &dip);
2452
2453         if (!mlxsw_sp->router->rifs[rif]) {
2454                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2455                 return;
2456         }
2457
2458         dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
2459         n = neigh_lookup(&nd_tbl, &dip, dev);
2460         if (!n)
2461                 return;
2462
2463         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2464         neigh_event_send(n, NULL);
2465         neigh_release(n);
2466 }
2467 #else
2468 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2469                                                    char *rauhtd_pl,
2470                                                    int rec_index)
2471 {
2472 }
2473 #endif
2474
2475 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2476                                                    char *rauhtd_pl,
2477                                                    int rec_index)
2478 {
2479         u8 num_entries;
2480         int i;
2481
2482         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2483                                                                 rec_index);
2484         /* Hardware starts counting at 0, so add 1. */
2485         num_entries++;
2486
2487         /* Each record consists of several neighbour entries. */
2488         for (i = 0; i < num_entries; i++) {
2489                 int ent_index;
2490
2491                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2492                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2493                                                        ent_index);
2494         }
2495
2496 }
2497
2498 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2499                                                    char *rauhtd_pl,
2500                                                    int rec_index)
2501 {
2502         /* One record contains one entry. */
2503         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2504                                                rec_index);
2505 }
2506
2507 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2508                                               char *rauhtd_pl, int rec_index)
2509 {
2510         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2511         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2512                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2513                                                        rec_index);
2514                 break;
2515         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2516                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2517                                                        rec_index);
2518                 break;
2519         }
2520 }
2521
2522 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2523 {
2524         u8 num_rec, last_rec_index, num_entries;
2525
2526         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2527         last_rec_index = num_rec - 1;
2528
2529         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2530                 return false;
2531         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2532             MLXSW_REG_RAUHTD_TYPE_IPV6)
2533                 return true;
2534
2535         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2536                                                                 last_rec_index);
2537         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2538                 return true;
2539         return false;
2540 }
2541
2542 static int
2543 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2544                                        char *rauhtd_pl,
2545                                        enum mlxsw_reg_rauhtd_type type)
2546 {
2547         int i, num_rec;
2548         int err;
2549
2550         /* Ensure the RIF we read from the device does not change mid-dump. */
2551         mutex_lock(&mlxsw_sp->router->lock);
2552         do {
2553                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2554                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2555                                       rauhtd_pl);
2556                 if (err) {
2557                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2558                         break;
2559                 }
2560                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2561                 for (i = 0; i < num_rec; i++)
2562                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2563                                                           i);
2564         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2565         mutex_unlock(&mlxsw_sp->router->lock);
2566
2567         return err;
2568 }
2569
2570 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2571 {
2572         enum mlxsw_reg_rauhtd_type type;
2573         char *rauhtd_pl;
2574         int err;
2575
2576         if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count))
2577                 return 0;
2578
2579         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2580         if (!rauhtd_pl)
2581                 return -ENOMEM;
2582
2583         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2584         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2585         if (err)
2586                 goto out;
2587
2588         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2589         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2590 out:
2591         kfree(rauhtd_pl);
2592         return err;
2593 }
2594
2595 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2596 {
2597         struct mlxsw_sp_neigh_entry *neigh_entry;
2598
2599         mutex_lock(&mlxsw_sp->router->lock);
2600         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2601                             nexthop_neighs_list_node)
2602                 /* If this neigh have nexthops, make the kernel think this neigh
2603                  * is active regardless of the traffic.
2604                  */
2605                 neigh_event_send(neigh_entry->key.n, NULL);
2606         mutex_unlock(&mlxsw_sp->router->lock);
2607 }
2608
2609 static void
2610 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2611 {
2612         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2613
2614         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2615                                msecs_to_jiffies(interval));
2616 }
2617
2618 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2619 {
2620         struct mlxsw_sp_router *router;
2621         int err;
2622
2623         router = container_of(work, struct mlxsw_sp_router,
2624                               neighs_update.dw.work);
2625         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2626         if (err)
2627                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2628
2629         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2630
2631         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2632 }
2633
2634 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2635 {
2636         struct mlxsw_sp_neigh_entry *neigh_entry;
2637         struct mlxsw_sp_router *router;
2638
2639         router = container_of(work, struct mlxsw_sp_router,
2640                               nexthop_probe_dw.work);
2641         /* Iterate over nexthop neighbours, find those who are unresolved and
2642          * send arp on them. This solves the chicken-egg problem when
2643          * the nexthop wouldn't get offloaded until the neighbor is resolved
2644          * but it wouldn't get resolved ever in case traffic is flowing in HW
2645          * using different nexthop.
2646          */
2647         mutex_lock(&router->lock);
2648         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2649                             nexthop_neighs_list_node)
2650                 if (!neigh_entry->connected)
2651                         neigh_event_send(neigh_entry->key.n, NULL);
2652         mutex_unlock(&router->lock);
2653
2654         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2655                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2656 }
2657
2658 static void
2659 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2660                               struct mlxsw_sp_neigh_entry *neigh_entry,
2661                               bool removing, bool dead);
2662
2663 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2664 {
2665         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2666                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2667 }
2668
2669 static int
2670 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2671                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2672                                 enum mlxsw_reg_rauht_op op)
2673 {
2674         struct neighbour *n = neigh_entry->key.n;
2675         u32 dip = ntohl(*((__be32 *) n->primary_key));
2676         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2677
2678         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2679                               dip);
2680         if (neigh_entry->counter_valid)
2681                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2682                                              neigh_entry->counter_index);
2683         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2684 }
2685
2686 static int
2687 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2688                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2689                                 enum mlxsw_reg_rauht_op op)
2690 {
2691         struct neighbour *n = neigh_entry->key.n;
2692         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2693         const char *dip = n->primary_key;
2694
2695         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2696                               dip);
2697         if (neigh_entry->counter_valid)
2698                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2699                                              neigh_entry->counter_index);
2700         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2701 }
2702
2703 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2704 {
2705         struct neighbour *n = neigh_entry->key.n;
2706
2707         /* Packets with a link-local destination address are trapped
2708          * after LPM lookup and never reach the neighbour table, so
2709          * there is no need to program such neighbours to the device.
2710          */
2711         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2712             IPV6_ADDR_LINKLOCAL)
2713                 return true;
2714         return false;
2715 }
2716
2717 static void
2718 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2719                             struct mlxsw_sp_neigh_entry *neigh_entry,
2720                             bool adding)
2721 {
2722         enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2723         int err;
2724
2725         if (!adding && !neigh_entry->connected)
2726                 return;
2727         neigh_entry->connected = adding;
2728         if (neigh_entry->key.n->tbl->family == AF_INET) {
2729                 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2730                                                       op);
2731                 if (err)
2732                         return;
2733         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2734                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2735                         return;
2736                 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2737                                                       op);
2738                 if (err)
2739                         return;
2740         } else {
2741                 WARN_ON_ONCE(1);
2742                 return;
2743         }
2744
2745         if (adding)
2746                 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2747         else
2748                 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2749 }
2750
2751 void
2752 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2753                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2754                                     bool adding)
2755 {
2756         if (adding)
2757                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2758         else
2759                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2760         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2761 }
2762
2763 struct mlxsw_sp_netevent_work {
2764         struct work_struct work;
2765         struct mlxsw_sp *mlxsw_sp;
2766         struct neighbour *n;
2767 };
2768
2769 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2770 {
2771         struct mlxsw_sp_netevent_work *net_work =
2772                 container_of(work, struct mlxsw_sp_netevent_work, work);
2773         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2774         struct mlxsw_sp_neigh_entry *neigh_entry;
2775         struct neighbour *n = net_work->n;
2776         unsigned char ha[ETH_ALEN];
2777         bool entry_connected;
2778         u8 nud_state, dead;
2779
2780         /* If these parameters are changed after we release the lock,
2781          * then we are guaranteed to receive another event letting us
2782          * know about it.
2783          */
2784         read_lock_bh(&n->lock);
2785         memcpy(ha, n->ha, ETH_ALEN);
2786         nud_state = n->nud_state;
2787         dead = n->dead;
2788         read_unlock_bh(&n->lock);
2789
2790         mutex_lock(&mlxsw_sp->router->lock);
2791         mlxsw_sp_span_respin(mlxsw_sp);
2792
2793         entry_connected = nud_state & NUD_VALID && !dead;
2794         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2795         if (!entry_connected && !neigh_entry)
2796                 goto out;
2797         if (!neigh_entry) {
2798                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2799                 if (IS_ERR(neigh_entry))
2800                         goto out;
2801         }
2802
2803         if (neigh_entry->connected && entry_connected &&
2804             !memcmp(neigh_entry->ha, ha, ETH_ALEN))
2805                 goto out;
2806
2807         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2808         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2809         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2810                                       dead);
2811
2812         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2813                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2814
2815 out:
2816         mutex_unlock(&mlxsw_sp->router->lock);
2817         neigh_release(n);
2818         kfree(net_work);
2819 }
2820
2821 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2822
2823 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2824 {
2825         struct mlxsw_sp_netevent_work *net_work =
2826                 container_of(work, struct mlxsw_sp_netevent_work, work);
2827         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2828
2829         mlxsw_sp_mp_hash_init(mlxsw_sp);
2830         kfree(net_work);
2831 }
2832
2833 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2834
2835 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2836 {
2837         struct mlxsw_sp_netevent_work *net_work =
2838                 container_of(work, struct mlxsw_sp_netevent_work, work);
2839         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2840
2841         __mlxsw_sp_router_init(mlxsw_sp);
2842         kfree(net_work);
2843 }
2844
2845 static int mlxsw_sp_router_schedule_work(struct net *net,
2846                                          struct mlxsw_sp_router *router,
2847                                          struct neighbour *n,
2848                                          void (*cb)(struct work_struct *))
2849 {
2850         struct mlxsw_sp_netevent_work *net_work;
2851
2852         if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2853                 return NOTIFY_DONE;
2854
2855         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2856         if (!net_work)
2857                 return NOTIFY_BAD;
2858
2859         INIT_WORK(&net_work->work, cb);
2860         net_work->mlxsw_sp = router->mlxsw_sp;
2861         net_work->n = n;
2862         mlxsw_core_schedule_work(&net_work->work);
2863         return NOTIFY_DONE;
2864 }
2865
2866 static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev)
2867 {
2868         struct mlxsw_sp_port *mlxsw_sp_port;
2869
2870         rcu_read_lock();
2871         mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev);
2872         rcu_read_unlock();
2873         return !!mlxsw_sp_port;
2874 }
2875
2876 static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router,
2877                                                struct neighbour *n)
2878 {
2879         struct net *net;
2880
2881         net = neigh_parms_net(n->parms);
2882
2883         /* Take a reference to ensure the neighbour won't be destructed until we
2884          * drop the reference in delayed work.
2885          */
2886         neigh_clone(n);
2887         return mlxsw_sp_router_schedule_work(net, router, n,
2888                                              mlxsw_sp_router_neigh_event_work);
2889 }
2890
2891 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2892                                           unsigned long event, void *ptr)
2893 {
2894         struct mlxsw_sp_router *router;
2895         unsigned long interval;
2896         struct neigh_parms *p;
2897         struct neighbour *n;
2898
2899         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2900
2901         switch (event) {
2902         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2903                 p = ptr;
2904
2905                 /* We don't care about changes in the default table. */
2906                 if (!p->dev || (p->tbl->family != AF_INET &&
2907                                 p->tbl->family != AF_INET6))
2908                         return NOTIFY_DONE;
2909
2910                 /* We are in atomic context and can't take RTNL mutex,
2911                  * so use RCU variant to walk the device chain.
2912                  */
2913                 if (!mlxsw_sp_dev_lower_is_port(p->dev))
2914                         return NOTIFY_DONE;
2915
2916                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2917                 router->neighs_update.interval = interval;
2918                 break;
2919         case NETEVENT_NEIGH_UPDATE:
2920                 n = ptr;
2921
2922                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2923                         return NOTIFY_DONE;
2924
2925                 if (!mlxsw_sp_dev_lower_is_port(n->dev))
2926                         return NOTIFY_DONE;
2927
2928                 return mlxsw_sp_router_schedule_neigh_work(router, n);
2929
2930         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2931         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2932                 return mlxsw_sp_router_schedule_work(ptr, router, NULL,
2933                                 mlxsw_sp_router_mp_hash_event_work);
2934
2935         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2936                 return mlxsw_sp_router_schedule_work(ptr, router, NULL,
2937                                 mlxsw_sp_router_update_priority_work);
2938         }
2939
2940         return NOTIFY_DONE;
2941 }
2942
2943 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2944 {
2945         int err;
2946
2947         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2948                               &mlxsw_sp_neigh_ht_params);
2949         if (err)
2950                 return err;
2951
2952         /* Initialize the polling interval according to the default
2953          * table.
2954          */
2955         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2956
2957         /* Create the delayed works for the activity_update */
2958         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2959                           mlxsw_sp_router_neighs_update_work);
2960         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2961                           mlxsw_sp_router_probe_unresolved_nexthops);
2962         atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0);
2963         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2964         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2965         return 0;
2966 }
2967
2968 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2969 {
2970         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2971         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2972         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2973 }
2974
2975 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2976                                          struct mlxsw_sp_rif *rif)
2977 {
2978         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2979
2980         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2981                                  rif_list_node) {
2982                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2983                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2984         }
2985 }
2986
2987 struct mlxsw_sp_neigh_rif_made_sync {
2988         struct mlxsw_sp *mlxsw_sp;
2989         struct mlxsw_sp_rif *rif;
2990         int err;
2991 };
2992
2993 static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data)
2994 {
2995         struct mlxsw_sp_neigh_rif_made_sync *rms = data;
2996         int rc;
2997
2998         if (rms->err)
2999                 return;
3000         if (n->dev != mlxsw_sp_rif_dev(rms->rif))
3001                 return;
3002         rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n);
3003         if (rc != NOTIFY_DONE)
3004                 rms->err = -ENOMEM;
3005 }
3006
3007 static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
3008                                         struct mlxsw_sp_rif *rif)
3009 {
3010         struct mlxsw_sp_neigh_rif_made_sync rms = {
3011                 .mlxsw_sp = mlxsw_sp,
3012                 .rif = rif,
3013         };
3014
3015         neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
3016         if (rms.err)
3017                 goto err_arp;
3018
3019 #if IS_ENABLED(CONFIG_IPV6)
3020         neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
3021 #endif
3022         if (rms.err)
3023                 goto err_nd;
3024
3025         return 0;
3026
3027 err_nd:
3028 err_arp:
3029         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
3030         return rms.err;
3031 }
3032
3033 enum mlxsw_sp_nexthop_type {
3034         MLXSW_SP_NEXTHOP_TYPE_ETH,
3035         MLXSW_SP_NEXTHOP_TYPE_IPIP,
3036 };
3037
3038 enum mlxsw_sp_nexthop_action {
3039         /* Nexthop forwards packets to an egress RIF */
3040         MLXSW_SP_NEXTHOP_ACTION_FORWARD,
3041         /* Nexthop discards packets */
3042         MLXSW_SP_NEXTHOP_ACTION_DISCARD,
3043         /* Nexthop traps packets */
3044         MLXSW_SP_NEXTHOP_ACTION_TRAP,
3045 };
3046
3047 struct mlxsw_sp_nexthop_key {
3048         struct fib_nh *fib_nh;
3049 };
3050
3051 struct mlxsw_sp_nexthop {
3052         struct list_head neigh_list_node; /* member of neigh entry list */
3053         struct list_head crif_list_node;
3054         struct list_head router_list_node;
3055         struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
3056                                                    * this nexthop belongs to
3057                                                    */
3058         struct rhash_head ht_node;
3059         struct neigh_table *neigh_tbl;
3060         struct mlxsw_sp_nexthop_key key;
3061         unsigned char gw_addr[sizeof(struct in6_addr)];
3062         int ifindex;
3063         int nh_weight;
3064         int norm_nh_weight;
3065         int num_adj_entries;
3066         struct mlxsw_sp_crif *crif;
3067         u8 should_offload:1, /* set indicates this nexthop should be written
3068                               * to the adjacency table.
3069                               */
3070            offloaded:1, /* set indicates this nexthop was written to the
3071                          * adjacency table.
3072                          */
3073            update:1; /* set indicates this nexthop should be updated in the
3074                       * adjacency table (f.e., its MAC changed).
3075                       */
3076         enum mlxsw_sp_nexthop_action action;
3077         enum mlxsw_sp_nexthop_type type;
3078         union {
3079                 struct mlxsw_sp_neigh_entry *neigh_entry;
3080                 struct mlxsw_sp_ipip_entry *ipip_entry;
3081         };
3082         unsigned int counter_index;
3083         bool counter_valid;
3084 };
3085
3086 static struct net_device *
3087 mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh)
3088 {
3089         if (!nh->crif)
3090                 return NULL;
3091         return nh->crif->key.dev;
3092 }
3093
3094 enum mlxsw_sp_nexthop_group_type {
3095         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
3096         MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
3097         MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
3098 };
3099
3100 struct mlxsw_sp_nexthop_group_info {
3101         struct mlxsw_sp_nexthop_group *nh_grp;
3102         u32 adj_index;
3103         u16 ecmp_size;
3104         u16 count;
3105         int sum_norm_weight;
3106         u8 adj_index_valid:1,
3107            gateway:1, /* routes using the group use a gateway */
3108            is_resilient:1;
3109         struct list_head list; /* member in nh_res_grp_list */
3110         struct mlxsw_sp_nexthop nexthops[];
3111 };
3112
3113 static struct mlxsw_sp_rif *
3114 mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi)
3115 {
3116         struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif;
3117
3118         if (!crif)
3119                 return NULL;
3120         return crif->rif;
3121 }
3122
3123 struct mlxsw_sp_nexthop_group_vr_key {
3124         u16 vr_id;
3125         enum mlxsw_sp_l3proto proto;
3126 };
3127
3128 struct mlxsw_sp_nexthop_group_vr_entry {
3129         struct list_head list; /* member in vr_list */
3130         struct rhash_head ht_node; /* member in vr_ht */
3131         refcount_t ref_count;
3132         struct mlxsw_sp_nexthop_group_vr_key key;
3133 };
3134
3135 struct mlxsw_sp_nexthop_group {
3136         struct rhash_head ht_node;
3137         struct list_head fib_list; /* list of fib entries that use this group */
3138         union {
3139                 struct {
3140                         struct fib_info *fi;
3141                 } ipv4;
3142                 struct {
3143                         u32 id;
3144                 } obj;
3145         };
3146         struct mlxsw_sp_nexthop_group_info *nhgi;
3147         struct list_head vr_list;
3148         struct rhashtable vr_ht;
3149         enum mlxsw_sp_nexthop_group_type type;
3150         bool can_destroy;
3151 };
3152
3153 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
3154                                     struct mlxsw_sp_nexthop *nh)
3155 {
3156         struct devlink *devlink;
3157
3158         devlink = priv_to_devlink(mlxsw_sp->core);
3159         if (!devlink_dpipe_table_counter_enabled(devlink,
3160                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
3161                 return;
3162
3163         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
3164                 return;
3165
3166         nh->counter_valid = true;
3167 }
3168
3169 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
3170                                    struct mlxsw_sp_nexthop *nh)
3171 {
3172         if (!nh->counter_valid)
3173                 return;
3174         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
3175         nh->counter_valid = false;
3176 }
3177
3178 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
3179                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
3180 {
3181         if (!nh->counter_valid)
3182                 return -EINVAL;
3183
3184         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
3185                                          p_counter, NULL);
3186 }
3187
3188 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
3189                                                struct mlxsw_sp_nexthop *nh)
3190 {
3191         if (!nh) {
3192                 if (list_empty(&router->nexthop_list))
3193                         return NULL;
3194                 else
3195                         return list_first_entry(&router->nexthop_list,
3196                                                 typeof(*nh), router_list_node);
3197         }
3198         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
3199                 return NULL;
3200         return list_next_entry(nh, router_list_node);
3201 }
3202
3203 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
3204 {
3205         return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3206 }
3207
3208 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3209 {
3210         if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3211             !mlxsw_sp_nexthop_is_forward(nh))
3212                 return NULL;
3213         return nh->neigh_entry->ha;
3214 }
3215
3216 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3217                              u32 *p_adj_size, u32 *p_adj_hash_index)
3218 {
3219         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3220         u32 adj_hash_index = 0;
3221         int i;
3222
3223         if (!nh->offloaded || !nhgi->adj_index_valid)
3224                 return -EINVAL;
3225
3226         *p_adj_index = nhgi->adj_index;
3227         *p_adj_size = nhgi->ecmp_size;
3228
3229         for (i = 0; i < nhgi->count; i++) {
3230                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3231
3232                 if (nh_iter == nh)
3233                         break;
3234                 if (nh_iter->offloaded)
3235                         adj_hash_index += nh_iter->num_adj_entries;
3236         }
3237
3238         *p_adj_hash_index = adj_hash_index;
3239         return 0;
3240 }
3241
3242 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3243 {
3244         if (WARN_ON(!nh->crif))
3245                 return NULL;
3246         return nh->crif->rif;
3247 }
3248
3249 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3250 {
3251         struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3252         int i;
3253
3254         for (i = 0; i < nhgi->count; i++) {
3255                 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3256
3257                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3258                         return true;
3259         }
3260         return false;
3261 }
3262
3263 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3264         .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3265         .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3266         .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3267         .automatic_shrinking = true,
3268 };
3269
3270 static struct mlxsw_sp_nexthop_group_vr_entry *
3271 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3272                                        const struct mlxsw_sp_fib *fib)
3273 {
3274         struct mlxsw_sp_nexthop_group_vr_key key;
3275
3276         memset(&key, 0, sizeof(key));
3277         key.vr_id = fib->vr->id;
3278         key.proto = fib->proto;
3279         return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
3280                                       mlxsw_sp_nexthop_group_vr_ht_params);
3281 }
3282
3283 static int
3284 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3285                                        const struct mlxsw_sp_fib *fib)
3286 {
3287         struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3288         int err;
3289
3290         vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
3291         if (!vr_entry)
3292                 return -ENOMEM;
3293
3294         vr_entry->key.vr_id = fib->vr->id;
3295         vr_entry->key.proto = fib->proto;
3296         refcount_set(&vr_entry->ref_count, 1);
3297
3298         err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3299                                      mlxsw_sp_nexthop_group_vr_ht_params);
3300         if (err)
3301                 goto err_hashtable_insert;
3302
3303         list_add(&vr_entry->list, &nh_grp->vr_list);
3304
3305         return 0;
3306
3307 err_hashtable_insert:
3308         kfree(vr_entry);
3309         return err;
3310 }
3311
3312 static void
3313 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3314                                         struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3315 {
3316         list_del(&vr_entry->list);
3317         rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3318                                mlxsw_sp_nexthop_group_vr_ht_params);
3319         kfree(vr_entry);
3320 }
3321
3322 static int
3323 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3324                                const struct mlxsw_sp_fib *fib)
3325 {
3326         struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3327
3328         vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3329         if (vr_entry) {
3330                 refcount_inc(&vr_entry->ref_count);
3331                 return 0;
3332         }
3333
3334         return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3335 }
3336
3337 static void
3338 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3339                                  const struct mlxsw_sp_fib *fib)
3340 {
3341         struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3342
3343         vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3344         if (WARN_ON_ONCE(!vr_entry))
3345                 return;
3346
3347         if (!refcount_dec_and_test(&vr_entry->ref_count))
3348                 return;
3349
3350         mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3351 }
3352
3353 struct mlxsw_sp_nexthop_group_cmp_arg {
3354         enum mlxsw_sp_nexthop_group_type type;
3355         union {
3356                 struct fib_info *fi;
3357                 struct mlxsw_sp_fib6_entry *fib6_entry;
3358                 u32 id;
3359         };
3360 };
3361
3362 static bool
3363 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3364                                     const struct in6_addr *gw, int ifindex,
3365                                     int weight)
3366 {
3367         int i;
3368
3369         for (i = 0; i < nh_grp->nhgi->count; i++) {
3370                 const struct mlxsw_sp_nexthop *nh;
3371
3372                 nh = &nh_grp->nhgi->nexthops[i];
3373                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3374                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3375                         return true;
3376         }
3377
3378         return false;
3379 }
3380
3381 static bool
3382 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3383                             const struct mlxsw_sp_fib6_entry *fib6_entry)
3384 {
3385         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3386
3387         if (nh_grp->nhgi->count != fib6_entry->nrt6)
3388                 return false;
3389
3390         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3391                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3392                 struct in6_addr *gw;
3393                 int ifindex, weight;
3394
3395                 ifindex = fib6_nh->fib_nh_dev->ifindex;
3396                 weight = fib6_nh->fib_nh_weight;
3397                 gw = &fib6_nh->fib_nh_gw6;
3398                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3399                                                          weight))
3400                         return false;
3401         }
3402
3403         return true;
3404 }
3405
3406 static int
3407 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3408 {
3409         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3410         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3411
3412         if (nh_grp->type != cmp_arg->type)
3413                 return 1;
3414
3415         switch (cmp_arg->type) {
3416         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3417                 return cmp_arg->fi != nh_grp->ipv4.fi;
3418         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3419                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3420                                                     cmp_arg->fib6_entry);
3421         case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3422                 return cmp_arg->id != nh_grp->obj.id;
3423         default:
3424                 WARN_ON(1);
3425                 return 1;
3426         }
3427 }
3428
3429 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3430 {
3431         const struct mlxsw_sp_nexthop_group *nh_grp = data;
3432         const struct mlxsw_sp_nexthop *nh;
3433         struct fib_info *fi;
3434         unsigned int val;
3435         int i;
3436
3437         switch (nh_grp->type) {
3438         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3439                 fi = nh_grp->ipv4.fi;
3440                 return jhash(&fi, sizeof(fi), seed);
3441         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3442                 val = nh_grp->nhgi->count;
3443                 for (i = 0; i < nh_grp->nhgi->count; i++) {
3444                         nh = &nh_grp->nhgi->nexthops[i];
3445                         val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3446                         val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3447                 }
3448                 return jhash(&val, sizeof(val), seed);
3449         case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3450                 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
3451         default:
3452                 WARN_ON(1);
3453                 return 0;
3454         }
3455 }
3456
3457 static u32
3458 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3459 {
3460         unsigned int val = fib6_entry->nrt6;
3461         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3462
3463         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3464                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3465                 struct net_device *dev = fib6_nh->fib_nh_dev;
3466                 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3467
3468                 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3469                 val ^= jhash(gw, sizeof(*gw), seed);
3470         }
3471
3472         return jhash(&val, sizeof(val), seed);
3473 }
3474
3475 static u32
3476 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3477 {
3478         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3479
3480         switch (cmp_arg->type) {
3481         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3482                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3483         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3484                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3485         case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3486                 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
3487         default:
3488                 WARN_ON(1);
3489                 return 0;
3490         }
3491 }
3492
3493 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3494         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3495         .hashfn      = mlxsw_sp_nexthop_group_hash,
3496         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3497         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3498 };
3499
3500 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3501                                          struct mlxsw_sp_nexthop_group *nh_grp)
3502 {
3503         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3504             !nh_grp->nhgi->gateway)
3505                 return 0;
3506
3507         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3508                                       &nh_grp->ht_node,
3509                                       mlxsw_sp_nexthop_group_ht_params);
3510 }
3511
3512 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3513                                           struct mlxsw_sp_nexthop_group *nh_grp)
3514 {
3515         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3516             !nh_grp->nhgi->gateway)
3517                 return;
3518
3519         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3520                                &nh_grp->ht_node,
3521                                mlxsw_sp_nexthop_group_ht_params);
3522 }
3523
3524 static struct mlxsw_sp_nexthop_group *
3525 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3526                                struct fib_info *fi)
3527 {
3528         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3529
3530         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3531         cmp_arg.fi = fi;
3532         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3533                                       &cmp_arg,
3534                                       mlxsw_sp_nexthop_group_ht_params);
3535 }
3536
3537 static struct mlxsw_sp_nexthop_group *
3538 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3539                                struct mlxsw_sp_fib6_entry *fib6_entry)
3540 {
3541         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3542
3543         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3544         cmp_arg.fib6_entry = fib6_entry;
3545         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3546                                       &cmp_arg,
3547                                       mlxsw_sp_nexthop_group_ht_params);
3548 }
3549
3550 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3551         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3552         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3553         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3554 };
3555
3556 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3557                                    struct mlxsw_sp_nexthop *nh)
3558 {
3559         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3560                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3561 }
3562
3563 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3564                                     struct mlxsw_sp_nexthop *nh)
3565 {
3566         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3567                                mlxsw_sp_nexthop_ht_params);
3568 }
3569
3570 static struct mlxsw_sp_nexthop *
3571 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3572                         struct mlxsw_sp_nexthop_key key)
3573 {
3574         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3575                                       mlxsw_sp_nexthop_ht_params);
3576 }
3577
3578 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3579                                              enum mlxsw_sp_l3proto proto,
3580                                              u16 vr_id,
3581                                              u32 adj_index, u16 ecmp_size,
3582                                              u32 new_adj_index,
3583                                              u16 new_ecmp_size)
3584 {
3585         char raleu_pl[MLXSW_REG_RALEU_LEN];
3586
3587         mlxsw_reg_raleu_pack(raleu_pl,
3588                              (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
3589                              adj_index, ecmp_size, new_adj_index,
3590                              new_ecmp_size);
3591         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3592 }
3593
3594 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3595                                           struct mlxsw_sp_nexthop_group *nh_grp,
3596                                           u32 old_adj_index, u16 old_ecmp_size)
3597 {
3598         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3599         struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3600         int err;
3601
3602         list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3603                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3604                                                         vr_entry->key.proto,
3605                                                         vr_entry->key.vr_id,
3606                                                         old_adj_index,
3607                                                         old_ecmp_size,
3608                                                         nhgi->adj_index,
3609                                                         nhgi->ecmp_size);
3610                 if (err)
3611                         goto err_mass_update_vr;
3612         }
3613         return 0;
3614
3615 err_mass_update_vr:
3616         list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3617                 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
3618                                                   vr_entry->key.vr_id,
3619                                                   nhgi->adj_index,
3620                                                   nhgi->ecmp_size,
3621                                                   old_adj_index, old_ecmp_size);
3622         return err;
3623 }
3624
3625 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3626                                          u32 adj_index,
3627                                          struct mlxsw_sp_nexthop *nh,
3628                                          bool force, char *ratr_pl)
3629 {
3630         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3631         struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
3632         enum mlxsw_reg_ratr_op op;
3633         u16 rif_index;
3634
3635         rif_index = rif ? rif->rif_index :
3636                           mlxsw_sp->router->lb_crif->rif->rif_index;
3637         op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
3638                      MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
3639         mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET,
3640                             adj_index, rif_index);
3641         switch (nh->action) {
3642         case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3643                 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3644                 break;
3645         case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3646                 mlxsw_reg_ratr_trap_action_set(ratr_pl,
3647                                                MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3648                 break;
3649         case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3650                 mlxsw_reg_ratr_trap_action_set(ratr_pl,
3651                                                MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3652                 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
3653                 break;
3654         default:
3655                 WARN_ON_ONCE(1);
3656                 return -EINVAL;
3657         }
3658         if (nh->counter_valid)
3659                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3660         else
3661                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3662
3663         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3664 }
3665
3666 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3667                                 struct mlxsw_sp_nexthop *nh, bool force,
3668                                 char *ratr_pl)
3669 {
3670         int i;
3671
3672         for (i = 0; i < nh->num_adj_entries; i++) {
3673                 int err;
3674
3675                 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
3676                                                     nh, force, ratr_pl);
3677                 if (err)
3678                         return err;
3679         }
3680
3681         return 0;
3682 }
3683
3684 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3685                                           u32 adj_index,
3686                                           struct mlxsw_sp_nexthop *nh,
3687                                           bool force, char *ratr_pl)
3688 {
3689         const struct mlxsw_sp_ipip_ops *ipip_ops;
3690
3691         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3692         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
3693                                         force, ratr_pl);
3694 }
3695
3696 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3697                                         u32 adj_index,
3698                                         struct mlxsw_sp_nexthop *nh, bool force,
3699                                         char *ratr_pl)
3700 {
3701         int i;
3702
3703         for (i = 0; i < nh->num_adj_entries; i++) {
3704                 int err;
3705
3706                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3707                                                      nh, force, ratr_pl);
3708                 if (err)
3709                         return err;
3710         }
3711
3712         return 0;
3713 }
3714
3715 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3716                                    struct mlxsw_sp_nexthop *nh, bool force,
3717                                    char *ratr_pl)
3718 {
3719         /* When action is discard or trap, the nexthop must be
3720          * programmed as an Ethernet nexthop.
3721          */
3722         if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3723             nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3724             nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3725                 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
3726                                                    force, ratr_pl);
3727         else
3728                 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
3729                                                     force, ratr_pl);
3730 }
3731
3732 static int
3733 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3734                               struct mlxsw_sp_nexthop_group_info *nhgi,
3735                               bool reallocate)
3736 {
3737         char ratr_pl[MLXSW_REG_RATR_LEN];
3738         u32 adj_index = nhgi->adj_index; /* base */
3739         struct mlxsw_sp_nexthop *nh;
3740         int i;
3741
3742         for (i = 0; i < nhgi->count; i++) {
3743                 nh = &nhgi->nexthops[i];
3744
3745                 if (!nh->should_offload) {
3746                         nh->offloaded = 0;
3747                         continue;
3748                 }
3749
3750                 if (nh->update || reallocate) {
3751                         int err = 0;
3752
3753                         err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
3754                                                       true, ratr_pl);
3755                         if (err)
3756                                 return err;
3757                         nh->update = 0;
3758                         nh->offloaded = 1;
3759                 }
3760                 adj_index += nh->num_adj_entries;
3761         }
3762         return 0;
3763 }
3764
3765 static int
3766 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3767                                     struct mlxsw_sp_nexthop_group *nh_grp)
3768 {
3769         struct mlxsw_sp_fib_entry *fib_entry;
3770         int err;
3771
3772         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3773                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3774                 if (err)
3775                         return err;
3776         }
3777         return 0;
3778 }
3779
3780 struct mlxsw_sp_adj_grp_size_range {
3781         u16 start; /* Inclusive */
3782         u16 end; /* Inclusive */
3783 };
3784
3785 /* Ordered by range start value */
3786 static const struct mlxsw_sp_adj_grp_size_range
3787 mlxsw_sp1_adj_grp_size_ranges[] = {
3788         { .start = 1, .end = 64 },
3789         { .start = 512, .end = 512 },
3790         { .start = 1024, .end = 1024 },
3791         { .start = 2048, .end = 2048 },
3792         { .start = 4096, .end = 4096 },
3793 };
3794
3795 /* Ordered by range start value */
3796 static const struct mlxsw_sp_adj_grp_size_range
3797 mlxsw_sp2_adj_grp_size_ranges[] = {
3798         { .start = 1, .end = 128 },
3799         { .start = 256, .end = 256 },
3800         { .start = 512, .end = 512 },
3801         { .start = 1024, .end = 1024 },
3802         { .start = 2048, .end = 2048 },
3803         { .start = 4096, .end = 4096 },
3804 };
3805
3806 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
3807                                            u16 *p_adj_grp_size)
3808 {
3809         int i;
3810
3811         for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
3812                 const struct mlxsw_sp_adj_grp_size_range *size_range;
3813
3814                 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3815
3816                 if (*p_adj_grp_size >= size_range->start &&
3817                     *p_adj_grp_size <= size_range->end)
3818                         return;
3819
3820                 if (*p_adj_grp_size <= size_range->end) {
3821                         *p_adj_grp_size = size_range->end;
3822                         return;
3823                 }
3824         }
3825 }
3826
3827 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
3828                                              u16 *p_adj_grp_size,
3829                                              unsigned int alloc_size)
3830 {
3831         int i;
3832
3833         for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
3834                 const struct mlxsw_sp_adj_grp_size_range *size_range;
3835
3836                 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3837
3838                 if (alloc_size >= size_range->end) {
3839                         *p_adj_grp_size = size_range->end;
3840                         return;
3841                 }
3842         }
3843 }
3844
3845 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3846                                      u16 *p_adj_grp_size)
3847 {
3848         unsigned int alloc_size;
3849         int err;
3850
3851         /* Round up the requested group size to the next size supported
3852          * by the device and make sure the request can be satisfied.
3853          */
3854         mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
3855         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3856                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3857                                               *p_adj_grp_size, &alloc_size);
3858         if (err)
3859                 return err;
3860         /* It is possible the allocation results in more allocated
3861          * entries than requested. Try to use as much of them as
3862          * possible.
3863          */
3864         mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
3865
3866         return 0;
3867 }
3868
3869 static void
3870 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3871 {
3872         int i, g = 0, sum_norm_weight = 0;
3873         struct mlxsw_sp_nexthop *nh;
3874
3875         for (i = 0; i < nhgi->count; i++) {
3876                 nh = &nhgi->nexthops[i];
3877
3878                 if (!nh->should_offload)
3879                         continue;
3880                 if (g > 0)
3881                         g = gcd(nh->nh_weight, g);
3882                 else
3883                         g = nh->nh_weight;
3884         }
3885
3886         for (i = 0; i < nhgi->count; i++) {
3887                 nh = &nhgi->nexthops[i];
3888
3889                 if (!nh->should_offload)
3890                         continue;
3891                 nh->norm_nh_weight = nh->nh_weight / g;
3892                 sum_norm_weight += nh->norm_nh_weight;
3893         }
3894
3895         nhgi->sum_norm_weight = sum_norm_weight;
3896 }
3897
3898 static void
3899 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3900 {
3901         int i, weight = 0, lower_bound = 0;
3902         int total = nhgi->sum_norm_weight;
3903         u16 ecmp_size = nhgi->ecmp_size;
3904
3905         for (i = 0; i < nhgi->count; i++) {
3906                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3907                 int upper_bound;
3908
3909                 if (!nh->should_offload)
3910                         continue;
3911                 weight += nh->norm_nh_weight;
3912                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3913                 nh->num_adj_entries = upper_bound - lower_bound;
3914                 lower_bound = upper_bound;
3915         }
3916 }
3917
3918 static struct mlxsw_sp_nexthop *
3919 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3920                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3921
3922 static void
3923 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3924                                         struct mlxsw_sp_nexthop_group *nh_grp)
3925 {
3926         int i;
3927
3928         for (i = 0; i < nh_grp->nhgi->count; i++) {
3929                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3930
3931                 if (nh->offloaded)
3932                         nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3933                 else
3934                         nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3935         }
3936 }
3937
3938 static void
3939 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3940                                           struct mlxsw_sp_fib6_entry *fib6_entry)
3941 {
3942         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3943
3944         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3945                 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3946                 struct mlxsw_sp_nexthop *nh;
3947
3948                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3949                 if (nh && nh->offloaded)
3950                         fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3951                 else
3952                         fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3953         }
3954 }
3955
3956 static void
3957 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3958                                         struct mlxsw_sp_nexthop_group *nh_grp)
3959 {
3960         struct mlxsw_sp_fib6_entry *fib6_entry;
3961
3962         /* Unfortunately, in IPv6 the route and the nexthop are described by
3963          * the same struct, so we need to iterate over all the routes using the
3964          * nexthop group and set / clear the offload indication for them.
3965          */
3966         list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3967                             common.nexthop_group_node)
3968                 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3969 }
3970
3971 static void
3972 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3973                                         const struct mlxsw_sp_nexthop *nh,
3974                                         u16 bucket_index)
3975 {
3976         struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
3977         bool offload = false, trap = false;
3978
3979         if (nh->offloaded) {
3980                 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3981                         trap = true;
3982                 else
3983                         offload = true;
3984         }
3985         nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3986                                     bucket_index, offload, trap);
3987 }
3988
3989 static void
3990 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3991                                            struct mlxsw_sp_nexthop_group *nh_grp)
3992 {
3993         int i;
3994
3995         /* Do not update the flags if the nexthop group is being destroyed
3996          * since:
3997          * 1. The nexthop objects is being deleted, in which case the flags are
3998          * irrelevant.
3999          * 2. The nexthop group was replaced by a newer group, in which case
4000          * the flags of the nexthop object were already updated based on the
4001          * new group.
4002          */
4003         if (nh_grp->can_destroy)
4004                 return;
4005
4006         nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4007                              nh_grp->nhgi->adj_index_valid, false);
4008
4009         /* Update flags of individual nexthop buckets in case of a resilient
4010          * nexthop group.
4011          */
4012         if (!nh_grp->nhgi->is_resilient)
4013                 return;
4014
4015         for (i = 0; i < nh_grp->nhgi->count; i++) {
4016                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4017
4018                 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i);
4019         }
4020 }
4021
4022 static void
4023 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
4024                                        struct mlxsw_sp_nexthop_group *nh_grp)
4025 {
4026         switch (nh_grp->type) {
4027         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
4028                 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
4029                 break;
4030         case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
4031                 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
4032                 break;
4033         case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
4034                 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
4035                 break;
4036         }
4037 }
4038
4039 static int
4040 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
4041                                struct mlxsw_sp_nexthop_group *nh_grp)
4042 {
4043         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4044         u16 ecmp_size, old_ecmp_size;
4045         struct mlxsw_sp_nexthop *nh;
4046         bool offload_change = false;
4047         u32 adj_index;
4048         bool old_adj_index_valid;
4049         u32 old_adj_index;
4050         int i, err2, err;
4051
4052         if (!nhgi->gateway)
4053                 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4054
4055         for (i = 0; i < nhgi->count; i++) {
4056                 nh = &nhgi->nexthops[i];
4057
4058                 if (nh->should_offload != nh->offloaded) {
4059                         offload_change = true;
4060                         if (nh->should_offload)
4061                                 nh->update = 1;
4062                 }
4063         }
4064         if (!offload_change) {
4065                 /* Nothing was added or removed, so no need to reallocate. Just
4066                  * update MAC on existing adjacency indexes.
4067                  */
4068                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
4069                 if (err) {
4070                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4071                         goto set_trap;
4072                 }
4073                 /* Flags of individual nexthop buckets might need to be
4074                  * updated.
4075                  */
4076                 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4077                 return 0;
4078         }
4079         mlxsw_sp_nexthop_group_normalize(nhgi);
4080         if (!nhgi->sum_norm_weight) {
4081                 /* No neigh of this group is connected so we just set
4082                  * the trap and let everthing flow through kernel.
4083                  */
4084                 err = 0;
4085                 goto set_trap;
4086         }
4087
4088         ecmp_size = nhgi->sum_norm_weight;
4089         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
4090         if (err)
4091                 /* No valid allocation size available. */
4092                 goto set_trap;
4093
4094         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4095                                   ecmp_size, &adj_index);
4096         if (err) {
4097                 /* We ran out of KVD linear space, just set the
4098                  * trap and let everything flow through kernel.
4099                  */
4100                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
4101                 goto set_trap;
4102         }
4103         old_adj_index_valid = nhgi->adj_index_valid;
4104         old_adj_index = nhgi->adj_index;
4105         old_ecmp_size = nhgi->ecmp_size;
4106         nhgi->adj_index_valid = 1;
4107         nhgi->adj_index = adj_index;
4108         nhgi->ecmp_size = ecmp_size;
4109         mlxsw_sp_nexthop_group_rebalance(nhgi);
4110         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
4111         if (err) {
4112                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4113                 goto set_trap;
4114         }
4115
4116         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4117
4118         if (!old_adj_index_valid) {
4119                 /* The trap was set for fib entries, so we have to call
4120                  * fib entry update to unset it and use adjacency index.
4121                  */
4122                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4123                 if (err) {
4124                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
4125                         goto set_trap;
4126                 }
4127                 return 0;
4128         }
4129
4130         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
4131                                              old_adj_index, old_ecmp_size);
4132         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4133                            old_ecmp_size, old_adj_index);
4134         if (err) {
4135                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
4136                 goto set_trap;
4137         }
4138
4139         return 0;
4140
4141 set_trap:
4142         old_adj_index_valid = nhgi->adj_index_valid;
4143         nhgi->adj_index_valid = 0;
4144         for (i = 0; i < nhgi->count; i++) {
4145                 nh = &nhgi->nexthops[i];
4146                 nh->offloaded = 0;
4147         }
4148         err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4149         if (err2)
4150                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
4151         mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4152         if (old_adj_index_valid)
4153                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4154                                    nhgi->ecmp_size, nhgi->adj_index);
4155         return err;
4156 }
4157
4158 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
4159                                             bool removing)
4160 {
4161         if (!removing) {
4162                 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
4163                 nh->should_offload = 1;
4164         } else if (nh->nhgi->is_resilient) {
4165                 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
4166                 nh->should_offload = 1;
4167         } else {
4168                 nh->should_offload = 0;
4169         }
4170         nh->update = 1;
4171 }
4172
4173 static int
4174 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
4175                                     struct mlxsw_sp_neigh_entry *neigh_entry)
4176 {
4177         struct neighbour *n, *old_n = neigh_entry->key.n;
4178         struct mlxsw_sp_nexthop *nh;
4179         struct net_device *dev;
4180         bool entry_connected;
4181         u8 nud_state, dead;
4182         int err;
4183
4184         nh = list_first_entry(&neigh_entry->nexthop_list,
4185                               struct mlxsw_sp_nexthop, neigh_list_node);
4186         dev = mlxsw_sp_nexthop_dev(nh);
4187
4188         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev);
4189         if (!n) {
4190                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev);
4191                 if (IS_ERR(n))
4192                         return PTR_ERR(n);
4193                 neigh_event_send(n, NULL);
4194         }
4195
4196         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
4197         neigh_entry->key.n = n;
4198         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4199         if (err)
4200                 goto err_neigh_entry_insert;
4201
4202         read_lock_bh(&n->lock);
4203         nud_state = n->nud_state;
4204         dead = n->dead;
4205         read_unlock_bh(&n->lock);
4206         entry_connected = nud_state & NUD_VALID && !dead;
4207
4208         list_for_each_entry(nh, &neigh_entry->nexthop_list,
4209                             neigh_list_node) {
4210                 neigh_release(old_n);
4211                 neigh_clone(n);
4212                 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
4213                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4214         }
4215
4216         neigh_release(n);
4217
4218         return 0;
4219
4220 err_neigh_entry_insert:
4221         neigh_entry->key.n = old_n;
4222         mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4223         neigh_release(n);
4224         return err;
4225 }
4226
4227 static void
4228 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
4229                               struct mlxsw_sp_neigh_entry *neigh_entry,
4230                               bool removing, bool dead)
4231 {
4232         struct mlxsw_sp_nexthop *nh;
4233
4234         if (list_empty(&neigh_entry->nexthop_list))
4235                 return;
4236
4237         if (dead) {
4238                 int err;
4239
4240                 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
4241                                                           neigh_entry);
4242                 if (err)
4243                         dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
4244                 return;
4245         }
4246
4247         list_for_each_entry(nh, &neigh_entry->nexthop_list,
4248                             neigh_list_node) {
4249                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4250                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4251         }
4252 }
4253
4254 static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh,
4255                                        struct mlxsw_sp_crif *crif)
4256 {
4257         if (nh->crif)
4258                 return;
4259
4260         nh->crif = crif;
4261         list_add(&nh->crif_list_node, &crif->nexthop_list);
4262 }
4263
4264 static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh)
4265 {
4266         if (!nh->crif)
4267                 return;
4268
4269         list_del(&nh->crif_list_node);
4270         nh->crif = NULL;
4271 }
4272
4273 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
4274                                        struct mlxsw_sp_nexthop *nh)
4275 {
4276         struct mlxsw_sp_neigh_entry *neigh_entry;
4277         struct net_device *dev;
4278         struct neighbour *n;
4279         u8 nud_state, dead;
4280         int err;
4281
4282         if (WARN_ON(!nh->crif->rif))
4283                 return 0;
4284
4285         if (!nh->nhgi->gateway || nh->neigh_entry)
4286                 return 0;
4287         dev = mlxsw_sp_nexthop_dev(nh);
4288
4289         /* Take a reference of neigh here ensuring that neigh would
4290          * not be destructed before the nexthop entry is finished.
4291          * The reference is taken either in neigh_lookup() or
4292          * in neigh_create() in case n is not found.
4293          */
4294         n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev);
4295         if (!n) {
4296                 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev);
4297                 if (IS_ERR(n))
4298                         return PTR_ERR(n);
4299                 neigh_event_send(n, NULL);
4300         }
4301         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
4302         if (!neigh_entry) {
4303                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
4304                 if (IS_ERR(neigh_entry)) {
4305                         err = -EINVAL;
4306                         goto err_neigh_entry_create;
4307                 }
4308         }
4309
4310         /* If that is the first nexthop connected to that neigh, add to
4311          * nexthop_neighs_list
4312          */
4313         if (list_empty(&neigh_entry->nexthop_list))
4314                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
4315                               &mlxsw_sp->router->nexthop_neighs_list);
4316
4317         nh->neigh_entry = neigh_entry;
4318         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
4319         read_lock_bh(&n->lock);
4320         nud_state = n->nud_state;
4321         dead = n->dead;
4322         read_unlock_bh(&n->lock);
4323         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
4324
4325         return 0;
4326
4327 err_neigh_entry_create:
4328         neigh_release(n);
4329         return err;
4330 }
4331
4332 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4333                                         struct mlxsw_sp_nexthop *nh)
4334 {
4335         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4336         struct neighbour *n;
4337
4338         if (!neigh_entry)
4339                 return;
4340         n = neigh_entry->key.n;
4341
4342         __mlxsw_sp_nexthop_neigh_update(nh, true);
4343         list_del(&nh->neigh_list_node);
4344         nh->neigh_entry = NULL;
4345
4346         /* If that is the last nexthop connected to that neigh, remove from
4347          * nexthop_neighs_list
4348          */
4349         if (list_empty(&neigh_entry->nexthop_list))
4350                 list_del(&neigh_entry->nexthop_neighs_list_node);
4351
4352         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
4353                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4354
4355         neigh_release(n);
4356 }
4357
4358 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4359 {
4360         struct net_device *ul_dev;
4361         bool is_up;
4362
4363         rcu_read_lock();
4364         ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4365         is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4366         rcu_read_unlock();
4367
4368         return is_up;
4369 }
4370
4371 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4372                                        struct mlxsw_sp_nexthop *nh,
4373                                        struct mlxsw_sp_ipip_entry *ipip_entry)
4374 {
4375         struct mlxsw_sp_crif *crif;
4376         bool removing;
4377
4378         if (!nh->nhgi->gateway || nh->ipip_entry)
4379                 return;
4380
4381         crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev);
4382         if (WARN_ON(!crif))
4383                 return;
4384
4385         nh->ipip_entry = ipip_entry;
4386         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
4387         __mlxsw_sp_nexthop_neigh_update(nh, removing);
4388         mlxsw_sp_nexthop_crif_init(nh, crif);
4389 }
4390
4391 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4392                                        struct mlxsw_sp_nexthop *nh)
4393 {
4394         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4395
4396         if (!ipip_entry)
4397                 return;
4398
4399         __mlxsw_sp_nexthop_neigh_update(nh, true);
4400         nh->ipip_entry = NULL;
4401 }
4402
4403 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4404                                         const struct fib_nh *fib_nh,
4405                                         enum mlxsw_sp_ipip_type *p_ipipt)
4406 {
4407         struct net_device *dev = fib_nh->fib_nh_dev;
4408
4409         return dev &&
4410                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4411                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
4412 }
4413
4414 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4415                                       struct mlxsw_sp_nexthop *nh,
4416                                       const struct net_device *dev)
4417 {
4418         const struct mlxsw_sp_ipip_ops *ipip_ops;
4419         struct mlxsw_sp_ipip_entry *ipip_entry;
4420         struct mlxsw_sp_crif *crif;
4421         int err;
4422
4423         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4424         if (ipip_entry) {
4425                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4426                 if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4427                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4428                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4429                         return 0;
4430                 }
4431         }
4432
4433         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4434         crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev);
4435         if (!crif)
4436                 return 0;
4437
4438         mlxsw_sp_nexthop_crif_init(nh, crif);
4439
4440         if (!crif->rif)
4441                 return 0;
4442
4443         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4444         if (err)
4445                 goto err_neigh_init;
4446
4447         return 0;
4448
4449 err_neigh_init:
4450         mlxsw_sp_nexthop_crif_fini(nh);
4451         return err;
4452 }
4453
4454 static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp,
4455                                           struct mlxsw_sp_nexthop *nh)
4456 {
4457         switch (nh->type) {
4458         case MLXSW_SP_NEXTHOP_TYPE_ETH:
4459                 return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4460         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4461                 break;
4462         }
4463
4464         return 0;
4465 }
4466
4467 static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp,
4468                                            struct mlxsw_sp_nexthop *nh)
4469 {
4470         switch (nh->type) {
4471         case MLXSW_SP_NEXTHOP_TYPE_ETH:
4472                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4473                 break;
4474         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4475                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4476                 break;
4477         }
4478 }
4479
4480 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4481                                        struct mlxsw_sp_nexthop *nh)
4482 {
4483         mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4484         mlxsw_sp_nexthop_crif_fini(nh);
4485 }
4486
4487 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4488                                   struct mlxsw_sp_nexthop_group *nh_grp,
4489                                   struct mlxsw_sp_nexthop *nh,
4490                                   struct fib_nh *fib_nh)
4491 {
4492         struct net_device *dev = fib_nh->fib_nh_dev;
4493         struct in_device *in_dev;
4494         int err;
4495
4496         nh->nhgi = nh_grp->nhgi;
4497         nh->key.fib_nh = fib_nh;
4498 #ifdef CONFIG_IP_ROUTE_MULTIPATH
4499         nh->nh_weight = fib_nh->fib_nh_weight;
4500 #else
4501         nh->nh_weight = 1;
4502 #endif
4503         memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4504         nh->neigh_tbl = &arp_tbl;
4505         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4506         if (err)
4507                 return err;
4508
4509         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4510         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4511
4512         if (!dev)
4513                 return 0;
4514         nh->ifindex = dev->ifindex;
4515
4516         rcu_read_lock();
4517         in_dev = __in_dev_get_rcu(dev);
4518         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4519             fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4520                 rcu_read_unlock();
4521                 return 0;
4522         }
4523         rcu_read_unlock();
4524
4525         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4526         if (err)
4527                 goto err_nexthop_neigh_init;
4528
4529         return 0;
4530
4531 err_nexthop_neigh_init:
4532         list_del(&nh->router_list_node);
4533         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4534         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4535         return err;
4536 }
4537
4538 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4539                                    struct mlxsw_sp_nexthop *nh)
4540 {
4541         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4542         list_del(&nh->router_list_node);
4543         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4544         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4545 }
4546
4547 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4548                                     unsigned long event, struct fib_nh *fib_nh)
4549 {
4550         struct mlxsw_sp_nexthop_key key;
4551         struct mlxsw_sp_nexthop *nh;
4552
4553         key.fib_nh = fib_nh;
4554         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4555         if (!nh)
4556                 return;
4557
4558         switch (event) {
4559         case FIB_EVENT_NH_ADD:
4560                 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4561                 break;
4562         case FIB_EVENT_NH_DEL:
4563                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4564                 break;
4565         }
4566
4567         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4568 }
4569
4570 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4571                                         struct mlxsw_sp_rif *rif)
4572 {
4573         struct net_device *dev = mlxsw_sp_rif_dev(rif);
4574         struct mlxsw_sp_nexthop *nh;
4575         bool removing;
4576
4577         list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) {
4578                 switch (nh->type) {
4579                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4580                         removing = false;
4581                         break;
4582                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4583                         removing = !mlxsw_sp_ipip_netdev_ul_up(dev);
4584                         break;
4585                 default:
4586                         WARN_ON(1);
4587                         continue;
4588                 }
4589
4590                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4591                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4592         }
4593 }
4594
4595 static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
4596                                           struct mlxsw_sp_rif *rif)
4597 {
4598         struct mlxsw_sp_nexthop *nh, *tmp;
4599         unsigned int n = 0;
4600         int err;
4601
4602         list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4603                                  crif_list_node) {
4604                 err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh);
4605                 if (err)
4606                         goto err_nexthop_type_rif;
4607                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4608                 n++;
4609         }
4610
4611         return 0;
4612
4613 err_nexthop_type_rif:
4614         list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4615                                  crif_list_node) {
4616                 if (!n--)
4617                         break;
4618                 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4619                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4620         }
4621         return err;
4622 }
4623
4624 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4625                                            struct mlxsw_sp_rif *rif)
4626 {
4627         struct mlxsw_sp_nexthop *nh, *tmp;
4628
4629         list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4630                                  crif_list_node) {
4631                 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4632                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4633         }
4634 }
4635
4636 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp)
4637 {
4638         enum mlxsw_reg_ratr_trap_action trap_action;
4639         char ratr_pl[MLXSW_REG_RATR_LEN];
4640         int err;
4641
4642         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4643                                   &mlxsw_sp->router->adj_trap_index);
4644         if (err)
4645                 return err;
4646
4647         trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
4648         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4649                             MLXSW_REG_RATR_TYPE_ETHERNET,
4650                             mlxsw_sp->router->adj_trap_index,
4651                             mlxsw_sp->router->lb_crif->rif->rif_index);
4652         mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4653         mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
4654         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4655         if (err)
4656                 goto err_ratr_write;
4657
4658         return 0;
4659
4660 err_ratr_write:
4661         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4662                            mlxsw_sp->router->adj_trap_index);
4663         return err;
4664 }
4665
4666 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp)
4667 {
4668         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4669                            mlxsw_sp->router->adj_trap_index);
4670 }
4671
4672 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp)
4673 {
4674         int err;
4675
4676         if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups))
4677                 return 0;
4678
4679         err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp);
4680         if (err)
4681                 return err;
4682
4683         refcount_set(&mlxsw_sp->router->num_groups, 1);
4684
4685         return 0;
4686 }
4687
4688 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp)
4689 {
4690         if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups))
4691                 return;
4692
4693         mlxsw_sp_adj_trap_entry_fini(mlxsw_sp);
4694 }
4695
4696 static void
4697 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
4698                              const struct mlxsw_sp_nexthop_group *nh_grp,
4699                              unsigned long *activity)
4700 {
4701         char *ratrad_pl;
4702         int i, err;
4703
4704         ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
4705         if (!ratrad_pl)
4706                 return;
4707
4708         mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index,
4709                               nh_grp->nhgi->count);
4710         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl);
4711         if (err)
4712                 goto out;
4713
4714         for (i = 0; i < nh_grp->nhgi->count; i++) {
4715                 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i))
4716                         continue;
4717                 bitmap_set(activity, i, 1);
4718         }
4719
4720 out:
4721         kfree(ratrad_pl);
4722 }
4723
4724 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
4725
4726 static void
4727 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
4728                                 const struct mlxsw_sp_nexthop_group *nh_grp)
4729 {
4730         unsigned long *activity;
4731
4732         activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL);
4733         if (!activity)
4734                 return;
4735
4736         mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
4737         nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4738                                         nh_grp->nhgi->count, activity);
4739
4740         bitmap_free(activity);
4741 }
4742
4743 static void
4744 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
4745 {
4746         unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
4747
4748         mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw,
4749                                msecs_to_jiffies(interval));
4750 }
4751
4752 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
4753 {
4754         struct mlxsw_sp_nexthop_group_info *nhgi;
4755         struct mlxsw_sp_router *router;
4756         bool reschedule = false;
4757
4758         router = container_of(work, struct mlxsw_sp_router,
4759                               nh_grp_activity_dw.work);
4760
4761         mutex_lock(&router->lock);
4762
4763         list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
4764                 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp);
4765                 reschedule = true;
4766         }
4767
4768         mutex_unlock(&router->lock);
4769
4770         if (!reschedule)
4771                 return;
4772         mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp);
4773 }
4774
4775 static int
4776 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4777                                      const struct nh_notifier_single_info *nh,
4778                                      struct netlink_ext_ack *extack)
4779 {
4780         int err = -EINVAL;
4781
4782         if (nh->is_fdb)
4783                 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4784         else if (nh->has_encap)
4785                 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4786         else
4787                 err = 0;
4788
4789         return err;
4790 }
4791
4792 static int
4793 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4794                                           const struct nh_notifier_single_info *nh,
4795                                           struct netlink_ext_ack *extack)
4796 {
4797         int err;
4798
4799         err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4800         if (err)
4801                 return err;
4802
4803         /* Device only nexthops with an IPIP device are programmed as
4804          * encapsulating adjacency entries.
4805          */
4806         if (!nh->gw_family && !nh->is_reject &&
4807             !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
4808                 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4809                 return -EINVAL;
4810         }
4811
4812         return 0;
4813 }
4814
4815 static int
4816 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4817                                     const struct nh_notifier_grp_info *nh_grp,
4818                                     struct netlink_ext_ack *extack)
4819 {
4820         int i;
4821
4822         if (nh_grp->is_fdb) {
4823                 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4824                 return -EINVAL;
4825         }
4826
4827         for (i = 0; i < nh_grp->num_nh; i++) {
4828                 const struct nh_notifier_single_info *nh;
4829                 int err;
4830
4831                 nh = &nh_grp->nh_entries[i].nh;
4832                 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4833                                                                 extack);
4834                 if (err)
4835                         return err;
4836         }
4837
4838         return 0;
4839 }
4840
4841 static int
4842 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
4843                                              const struct nh_notifier_res_table_info *nh_res_table,
4844                                              struct netlink_ext_ack *extack)
4845 {
4846         unsigned int alloc_size;
4847         bool valid_size = false;
4848         int err, i;
4849
4850         if (nh_res_table->num_nh_buckets < 32) {
4851                 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
4852                 return -EINVAL;
4853         }
4854
4855         for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
4856                 const struct mlxsw_sp_adj_grp_size_range *size_range;
4857
4858                 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
4859
4860                 if (nh_res_table->num_nh_buckets >= size_range->start &&
4861                     nh_res_table->num_nh_buckets <= size_range->end) {
4862                         valid_size = true;
4863                         break;
4864                 }
4865         }
4866
4867         if (!valid_size) {
4868                 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
4869                 return -EINVAL;
4870         }
4871
4872         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
4873                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4874                                               nh_res_table->num_nh_buckets,
4875                                               &alloc_size);
4876         if (err || nh_res_table->num_nh_buckets != alloc_size) {
4877                 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
4878                 return -EINVAL;
4879         }
4880
4881         return 0;
4882 }
4883
4884 static int
4885 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
4886                                         const struct nh_notifier_res_table_info *nh_res_table,
4887                                         struct netlink_ext_ack *extack)
4888 {
4889         int err;
4890         u16 i;
4891
4892         err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
4893                                                            nh_res_table,
4894                                                            extack);
4895         if (err)
4896                 return err;
4897
4898         for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
4899                 const struct nh_notifier_single_info *nh;
4900                 int err;
4901
4902                 nh = &nh_res_table->nhs[i];
4903                 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4904                                                                 extack);
4905                 if (err)
4906                         return err;
4907         }
4908
4909         return 0;
4910 }
4911
4912 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4913                                          unsigned long event,
4914                                          struct nh_notifier_info *info)
4915 {
4916         struct nh_notifier_single_info *nh;
4917
4918         if (event != NEXTHOP_EVENT_REPLACE &&
4919             event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
4920             event != NEXTHOP_EVENT_BUCKET_REPLACE)
4921                 return 0;
4922
4923         switch (info->type) {
4924         case NH_NOTIFIER_INFO_TYPE_SINGLE:
4925                 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
4926                                                             info->extack);
4927         case NH_NOTIFIER_INFO_TYPE_GRP:
4928                 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4929                                                            info->nh_grp,
4930                                                            info->extack);
4931         case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4932                 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
4933                                                                info->nh_res_table,
4934                                                                info->extack);
4935         case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
4936                 nh = &info->nh_res_bucket->new_nh;
4937                 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4938                                                                  info->extack);
4939         default:
4940                 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4941                 return -EOPNOTSUPP;
4942         }
4943 }
4944
4945 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4946                                             const struct nh_notifier_info *info)
4947 {
4948         const struct net_device *dev;
4949
4950         switch (info->type) {
4951         case NH_NOTIFIER_INFO_TYPE_SINGLE:
4952                 dev = info->nh->dev;
4953                 return info->nh->gw_family || info->nh->is_reject ||
4954                        mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
4955         case NH_NOTIFIER_INFO_TYPE_GRP:
4956         case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4957                 /* Already validated earlier. */
4958                 return true;
4959         default:
4960                 return false;
4961         }
4962 }
4963
4964 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
4965                                                 struct mlxsw_sp_nexthop *nh)
4966 {
4967         nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
4968         nh->should_offload = 1;
4969         /* While nexthops that discard packets do not forward packets
4970          * via an egress RIF, they still need to be programmed using a
4971          * valid RIF, so use the loopback RIF created during init.
4972          */
4973         nh->crif = mlxsw_sp->router->lb_crif;
4974 }
4975
4976 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
4977                                                 struct mlxsw_sp_nexthop *nh)
4978 {
4979         nh->crif = NULL;
4980         nh->should_offload = 0;
4981 }
4982
4983 static int
4984 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
4985                           struct mlxsw_sp_nexthop_group *nh_grp,
4986                           struct mlxsw_sp_nexthop *nh,
4987                           struct nh_notifier_single_info *nh_obj, int weight)
4988 {
4989         struct net_device *dev = nh_obj->dev;
4990         int err;
4991
4992         nh->nhgi = nh_grp->nhgi;
4993         nh->nh_weight = weight;
4994
4995         switch (nh_obj->gw_family) {
4996         case AF_INET:
4997                 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
4998                 nh->neigh_tbl = &arp_tbl;
4999                 break;
5000         case AF_INET6:
5001                 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
5002 #if IS_ENABLED(CONFIG_IPV6)
5003                 nh->neigh_tbl = &nd_tbl;
5004 #endif
5005                 break;
5006         }
5007
5008         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5009         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5010         nh->ifindex = dev->ifindex;
5011
5012         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5013         if (err)
5014                 goto err_type_init;
5015
5016         if (nh_obj->is_reject)
5017                 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
5018
5019         /* In a resilient nexthop group, all the nexthops must be written to
5020          * the adjacency table. Even if they do not have a valid neighbour or
5021          * RIF.
5022          */
5023         if (nh_grp->nhgi->is_resilient && !nh->should_offload) {
5024                 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
5025                 nh->should_offload = 1;
5026         }
5027
5028         return 0;
5029
5030 err_type_init:
5031         list_del(&nh->router_list_node);
5032         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5033         return err;
5034 }
5035
5036 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
5037                                       struct mlxsw_sp_nexthop *nh)
5038 {
5039         if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
5040                 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
5041         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5042         list_del(&nh->router_list_node);
5043         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5044         nh->should_offload = 0;
5045 }
5046
5047 static int
5048 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
5049                                      struct mlxsw_sp_nexthop_group *nh_grp,
5050                                      struct nh_notifier_info *info)
5051 {
5052         struct mlxsw_sp_nexthop_group_info *nhgi;
5053         struct mlxsw_sp_nexthop *nh;
5054         bool is_resilient = false;
5055         unsigned int nhs;
5056         int err, i;
5057
5058         switch (info->type) {
5059         case NH_NOTIFIER_INFO_TYPE_SINGLE:
5060                 nhs = 1;
5061                 break;
5062         case NH_NOTIFIER_INFO_TYPE_GRP:
5063                 nhs = info->nh_grp->num_nh;
5064                 break;
5065         case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5066                 nhs = info->nh_res_table->num_nh_buckets;
5067                 is_resilient = true;
5068                 break;
5069         default:
5070                 return -EINVAL;
5071         }
5072
5073         nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5074         if (!nhgi)
5075                 return -ENOMEM;
5076         nh_grp->nhgi = nhgi;
5077         nhgi->nh_grp = nh_grp;
5078         nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
5079         nhgi->is_resilient = is_resilient;
5080         nhgi->count = nhs;
5081         for (i = 0; i < nhgi->count; i++) {
5082                 struct nh_notifier_single_info *nh_obj;
5083                 int weight;
5084
5085                 nh = &nhgi->nexthops[i];
5086                 switch (info->type) {
5087                 case NH_NOTIFIER_INFO_TYPE_SINGLE:
5088                         nh_obj = info->nh;
5089                         weight = 1;
5090                         break;
5091                 case NH_NOTIFIER_INFO_TYPE_GRP:
5092                         nh_obj = &info->nh_grp->nh_entries[i].nh;
5093                         weight = info->nh_grp->nh_entries[i].weight;
5094                         break;
5095                 case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5096                         nh_obj = &info->nh_res_table->nhs[i];
5097                         weight = 1;
5098                         break;
5099                 default:
5100                         err = -EINVAL;
5101                         goto err_nexthop_obj_init;
5102                 }
5103                 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
5104                                                 weight);
5105                 if (err)
5106                         goto err_nexthop_obj_init;
5107         }
5108         err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5109         if (err)
5110                 goto err_group_inc;
5111         err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5112         if (err) {
5113                 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
5114                 goto err_group_refresh;
5115         }
5116
5117         /* Add resilient nexthop groups to a list so that the activity of their
5118          * nexthop buckets will be periodically queried and cleared.
5119          */
5120         if (nhgi->is_resilient) {
5121                 if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
5122                         mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp);
5123                 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list);
5124         }
5125
5126         return 0;
5127
5128 err_group_refresh:
5129         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5130 err_group_inc:
5131         i = nhgi->count;
5132 err_nexthop_obj_init:
5133         for (i--; i >= 0; i--) {
5134                 nh = &nhgi->nexthops[i];
5135                 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5136         }
5137         kfree(nhgi);
5138         return err;
5139 }
5140
5141 static void
5142 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5143                                      struct mlxsw_sp_nexthop_group *nh_grp)
5144 {
5145         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5146         struct mlxsw_sp_router *router = mlxsw_sp->router;
5147         int i;
5148
5149         if (nhgi->is_resilient) {
5150                 list_del(&nhgi->list);
5151                 if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
5152                         cancel_delayed_work(&router->nh_grp_activity_dw);
5153         }
5154
5155         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5156         for (i = nhgi->count - 1; i >= 0; i--) {
5157                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5158
5159                 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5160         }
5161         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5162         WARN_ON_ONCE(nhgi->adj_index_valid);
5163         kfree(nhgi);
5164 }
5165
5166 static struct mlxsw_sp_nexthop_group *
5167 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
5168                                   struct nh_notifier_info *info)
5169 {
5170         struct mlxsw_sp_nexthop_group *nh_grp;
5171         int err;
5172
5173         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5174         if (!nh_grp)
5175                 return ERR_PTR(-ENOMEM);
5176         INIT_LIST_HEAD(&nh_grp->vr_list);
5177         err = rhashtable_init(&nh_grp->vr_ht,
5178                               &mlxsw_sp_nexthop_group_vr_ht_params);
5179         if (err)
5180                 goto err_nexthop_group_vr_ht_init;
5181         INIT_LIST_HEAD(&nh_grp->fib_list);
5182         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5183         nh_grp->obj.id = info->id;
5184
5185         err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
5186         if (err)
5187                 goto err_nexthop_group_info_init;
5188
5189         nh_grp->can_destroy = false;
5190
5191         return nh_grp;
5192
5193 err_nexthop_group_info_init:
5194         rhashtable_destroy(&nh_grp->vr_ht);
5195 err_nexthop_group_vr_ht_init:
5196         kfree(nh_grp);
5197         return ERR_PTR(err);
5198 }
5199
5200 static void
5201 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
5202                                    struct mlxsw_sp_nexthop_group *nh_grp)
5203 {
5204         if (!nh_grp->can_destroy)
5205                 return;
5206         mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
5207         WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
5208         WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5209         rhashtable_destroy(&nh_grp->vr_ht);
5210         kfree(nh_grp);
5211 }
5212
5213 static struct mlxsw_sp_nexthop_group *
5214 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
5215 {
5216         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
5217
5218         cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5219         cmp_arg.id = id;
5220         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
5221                                       &cmp_arg,
5222                                       mlxsw_sp_nexthop_group_ht_params);
5223 }
5224
5225 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
5226                                           struct mlxsw_sp_nexthop_group *nh_grp)
5227 {
5228         return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5229 }
5230
5231 static int
5232 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
5233                                    struct mlxsw_sp_nexthop_group *nh_grp,
5234                                    struct mlxsw_sp_nexthop_group *old_nh_grp,
5235                                    struct netlink_ext_ack *extack)
5236 {
5237         struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
5238         struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
5239         int err;
5240
5241         old_nh_grp->nhgi = new_nhgi;
5242         new_nhgi->nh_grp = old_nh_grp;
5243         nh_grp->nhgi = old_nhgi;
5244         old_nhgi->nh_grp = nh_grp;
5245
5246         if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5247                 /* Both the old adjacency index and the new one are valid.
5248                  * Routes are currently using the old one. Tell the device to
5249                  * replace the old adjacency index with the new one.
5250                  */
5251                 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp,
5252                                                      old_nhgi->adj_index,
5253                                                      old_nhgi->ecmp_size);
5254                 if (err) {
5255                         NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
5256                         goto err_out;
5257                 }
5258         } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
5259                 /* The old adjacency index is valid, while the new one is not.
5260                  * Iterate over all the routes using the group and change them
5261                  * to trap packets to the CPU.
5262                  */
5263                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5264                 if (err) {
5265                         NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
5266                         goto err_out;
5267                 }
5268         } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5269                 /* The old adjacency index is invalid, while the new one is.
5270                  * Iterate over all the routes using the group and change them
5271                  * to forward packets using the new valid index.
5272                  */
5273                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5274                 if (err) {
5275                         NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
5276                         goto err_out;
5277                 }
5278         }
5279
5280         /* Make sure the flags are set / cleared based on the new nexthop group
5281          * information.
5282          */
5283         mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp);
5284
5285         /* At this point 'nh_grp' is just a shell that is not used by anyone
5286          * and its nexthop group info is the old info that was just replaced
5287          * with the new one. Remove it.
5288          */
5289         nh_grp->can_destroy = true;
5290         mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5291
5292         return 0;
5293
5294 err_out:
5295         old_nhgi->nh_grp = old_nh_grp;
5296         nh_grp->nhgi = new_nhgi;
5297         new_nhgi->nh_grp = nh_grp;
5298         old_nh_grp->nhgi = old_nhgi;
5299         return err;
5300 }
5301
5302 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
5303                                     struct nh_notifier_info *info)
5304 {
5305         struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
5306         struct netlink_ext_ack *extack = info->extack;
5307         int err;
5308
5309         nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
5310         if (IS_ERR(nh_grp))
5311                 return PTR_ERR(nh_grp);
5312
5313         old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5314         if (!old_nh_grp)
5315                 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
5316         else
5317                 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
5318                                                          old_nh_grp, extack);
5319
5320         if (err) {
5321                 nh_grp->can_destroy = true;
5322                 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5323         }
5324
5325         return err;
5326 }
5327
5328 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
5329                                      struct nh_notifier_info *info)
5330 {
5331         struct mlxsw_sp_nexthop_group *nh_grp;
5332
5333         nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5334         if (!nh_grp)
5335                 return;
5336
5337         nh_grp->can_destroy = true;
5338         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5339
5340         /* If the group still has routes using it, then defer the delete
5341          * operation until the last route using it is deleted.
5342          */
5343         if (!list_empty(&nh_grp->fib_list))
5344                 return;
5345         mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5346 }
5347
5348 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp,
5349                                              u32 adj_index, char *ratr_pl)
5350 {
5351         MLXSW_REG_ZERO(ratr, ratr_pl);
5352         mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5353         mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index);
5354         mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16);
5355
5356         return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
5357 }
5358
5359 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new)
5360 {
5361         /* Clear the opcode and activity on both the old and new payload as
5362          * they are irrelevant for the comparison.
5363          */
5364         mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5365         mlxsw_reg_ratr_a_set(ratr_pl, 0);
5366         mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ);
5367         mlxsw_reg_ratr_a_set(ratr_pl_new, 0);
5368
5369         /* If the contents of the adjacency entry are consistent with the
5370          * replacement request, then replacement was successful.
5371          */
5372         if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN))
5373                 return 0;
5374
5375         return -EINVAL;
5376 }
5377
5378 static int
5379 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp,
5380                                        struct mlxsw_sp_nexthop *nh,
5381                                        struct nh_notifier_info *info)
5382 {
5383         u16 bucket_index = info->nh_res_bucket->bucket_index;
5384         struct netlink_ext_ack *extack = info->extack;
5385         bool force = info->nh_res_bucket->force;
5386         char ratr_pl_new[MLXSW_REG_RATR_LEN];
5387         char ratr_pl[MLXSW_REG_RATR_LEN];
5388         u32 adj_index;
5389         int err;
5390
5391         /* No point in trying an atomic replacement if the idle timer interval
5392          * is smaller than the interval in which we query and clear activity.
5393          */
5394         if (!force && info->nh_res_bucket->idle_timer_ms <
5395             MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL)
5396                 force = true;
5397
5398         adj_index = nh->nhgi->adj_index + bucket_index;
5399         err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl);
5400         if (err) {
5401                 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket");
5402                 return err;
5403         }
5404
5405         if (!force) {
5406                 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index,
5407                                                         ratr_pl_new);
5408                 if (err) {
5409                         NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent");
5410                         return err;
5411                 }
5412
5413                 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new);
5414                 if (err) {
5415                         NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement");
5416                         return err;
5417                 }
5418         }
5419
5420         nh->update = 0;
5421         nh->offloaded = 1;
5422         mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index);
5423
5424         return 0;
5425 }
5426
5427 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp,
5428                                                struct nh_notifier_info *info)
5429 {
5430         u16 bucket_index = info->nh_res_bucket->bucket_index;
5431         struct netlink_ext_ack *extack = info->extack;
5432         struct mlxsw_sp_nexthop_group_info *nhgi;
5433         struct nh_notifier_single_info *nh_obj;
5434         struct mlxsw_sp_nexthop_group *nh_grp;
5435         struct mlxsw_sp_nexthop *nh;
5436         int err;
5437
5438         nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5439         if (!nh_grp) {
5440                 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found");
5441                 return -EINVAL;
5442         }
5443
5444         nhgi = nh_grp->nhgi;
5445
5446         if (bucket_index >= nhgi->count) {
5447                 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range");
5448                 return -EINVAL;
5449         }
5450
5451         nh = &nhgi->nexthops[bucket_index];
5452         mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5453
5454         nh_obj = &info->nh_res_bucket->new_nh;
5455         err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5456         if (err) {
5457                 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement");
5458                 goto err_nexthop_obj_init;
5459         }
5460
5461         err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info);
5462         if (err)
5463                 goto err_nexthop_obj_bucket_adj_update;
5464
5465         return 0;
5466
5467 err_nexthop_obj_bucket_adj_update:
5468         mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5469 err_nexthop_obj_init:
5470         nh_obj = &info->nh_res_bucket->old_nh;
5471         mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5472         /* The old adjacency entry was not overwritten */
5473         nh->update = 0;
5474         nh->offloaded = 1;
5475         return err;
5476 }
5477
5478 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
5479                                       unsigned long event, void *ptr)
5480 {
5481         struct nh_notifier_info *info = ptr;
5482         struct mlxsw_sp_router *router;
5483         int err = 0;
5484
5485         router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
5486         err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info);
5487         if (err)
5488                 goto out;
5489
5490         mutex_lock(&router->lock);
5491
5492         switch (event) {
5493         case NEXTHOP_EVENT_REPLACE:
5494                 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info);
5495                 break;
5496         case NEXTHOP_EVENT_DEL:
5497                 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info);
5498                 break;
5499         case NEXTHOP_EVENT_BUCKET_REPLACE:
5500                 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp,
5501                                                           info);
5502                 break;
5503         default:
5504                 break;
5505         }
5506
5507         mutex_unlock(&router->lock);
5508
5509 out:
5510         return notifier_from_errno(err);
5511 }
5512
5513 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5514                                    struct fib_info *fi)
5515 {
5516         const struct fib_nh *nh = fib_info_nh(fi, 0);
5517
5518         return nh->fib_nh_gw_family ||
5519                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
5520 }
5521
5522 static int
5523 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
5524                                   struct mlxsw_sp_nexthop_group *nh_grp)
5525 {
5526         unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
5527         struct mlxsw_sp_nexthop_group_info *nhgi;
5528         struct mlxsw_sp_nexthop *nh;
5529         int err, i;
5530
5531         nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5532         if (!nhgi)
5533                 return -ENOMEM;
5534         nh_grp->nhgi = nhgi;
5535         nhgi->nh_grp = nh_grp;
5536         nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
5537         nhgi->count = nhs;
5538         for (i = 0; i < nhgi->count; i++) {
5539                 struct fib_nh *fib_nh;
5540
5541                 nh = &nhgi->nexthops[i];
5542                 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
5543                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
5544                 if (err)
5545                         goto err_nexthop4_init;
5546         }
5547         err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5548         if (err)
5549                 goto err_group_inc;
5550         err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5551         if (err)
5552                 goto err_group_refresh;
5553
5554         return 0;
5555
5556 err_group_refresh:
5557         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5558 err_group_inc:
5559         i = nhgi->count;
5560 err_nexthop4_init:
5561         for (i--; i >= 0; i--) {
5562                 nh = &nhgi->nexthops[i];
5563                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5564         }
5565         kfree(nhgi);
5566         return err;
5567 }
5568
5569 static void
5570 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5571                                   struct mlxsw_sp_nexthop_group *nh_grp)
5572 {
5573         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5574         int i;
5575
5576         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5577         for (i = nhgi->count - 1; i >= 0; i--) {
5578                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5579
5580                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5581         }
5582         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5583         WARN_ON_ONCE(nhgi->adj_index_valid);
5584         kfree(nhgi);
5585 }
5586
5587 static struct mlxsw_sp_nexthop_group *
5588 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
5589 {
5590         struct mlxsw_sp_nexthop_group *nh_grp;
5591         int err;
5592
5593         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5594         if (!nh_grp)
5595                 return ERR_PTR(-ENOMEM);
5596         INIT_LIST_HEAD(&nh_grp->vr_list);
5597         err = rhashtable_init(&nh_grp->vr_ht,
5598                               &mlxsw_sp_nexthop_group_vr_ht_params);
5599         if (err)
5600                 goto err_nexthop_group_vr_ht_init;
5601         INIT_LIST_HEAD(&nh_grp->fib_list);
5602         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
5603         nh_grp->ipv4.fi = fi;
5604         fib_info_hold(fi);
5605
5606         err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
5607         if (err)
5608                 goto err_nexthop_group_info_init;
5609
5610         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5611         if (err)
5612                 goto err_nexthop_group_insert;
5613
5614         nh_grp->can_destroy = true;
5615
5616         return nh_grp;
5617
5618 err_nexthop_group_insert:
5619         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5620 err_nexthop_group_info_init:
5621         fib_info_put(fi);
5622         rhashtable_destroy(&nh_grp->vr_ht);
5623 err_nexthop_group_vr_ht_init:
5624         kfree(nh_grp);
5625         return ERR_PTR(err);
5626 }
5627
5628 static void
5629 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
5630                                 struct mlxsw_sp_nexthop_group *nh_grp)
5631 {
5632         if (!nh_grp->can_destroy)
5633                 return;
5634         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5635         mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5636         fib_info_put(nh_grp->ipv4.fi);
5637         WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5638         rhashtable_destroy(&nh_grp->vr_ht);
5639         kfree(nh_grp);
5640 }
5641
5642 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
5643                                        struct mlxsw_sp_fib_entry *fib_entry,
5644                                        struct fib_info *fi)
5645 {
5646         struct mlxsw_sp_nexthop_group *nh_grp;
5647
5648         if (fi->nh) {
5649                 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
5650                                                            fi->nh->id);
5651                 if (WARN_ON_ONCE(!nh_grp))
5652                         return -EINVAL;
5653                 goto out;
5654         }
5655
5656         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
5657         if (!nh_grp) {
5658                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
5659                 if (IS_ERR(nh_grp))
5660                         return PTR_ERR(nh_grp);
5661         }
5662 out:
5663         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
5664         fib_entry->nh_group = nh_grp;
5665         return 0;
5666 }
5667
5668 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
5669                                         struct mlxsw_sp_fib_entry *fib_entry)
5670 {
5671         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5672
5673         list_del(&fib_entry->nexthop_group_node);
5674         if (!list_empty(&nh_grp->fib_list))
5675                 return;
5676
5677         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
5678                 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5679                 return;
5680         }
5681
5682         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
5683 }
5684
5685 static bool
5686 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5687 {
5688         struct mlxsw_sp_fib4_entry *fib4_entry;
5689
5690         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5691                                   common);
5692         return !fib4_entry->dscp;
5693 }
5694
5695 static bool
5696 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5697 {
5698         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5699
5700         switch (fib_entry->fib_node->fib->proto) {
5701         case MLXSW_SP_L3_PROTO_IPV4:
5702                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
5703                         return false;
5704                 break;
5705         case MLXSW_SP_L3_PROTO_IPV6:
5706                 break;
5707         }
5708
5709         switch (fib_entry->type) {
5710         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5711                 return !!nh_group->nhgi->adj_index_valid;
5712         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5713                 return !!mlxsw_sp_nhgi_rif(nh_group->nhgi);
5714         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5715         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5716         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5717                 return true;
5718         default:
5719                 return false;
5720         }
5721 }
5722
5723 static struct mlxsw_sp_nexthop *
5724 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
5725                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5726 {
5727         int i;
5728
5729         for (i = 0; i < nh_grp->nhgi->count; i++) {
5730                 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
5731                 struct net_device *dev = mlxsw_sp_nexthop_dev(nh);
5732                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5733
5734                 if (dev && dev == rt->fib6_nh->fib_nh_dev &&
5735                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
5736                                     &rt->fib6_nh->fib_nh_gw6))
5737                         return nh;
5738         }
5739
5740         return NULL;
5741 }
5742
5743 static void
5744 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5745                                       struct fib_entry_notifier_info *fen_info)
5746 {
5747         u32 *p_dst = (u32 *) &fen_info->dst;
5748         struct fib_rt_info fri;
5749
5750         fri.fi = fen_info->fi;
5751         fri.tb_id = fen_info->tb_id;
5752         fri.dst = cpu_to_be32(*p_dst);
5753         fri.dst_len = fen_info->dst_len;
5754         fri.dscp = fen_info->dscp;
5755         fri.type = fen_info->type;
5756         fri.offload = false;
5757         fri.trap = false;
5758         fri.offload_failed = true;
5759         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5760 }
5761
5762 static void
5763 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5764                                  struct mlxsw_sp_fib_entry *fib_entry)
5765 {
5766         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5767         int dst_len = fib_entry->fib_node->key.prefix_len;
5768         struct mlxsw_sp_fib4_entry *fib4_entry;
5769         struct fib_rt_info fri;
5770         bool should_offload;
5771
5772         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5773         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5774                                   common);
5775         fri.fi = fib4_entry->fi;
5776         fri.tb_id = fib4_entry->tb_id;
5777         fri.dst = cpu_to_be32(*p_dst);
5778         fri.dst_len = dst_len;
5779         fri.dscp = fib4_entry->dscp;
5780         fri.type = fib4_entry->type;
5781         fri.offload = should_offload;
5782         fri.trap = !should_offload;
5783         fri.offload_failed = false;
5784         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5785 }
5786
5787 static void
5788 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5789                                    struct mlxsw_sp_fib_entry *fib_entry)
5790 {
5791         u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5792         int dst_len = fib_entry->fib_node->key.prefix_len;
5793         struct mlxsw_sp_fib4_entry *fib4_entry;
5794         struct fib_rt_info fri;
5795
5796         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5797                                   common);
5798         fri.fi = fib4_entry->fi;
5799         fri.tb_id = fib4_entry->tb_id;
5800         fri.dst = cpu_to_be32(*p_dst);
5801         fri.dst_len = dst_len;
5802         fri.dscp = fib4_entry->dscp;
5803         fri.type = fib4_entry->type;
5804         fri.offload = false;
5805         fri.trap = false;
5806         fri.offload_failed = false;
5807         fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5808 }
5809
5810 #if IS_ENABLED(CONFIG_IPV6)
5811 static void
5812 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5813                                       struct fib6_info **rt_arr,
5814                                       unsigned int nrt6)
5815 {
5816         int i;
5817
5818         /* In IPv6 a multipath route is represented using multiple routes, so
5819          * we need to set the flags on all of them.
5820          */
5821         for (i = 0; i < nrt6; i++)
5822                 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i],
5823                                        false, false, true);
5824 }
5825 #else
5826 static void
5827 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5828                                       struct fib6_info **rt_arr,
5829                                       unsigned int nrt6)
5830 {
5831 }
5832 #endif
5833
5834 #if IS_ENABLED(CONFIG_IPV6)
5835 static void
5836 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5837                                  struct mlxsw_sp_fib_entry *fib_entry)
5838 {
5839         struct mlxsw_sp_fib6_entry *fib6_entry;
5840         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5841         bool should_offload;
5842
5843         should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5844
5845         /* In IPv6 a multipath route is represented using multiple routes, so
5846          * we need to set the flags on all of them.
5847          */
5848         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5849                                   common);
5850         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5851                 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5852                                        should_offload, !should_offload, false);
5853 }
5854 #else
5855 static void
5856 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5857                                  struct mlxsw_sp_fib_entry *fib_entry)
5858 {
5859 }
5860 #endif
5861
5862 #if IS_ENABLED(CONFIG_IPV6)
5863 static void
5864 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5865                                    struct mlxsw_sp_fib_entry *fib_entry)
5866 {
5867         struct mlxsw_sp_fib6_entry *fib6_entry;
5868         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5869
5870         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5871                                   common);
5872         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5873                 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5874                                        false, false, false);
5875 }
5876 #else
5877 static void
5878 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5879                                    struct mlxsw_sp_fib_entry *fib_entry)
5880 {
5881 }
5882 #endif
5883
5884 static void
5885 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5886                                 struct mlxsw_sp_fib_entry *fib_entry)
5887 {
5888         switch (fib_entry->fib_node->fib->proto) {
5889         case MLXSW_SP_L3_PROTO_IPV4:
5890                 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
5891                 break;
5892         case MLXSW_SP_L3_PROTO_IPV6:
5893                 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
5894                 break;
5895         }
5896 }
5897
5898 static void
5899 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5900                                   struct mlxsw_sp_fib_entry *fib_entry)
5901 {
5902         switch (fib_entry->fib_node->fib->proto) {
5903         case MLXSW_SP_L3_PROTO_IPV4:
5904                 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5905                 break;
5906         case MLXSW_SP_L3_PROTO_IPV6:
5907                 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5908                 break;
5909         }
5910 }
5911
5912 static void
5913 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
5914                                     struct mlxsw_sp_fib_entry *fib_entry,
5915                                     enum mlxsw_reg_ralue_op op)
5916 {
5917         switch (op) {
5918         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
5919                 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
5920                 break;
5921         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
5922                 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5923                 break;
5924         default:
5925                 break;
5926         }
5927 }
5928
5929 static void
5930 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
5931                               const struct mlxsw_sp_fib_entry *fib_entry,
5932                               enum mlxsw_reg_ralue_op op)
5933 {
5934         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
5935         enum mlxsw_reg_ralxx_protocol proto;
5936         u32 *p_dip;
5937
5938         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
5939
5940         switch (fib->proto) {
5941         case MLXSW_SP_L3_PROTO_IPV4:
5942                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
5943                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
5944                                       fib_entry->fib_node->key.prefix_len,
5945                                       *p_dip);
5946                 break;
5947         case MLXSW_SP_L3_PROTO_IPV6:
5948                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
5949                                       fib_entry->fib_node->key.prefix_len,
5950                                       fib_entry->fib_node->key.addr);
5951                 break;
5952         }
5953 }
5954
5955 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
5956                                         struct mlxsw_sp_fib_entry *fib_entry,
5957                                         enum mlxsw_reg_ralue_op op)
5958 {
5959         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5960         struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
5961         char ralue_pl[MLXSW_REG_RALUE_LEN];
5962         enum mlxsw_reg_ralue_trap_action trap_action;
5963         u16 trap_id = 0;
5964         u32 adjacency_index = 0;
5965         u16 ecmp_size = 0;
5966
5967         /* In case the nexthop group adjacency index is valid, use it
5968          * with provided ECMP size. Otherwise, setup trap and pass
5969          * traffic to kernel.
5970          */
5971         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5972                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5973                 adjacency_index = nhgi->adj_index;
5974                 ecmp_size = nhgi->ecmp_size;
5975         } else if (!nhgi->adj_index_valid && nhgi->count &&
5976                    mlxsw_sp_nhgi_rif(nhgi)) {
5977                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5978                 adjacency_index = mlxsw_sp->router->adj_trap_index;
5979                 ecmp_size = 1;
5980         } else {
5981                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5982                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5983         }
5984
5985         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5986         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
5987                                         adjacency_index, ecmp_size);
5988         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5989 }
5990
5991 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
5992                                        struct mlxsw_sp_fib_entry *fib_entry,
5993                                        enum mlxsw_reg_ralue_op op)
5994 {
5995         struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi);
5996         enum mlxsw_reg_ralue_trap_action trap_action;
5997         char ralue_pl[MLXSW_REG_RALUE_LEN];
5998         u16 trap_id = 0;
5999         u16 rif_index = 0;
6000
6001         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
6002                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
6003                 rif_index = rif->rif_index;
6004         } else {
6005                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6006                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
6007         }
6008
6009         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6010         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
6011                                        rif_index);
6012         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6013 }
6014
6015 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
6016                                       struct mlxsw_sp_fib_entry *fib_entry,
6017                                       enum mlxsw_reg_ralue_op op)
6018 {
6019         char ralue_pl[MLXSW_REG_RALUE_LEN];
6020
6021         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6022         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
6023         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6024 }
6025
6026 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
6027                                            struct mlxsw_sp_fib_entry *fib_entry,
6028                                            enum mlxsw_reg_ralue_op op)
6029 {
6030         enum mlxsw_reg_ralue_trap_action trap_action;
6031         char ralue_pl[MLXSW_REG_RALUE_LEN];
6032
6033         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
6034         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6035         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0);
6036         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6037 }
6038
6039 static int
6040 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
6041                                   struct mlxsw_sp_fib_entry *fib_entry,
6042                                   enum mlxsw_reg_ralue_op op)
6043 {
6044         enum mlxsw_reg_ralue_trap_action trap_action;
6045         char ralue_pl[MLXSW_REG_RALUE_LEN];
6046         u16 trap_id;
6047
6048         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6049         trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
6050
6051         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6052         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0);
6053         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6054 }
6055
6056 static int
6057 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
6058                                  struct mlxsw_sp_fib_entry *fib_entry,
6059                                  enum mlxsw_reg_ralue_op op)
6060 {
6061         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
6062         const struct mlxsw_sp_ipip_ops *ipip_ops;
6063         char ralue_pl[MLXSW_REG_RALUE_LEN];
6064         int err;
6065
6066         if (WARN_ON(!ipip_entry))
6067                 return -EINVAL;
6068
6069         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
6070         err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
6071                                      fib_entry->decap.tunnel_index);
6072         if (err)
6073                 return err;
6074
6075         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6076         mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
6077                                            fib_entry->decap.tunnel_index);
6078         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6079 }
6080
6081 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
6082                                            struct mlxsw_sp_fib_entry *fib_entry,
6083                                            enum mlxsw_reg_ralue_op op)
6084 {
6085         char ralue_pl[MLXSW_REG_RALUE_LEN];
6086
6087         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6088         mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
6089                                            fib_entry->decap.tunnel_index);
6090         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6091 }
6092
6093 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6094                                    struct mlxsw_sp_fib_entry *fib_entry,
6095                                    enum mlxsw_reg_ralue_op op)
6096 {
6097         switch (fib_entry->type) {
6098         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
6099                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
6100         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
6101                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
6102         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
6103                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
6104         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
6105                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op);
6106         case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
6107                 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry,
6108                                                          op);
6109         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6110                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
6111                                                         fib_entry, op);
6112         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
6113                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op);
6114         }
6115         return -EINVAL;
6116 }
6117
6118 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6119                                  struct mlxsw_sp_fib_entry *fib_entry,
6120                                  enum mlxsw_reg_ralue_op op)
6121 {
6122         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
6123
6124         if (err)
6125                 return err;
6126
6127         mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
6128
6129         return err;
6130 }
6131
6132 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
6133                                      struct mlxsw_sp_fib_entry *fib_entry)
6134 {
6135         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6136                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
6137 }
6138
6139 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
6140                                   struct mlxsw_sp_fib_entry *fib_entry)
6141 {
6142         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6143                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
6144 }
6145
6146 static int
6147 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
6148                              const struct fib_entry_notifier_info *fen_info,
6149                              struct mlxsw_sp_fib_entry *fib_entry)
6150 {
6151         struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
6152         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
6153         struct mlxsw_sp_router *router = mlxsw_sp->router;
6154         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
6155         int ifindex = nhgi->nexthops[0].ifindex;
6156         struct mlxsw_sp_ipip_entry *ipip_entry;
6157
6158         switch (fen_info->type) {
6159         case RTN_LOCAL:
6160                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
6161                                                                MLXSW_SP_L3_PROTO_IPV4, dip);
6162                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
6163                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
6164                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
6165                                                              fib_entry,
6166                                                              ipip_entry);
6167                 }
6168                 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
6169                                                  MLXSW_SP_L3_PROTO_IPV4,
6170                                                  &dip)) {
6171                         u32 tunnel_index;
6172
6173                         tunnel_index = router->nve_decap_config.tunnel_index;
6174                         fib_entry->decap.tunnel_index = tunnel_index;
6175                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
6176                         return 0;
6177                 }
6178                 fallthrough;
6179         case RTN_BROADCAST:
6180                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6181                 return 0;
6182         case RTN_BLACKHOLE:
6183                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
6184                 return 0;
6185         case RTN_UNREACHABLE:
6186         case RTN_PROHIBIT:
6187                 /* Packets hitting these routes need to be trapped, but
6188                  * can do so with a lower priority than packets directed
6189                  * at the host, so use action type local instead of trap.
6190                  */
6191                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
6192                 return 0;
6193         case RTN_UNICAST:
6194                 if (nhgi->gateway)
6195                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
6196                 else
6197                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
6198                 return 0;
6199         default:
6200                 return -EINVAL;
6201         }
6202 }
6203
6204 static void
6205 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6206                               struct mlxsw_sp_fib_entry *fib_entry)
6207 {
6208         switch (fib_entry->type) {
6209         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6210                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
6211                 break;
6212         default:
6213                 break;
6214         }
6215 }
6216
6217 static void
6218 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6219                                struct mlxsw_sp_fib4_entry *fib4_entry)
6220 {
6221         mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common);
6222 }
6223
6224 static struct mlxsw_sp_fib4_entry *
6225 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
6226                            struct mlxsw_sp_fib_node *fib_node,
6227                            const struct fib_entry_notifier_info *fen_info)
6228 {
6229         struct mlxsw_sp_fib4_entry *fib4_entry;
6230         struct mlxsw_sp_fib_entry *fib_entry;
6231         int err;
6232
6233         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
6234         if (!fib4_entry)
6235                 return ERR_PTR(-ENOMEM);
6236         fib_entry = &fib4_entry->common;
6237
6238         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
6239         if (err)
6240                 goto err_nexthop4_group_get;
6241
6242         err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
6243                                              fib_node->fib);
6244         if (err)
6245                 goto err_nexthop_group_vr_link;
6246
6247         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
6248         if (err)
6249                 goto err_fib4_entry_type_set;
6250
6251         fib4_entry->fi = fen_info->fi;
6252         fib_info_hold(fib4_entry->fi);
6253         fib4_entry->tb_id = fen_info->tb_id;
6254         fib4_entry->type = fen_info->type;
6255         fib4_entry->dscp = fen_info->dscp;
6256
6257         fib_entry->fib_node = fib_node;
6258
6259         return fib4_entry;
6260
6261 err_fib4_entry_type_set:
6262         mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
6263 err_nexthop_group_vr_link:
6264         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6265 err_nexthop4_group_get:
6266         kfree(fib4_entry);
6267         return ERR_PTR(err);
6268 }
6269
6270 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6271                                         struct mlxsw_sp_fib4_entry *fib4_entry)
6272 {
6273         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6274
6275         fib_info_put(fib4_entry->fi);
6276         mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry);
6277         mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group,
6278                                          fib_node->fib);
6279         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6280         kfree(fib4_entry);
6281 }
6282
6283 static struct mlxsw_sp_fib4_entry *
6284 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6285                            const struct fib_entry_notifier_info *fen_info)
6286 {
6287         struct mlxsw_sp_fib4_entry *fib4_entry;
6288         struct mlxsw_sp_fib_node *fib_node;
6289         struct mlxsw_sp_fib *fib;
6290         struct mlxsw_sp_vr *vr;
6291
6292         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
6293         if (!vr)
6294                 return NULL;
6295         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
6296
6297         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
6298                                             sizeof(fen_info->dst),
6299                                             fen_info->dst_len);
6300         if (!fib_node)
6301                 return NULL;
6302
6303         fib4_entry = container_of(fib_node->fib_entry,
6304                                   struct mlxsw_sp_fib4_entry, common);
6305         if (fib4_entry->tb_id == fen_info->tb_id &&
6306             fib4_entry->dscp == fen_info->dscp &&
6307             fib4_entry->type == fen_info->type &&
6308             fib4_entry->fi == fen_info->fi)
6309                 return fib4_entry;
6310
6311         return NULL;
6312 }
6313
6314 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
6315         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
6316         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
6317         .key_len = sizeof(struct mlxsw_sp_fib_key),
6318         .automatic_shrinking = true,
6319 };
6320
6321 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
6322                                     struct mlxsw_sp_fib_node *fib_node)
6323 {
6324         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
6325                                       mlxsw_sp_fib_ht_params);
6326 }
6327
6328 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
6329                                      struct mlxsw_sp_fib_node *fib_node)
6330 {
6331         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
6332                                mlxsw_sp_fib_ht_params);
6333 }
6334
6335 static struct mlxsw_sp_fib_node *
6336 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
6337                          size_t addr_len, unsigned char prefix_len)
6338 {
6339         struct mlxsw_sp_fib_key key;
6340
6341         memset(&key, 0, sizeof(key));
6342         memcpy(key.addr, addr, addr_len);
6343         key.prefix_len = prefix_len;
6344         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
6345 }
6346
6347 static struct mlxsw_sp_fib_node *
6348 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
6349                          size_t addr_len, unsigned char prefix_len)
6350 {
6351         struct mlxsw_sp_fib_node *fib_node;
6352
6353         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
6354         if (!fib_node)
6355                 return NULL;
6356
6357         list_add(&fib_node->list, &fib->node_list);
6358         memcpy(fib_node->key.addr, addr, addr_len);
6359         fib_node->key.prefix_len = prefix_len;
6360
6361         return fib_node;
6362 }
6363
6364 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
6365 {
6366         list_del(&fib_node->list);
6367         kfree(fib_node);
6368 }
6369
6370 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
6371                                       struct mlxsw_sp_fib_node *fib_node)
6372 {
6373         struct mlxsw_sp_prefix_usage req_prefix_usage;
6374         struct mlxsw_sp_fib *fib = fib_node->fib;
6375         struct mlxsw_sp_lpm_tree *lpm_tree;
6376         int err;
6377
6378         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
6379         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6380                 goto out;
6381
6382         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6383         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
6384         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6385                                          fib->proto);
6386         if (IS_ERR(lpm_tree))
6387                 return PTR_ERR(lpm_tree);
6388
6389         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6390         if (err)
6391                 goto err_lpm_tree_replace;
6392
6393 out:
6394         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
6395         return 0;
6396
6397 err_lpm_tree_replace:
6398         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6399         return err;
6400 }
6401
6402 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
6403                                          struct mlxsw_sp_fib_node *fib_node)
6404 {
6405         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
6406         struct mlxsw_sp_prefix_usage req_prefix_usage;
6407         struct mlxsw_sp_fib *fib = fib_node->fib;
6408         int err;
6409
6410         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6411                 return;
6412         /* Try to construct a new LPM tree from the current prefix usage
6413          * minus the unused one. If we fail, continue using the old one.
6414          */
6415         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6416         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
6417                                     fib_node->key.prefix_len);
6418         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6419                                          fib->proto);
6420         if (IS_ERR(lpm_tree))
6421                 return;
6422
6423         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6424         if (err)
6425                 goto err_lpm_tree_replace;
6426
6427         return;
6428
6429 err_lpm_tree_replace:
6430         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6431 }
6432
6433 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
6434                                   struct mlxsw_sp_fib_node *fib_node,
6435                                   struct mlxsw_sp_fib *fib)
6436 {
6437         int err;
6438
6439         err = mlxsw_sp_fib_node_insert(fib, fib_node);
6440         if (err)
6441                 return err;
6442         fib_node->fib = fib;
6443
6444         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
6445         if (err)
6446                 goto err_fib_lpm_tree_link;
6447
6448         return 0;
6449
6450 err_fib_lpm_tree_link:
6451         fib_node->fib = NULL;
6452         mlxsw_sp_fib_node_remove(fib, fib_node);
6453         return err;
6454 }
6455
6456 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
6457                                    struct mlxsw_sp_fib_node *fib_node)
6458 {
6459         struct mlxsw_sp_fib *fib = fib_node->fib;
6460
6461         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
6462         fib_node->fib = NULL;
6463         mlxsw_sp_fib_node_remove(fib, fib_node);
6464 }
6465
6466 static struct mlxsw_sp_fib_node *
6467 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
6468                       size_t addr_len, unsigned char prefix_len,
6469                       enum mlxsw_sp_l3proto proto)
6470 {
6471         struct mlxsw_sp_fib_node *fib_node;
6472         struct mlxsw_sp_fib *fib;
6473         struct mlxsw_sp_vr *vr;
6474         int err;
6475
6476         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
6477         if (IS_ERR(vr))
6478                 return ERR_CAST(vr);
6479         fib = mlxsw_sp_vr_fib(vr, proto);
6480
6481         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
6482         if (fib_node)
6483                 return fib_node;
6484
6485         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
6486         if (!fib_node) {
6487                 err = -ENOMEM;
6488                 goto err_fib_node_create;
6489         }
6490
6491         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
6492         if (err)
6493                 goto err_fib_node_init;
6494
6495         return fib_node;
6496
6497 err_fib_node_init:
6498         mlxsw_sp_fib_node_destroy(fib_node);
6499 err_fib_node_create:
6500         mlxsw_sp_vr_put(mlxsw_sp, vr);
6501         return ERR_PTR(err);
6502 }
6503
6504 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
6505                                   struct mlxsw_sp_fib_node *fib_node)
6506 {
6507         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
6508
6509         if (fib_node->fib_entry)
6510                 return;
6511         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
6512         mlxsw_sp_fib_node_destroy(fib_node);
6513         mlxsw_sp_vr_put(mlxsw_sp, vr);
6514 }
6515
6516 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
6517                                         struct mlxsw_sp_fib_entry *fib_entry)
6518 {
6519         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6520         int err;
6521
6522         fib_node->fib_entry = fib_entry;
6523
6524         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
6525         if (err)
6526                 goto err_fib_entry_update;
6527
6528         return 0;
6529
6530 err_fib_entry_update:
6531         fib_node->fib_entry = NULL;
6532         return err;
6533 }
6534
6535 static void
6536 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
6537                                struct mlxsw_sp_fib_entry *fib_entry)
6538 {
6539         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6540
6541         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
6542         fib_node->fib_entry = NULL;
6543 }
6544
6545 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
6546 {
6547         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6548         struct mlxsw_sp_fib4_entry *fib4_replaced;
6549
6550         if (!fib_node->fib_entry)
6551                 return true;
6552
6553         fib4_replaced = container_of(fib_node->fib_entry,
6554                                      struct mlxsw_sp_fib4_entry, common);
6555         if (fib4_entry->tb_id == RT_TABLE_MAIN &&
6556             fib4_replaced->tb_id == RT_TABLE_LOCAL)
6557                 return false;
6558
6559         return true;
6560 }
6561
6562 static int
6563 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
6564                              const struct fib_entry_notifier_info *fen_info)
6565 {
6566         struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
6567         struct mlxsw_sp_fib_entry *replaced;
6568         struct mlxsw_sp_fib_node *fib_node;
6569         int err;
6570
6571         if (fen_info->fi->nh &&
6572             !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id))
6573                 return 0;
6574
6575         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
6576                                          &fen_info->dst, sizeof(fen_info->dst),
6577                                          fen_info->dst_len,
6578                                          MLXSW_SP_L3_PROTO_IPV4);
6579         if (IS_ERR(fib_node)) {
6580                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
6581                 return PTR_ERR(fib_node);
6582         }
6583
6584         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
6585         if (IS_ERR(fib4_entry)) {
6586                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
6587                 err = PTR_ERR(fib4_entry);
6588                 goto err_fib4_entry_create;
6589         }
6590
6591         if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
6592                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6593                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6594                 return 0;
6595         }
6596
6597         replaced = fib_node->fib_entry;
6598         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common);
6599         if (err) {
6600                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
6601                 goto err_fib_node_entry_link;
6602         }
6603
6604         /* Nothing to replace */
6605         if (!replaced)
6606                 return 0;
6607
6608         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
6609         fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
6610                                      common);
6611         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
6612
6613         return 0;
6614
6615 err_fib_node_entry_link:
6616         fib_node->fib_entry = replaced;
6617         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6618 err_fib4_entry_create:
6619         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6620         return err;
6621 }
6622
6623 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
6624                                      struct fib_entry_notifier_info *fen_info)
6625 {
6626         struct mlxsw_sp_fib4_entry *fib4_entry;
6627         struct mlxsw_sp_fib_node *fib_node;
6628
6629         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
6630         if (!fib4_entry)
6631                 return;
6632         fib_node = fib4_entry->common.fib_node;
6633
6634         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common);
6635         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6636         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6637 }
6638
6639 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
6640 {
6641         /* Multicast routes aren't supported, so ignore them. Neighbour
6642          * Discovery packets are specifically trapped.
6643          */
6644         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
6645                 return true;
6646
6647         /* Cloned routes are irrelevant in the forwarding path. */
6648         if (rt->fib6_flags & RTF_CACHE)
6649                 return true;
6650
6651         return false;
6652 }
6653
6654 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
6655 {
6656         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6657
6658         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
6659         if (!mlxsw_sp_rt6)
6660                 return ERR_PTR(-ENOMEM);
6661
6662         /* In case of route replace, replaced route is deleted with
6663          * no notification. Take reference to prevent accessing freed
6664          * memory.
6665          */
6666         mlxsw_sp_rt6->rt = rt;
6667         fib6_info_hold(rt);
6668
6669         return mlxsw_sp_rt6;
6670 }
6671
6672 #if IS_ENABLED(CONFIG_IPV6)
6673 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6674 {
6675         fib6_info_release(rt);
6676 }
6677 #else
6678 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6679 {
6680 }
6681 #endif
6682
6683 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
6684 {
6685         struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
6686
6687         if (!mlxsw_sp_rt6->rt->nh)
6688                 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
6689         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
6690         kfree(mlxsw_sp_rt6);
6691 }
6692
6693 static struct fib6_info *
6694 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
6695 {
6696         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
6697                                 list)->rt;
6698 }
6699
6700 static struct mlxsw_sp_rt6 *
6701 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
6702                             const struct fib6_info *rt)
6703 {
6704         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6705
6706         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
6707                 if (mlxsw_sp_rt6->rt == rt)
6708                         return mlxsw_sp_rt6;
6709         }
6710
6711         return NULL;
6712 }
6713
6714 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
6715                                         const struct fib6_info *rt,
6716                                         enum mlxsw_sp_ipip_type *ret)
6717 {
6718         return rt->fib6_nh->fib_nh_dev &&
6719                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
6720 }
6721
6722 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
6723                                   struct mlxsw_sp_nexthop_group *nh_grp,
6724                                   struct mlxsw_sp_nexthop *nh,
6725                                   const struct fib6_info *rt)
6726 {
6727         struct net_device *dev = rt->fib6_nh->fib_nh_dev;
6728         int err;
6729
6730         nh->nhgi = nh_grp->nhgi;
6731         nh->nh_weight = rt->fib6_nh->fib_nh_weight;
6732         memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
6733 #if IS_ENABLED(CONFIG_IPV6)
6734         nh->neigh_tbl = &nd_tbl;
6735 #endif
6736         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
6737
6738         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
6739
6740         if (!dev)
6741                 return 0;
6742         nh->ifindex = dev->ifindex;
6743
6744         err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
6745         if (err)
6746                 goto err_nexthop_type_init;
6747
6748         return 0;
6749
6750 err_nexthop_type_init:
6751         list_del(&nh->router_list_node);
6752         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6753         return err;
6754 }
6755
6756 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
6757                                    struct mlxsw_sp_nexthop *nh)
6758 {
6759         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
6760         list_del(&nh->router_list_node);
6761         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6762 }
6763
6764 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
6765                                     const struct fib6_info *rt)
6766 {
6767         return rt->fib6_nh->fib_nh_gw_family ||
6768                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
6769 }
6770
6771 static int
6772 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
6773                                   struct mlxsw_sp_nexthop_group *nh_grp,
6774                                   struct mlxsw_sp_fib6_entry *fib6_entry)
6775 {
6776         struct mlxsw_sp_nexthop_group_info *nhgi;
6777         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6778         struct mlxsw_sp_nexthop *nh;
6779         int err, i;
6780
6781         nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
6782                        GFP_KERNEL);
6783         if (!nhgi)
6784                 return -ENOMEM;
6785         nh_grp->nhgi = nhgi;
6786         nhgi->nh_grp = nh_grp;
6787         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
6788                                         struct mlxsw_sp_rt6, list);
6789         nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
6790         nhgi->count = fib6_entry->nrt6;
6791         for (i = 0; i < nhgi->count; i++) {
6792                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
6793
6794                 nh = &nhgi->nexthops[i];
6795                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
6796                 if (err)
6797                         goto err_nexthop6_init;
6798                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
6799         }
6800         nh_grp->nhgi = nhgi;
6801         err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
6802         if (err)
6803                 goto err_group_inc;
6804         err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6805         if (err)
6806                 goto err_group_refresh;
6807
6808         return 0;
6809
6810 err_group_refresh:
6811         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6812 err_group_inc:
6813         i = nhgi->count;
6814 err_nexthop6_init:
6815         for (i--; i >= 0; i--) {
6816                 nh = &nhgi->nexthops[i];
6817                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6818         }
6819         kfree(nhgi);
6820         return err;
6821 }
6822
6823 static void
6824 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
6825                                   struct mlxsw_sp_nexthop_group *nh_grp)
6826 {
6827         struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
6828         int i;
6829
6830         mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6831         for (i = nhgi->count - 1; i >= 0; i--) {
6832                 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
6833
6834                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6835         }
6836         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6837         WARN_ON_ONCE(nhgi->adj_index_valid);
6838         kfree(nhgi);
6839 }
6840
6841 static struct mlxsw_sp_nexthop_group *
6842 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
6843                                struct mlxsw_sp_fib6_entry *fib6_entry)
6844 {
6845         struct mlxsw_sp_nexthop_group *nh_grp;
6846         int err;
6847
6848         nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
6849         if (!nh_grp)
6850                 return ERR_PTR(-ENOMEM);
6851         INIT_LIST_HEAD(&nh_grp->vr_list);
6852         err = rhashtable_init(&nh_grp->vr_ht,
6853                               &mlxsw_sp_nexthop_group_vr_ht_params);
6854         if (err)
6855                 goto err_nexthop_group_vr_ht_init;
6856         INIT_LIST_HEAD(&nh_grp->fib_list);
6857         nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
6858
6859         err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
6860         if (err)
6861                 goto err_nexthop_group_info_init;
6862
6863         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
6864         if (err)
6865                 goto err_nexthop_group_insert;
6866
6867         nh_grp->can_destroy = true;
6868
6869         return nh_grp;
6870
6871 err_nexthop_group_insert:
6872         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6873 err_nexthop_group_info_init:
6874         rhashtable_destroy(&nh_grp->vr_ht);
6875 err_nexthop_group_vr_ht_init:
6876         kfree(nh_grp);
6877         return ERR_PTR(err);
6878 }
6879
6880 static void
6881 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
6882                                 struct mlxsw_sp_nexthop_group *nh_grp)
6883 {
6884         if (!nh_grp->can_destroy)
6885                 return;
6886         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
6887         mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6888         WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
6889         rhashtable_destroy(&nh_grp->vr_ht);
6890         kfree(nh_grp);
6891 }
6892
6893 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
6894                                        struct mlxsw_sp_fib6_entry *fib6_entry)
6895 {
6896         struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6897         struct mlxsw_sp_nexthop_group *nh_grp;
6898
6899         if (rt->nh) {
6900                 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
6901                                                            rt->nh->id);
6902                 if (WARN_ON_ONCE(!nh_grp))
6903                         return -EINVAL;
6904                 goto out;
6905         }
6906
6907         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
6908         if (!nh_grp) {
6909                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
6910                 if (IS_ERR(nh_grp))
6911                         return PTR_ERR(nh_grp);
6912         }
6913
6914         /* The route and the nexthop are described by the same struct, so we
6915          * need to the update the nexthop offload indication for the new route.
6916          */
6917         __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
6918
6919 out:
6920         list_add_tail(&fib6_entry->common.nexthop_group_node,
6921                       &nh_grp->fib_list);
6922         fib6_entry->common.nh_group = nh_grp;
6923
6924         return 0;
6925 }
6926
6927 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
6928                                         struct mlxsw_sp_fib_entry *fib_entry)
6929 {
6930         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
6931
6932         list_del(&fib_entry->nexthop_group_node);
6933         if (!list_empty(&nh_grp->fib_list))
6934                 return;
6935
6936         if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
6937                 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
6938                 return;
6939         }
6940
6941         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
6942 }
6943
6944 static int
6945 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
6946                                struct mlxsw_sp_fib6_entry *fib6_entry)
6947 {
6948         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
6949         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6950         int err;
6951
6952         mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib);
6953         fib6_entry->common.nh_group = NULL;
6954         list_del(&fib6_entry->common.nexthop_group_node);
6955
6956         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6957         if (err)
6958                 goto err_nexthop6_group_get;
6959
6960         err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group,
6961                                              fib_node->fib);
6962         if (err)
6963                 goto err_nexthop_group_vr_link;
6964
6965         /* In case this entry is offloaded, then the adjacency index
6966          * currently associated with it in the device's table is that
6967          * of the old group. Start using the new one instead.
6968          */
6969         err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common);
6970         if (err)
6971                 goto err_fib_entry_update;
6972
6973         if (list_empty(&old_nh_grp->fib_list))
6974                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
6975
6976         return 0;
6977
6978 err_fib_entry_update:
6979         mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6980                                          fib_node->fib);
6981 err_nexthop_group_vr_link:
6982         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6983 err_nexthop6_group_get:
6984         list_add_tail(&fib6_entry->common.nexthop_group_node,
6985                       &old_nh_grp->fib_list);
6986         fib6_entry->common.nh_group = old_nh_grp;
6987         mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib);
6988         return err;
6989 }
6990
6991 static int
6992 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
6993                                 struct mlxsw_sp_fib6_entry *fib6_entry,
6994                                 struct fib6_info **rt_arr, unsigned int nrt6)
6995 {
6996         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6997         int err, i;
6998
6999         for (i = 0; i < nrt6; i++) {
7000                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
7001                 if (IS_ERR(mlxsw_sp_rt6)) {
7002                         err = PTR_ERR(mlxsw_sp_rt6);
7003                         goto err_rt6_unwind;
7004                 }
7005
7006                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
7007                 fib6_entry->nrt6++;
7008         }
7009
7010         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7011         if (err)
7012                 goto err_rt6_unwind;
7013
7014         return 0;
7015
7016 err_rt6_unwind:
7017         for (; i > 0; i--) {
7018                 fib6_entry->nrt6--;
7019                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7020                                                struct mlxsw_sp_rt6, list);
7021                 list_del(&mlxsw_sp_rt6->list);
7022                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7023         }
7024         return err;
7025 }
7026
7027 static void
7028 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
7029                                 struct mlxsw_sp_fib6_entry *fib6_entry,
7030                                 struct fib6_info **rt_arr, unsigned int nrt6)
7031 {
7032         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7033         int i;
7034
7035         for (i = 0; i < nrt6; i++) {
7036                 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
7037                                                            rt_arr[i]);
7038                 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
7039                         continue;
7040
7041                 fib6_entry->nrt6--;
7042                 list_del(&mlxsw_sp_rt6->list);
7043                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7044         }
7045
7046         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7047 }
7048
7049 static int
7050 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp,
7051                                    struct mlxsw_sp_fib_entry *fib_entry,
7052                                    const struct fib6_info *rt)
7053 {
7054         struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
7055         union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr };
7056         u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id);
7057         struct mlxsw_sp_router *router = mlxsw_sp->router;
7058         int ifindex = nhgi->nexthops[0].ifindex;
7059         struct mlxsw_sp_ipip_entry *ipip_entry;
7060
7061         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7062         ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
7063                                                        MLXSW_SP_L3_PROTO_IPV6,
7064                                                        dip);
7065
7066         if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
7067                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
7068                 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry,
7069                                                      ipip_entry);
7070         }
7071         if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
7072                                          MLXSW_SP_L3_PROTO_IPV6, &dip)) {
7073                 u32 tunnel_index;
7074
7075                 tunnel_index = router->nve_decap_config.tunnel_index;
7076                 fib_entry->decap.tunnel_index = tunnel_index;
7077                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
7078         }
7079
7080         return 0;
7081 }
7082
7083 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
7084                                         struct mlxsw_sp_fib_entry *fib_entry,
7085                                         const struct fib6_info *rt)
7086 {
7087         if (rt->fib6_flags & RTF_LOCAL)
7088                 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry,
7089                                                           rt);
7090         if (rt->fib6_flags & RTF_ANYCAST)
7091                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7092         else if (rt->fib6_type == RTN_BLACKHOLE)
7093                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
7094         else if (rt->fib6_flags & RTF_REJECT)
7095                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
7096         else if (fib_entry->nh_group->nhgi->gateway)
7097                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
7098         else
7099                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
7100
7101         return 0;
7102 }
7103
7104 static void
7105 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
7106 {
7107         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
7108
7109         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
7110                                  list) {
7111                 fib6_entry->nrt6--;
7112                 list_del(&mlxsw_sp_rt6->list);
7113                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7114         }
7115 }
7116
7117 static struct mlxsw_sp_fib6_entry *
7118 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
7119                            struct mlxsw_sp_fib_node *fib_node,
7120                            struct fib6_info **rt_arr, unsigned int nrt6)
7121 {
7122         struct mlxsw_sp_fib6_entry *fib6_entry;
7123         struct mlxsw_sp_fib_entry *fib_entry;
7124         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7125         int err, i;
7126
7127         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
7128         if (!fib6_entry)
7129                 return ERR_PTR(-ENOMEM);
7130         fib_entry = &fib6_entry->common;
7131
7132         INIT_LIST_HEAD(&fib6_entry->rt6_list);
7133
7134         for (i = 0; i < nrt6; i++) {
7135                 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
7136                 if (IS_ERR(mlxsw_sp_rt6)) {
7137                         err = PTR_ERR(mlxsw_sp_rt6);
7138                         goto err_rt6_unwind;
7139                 }
7140                 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
7141                 fib6_entry->nrt6++;
7142         }
7143
7144         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
7145         if (err)
7146                 goto err_rt6_unwind;
7147
7148         err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
7149                                              fib_node->fib);
7150         if (err)
7151                 goto err_nexthop_group_vr_link;
7152
7153         err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
7154         if (err)
7155                 goto err_fib6_entry_type_set;
7156
7157         fib_entry->fib_node = fib_node;
7158
7159         return fib6_entry;
7160
7161 err_fib6_entry_type_set:
7162         mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
7163 err_nexthop_group_vr_link:
7164         mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
7165 err_rt6_unwind:
7166         for (; i > 0; i--) {
7167                 fib6_entry->nrt6--;
7168                 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7169                                                struct mlxsw_sp_rt6, list);
7170                 list_del(&mlxsw_sp_rt6->list);
7171                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7172         }
7173         kfree(fib6_entry);
7174         return ERR_PTR(err);
7175 }
7176
7177 static void
7178 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
7179                                struct mlxsw_sp_fib6_entry *fib6_entry)
7180 {
7181         mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common);
7182 }
7183
7184 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
7185                                         struct mlxsw_sp_fib6_entry *fib6_entry)
7186 {
7187         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7188
7189         mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry);
7190         mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
7191                                          fib_node->fib);
7192         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
7193         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
7194         WARN_ON(fib6_entry->nrt6);
7195         kfree(fib6_entry);
7196 }
7197
7198 static struct mlxsw_sp_fib6_entry *
7199 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
7200                            const struct fib6_info *rt)
7201 {
7202         struct mlxsw_sp_fib6_entry *fib6_entry;
7203         struct mlxsw_sp_fib_node *fib_node;
7204         struct mlxsw_sp_fib *fib;
7205         struct fib6_info *cmp_rt;
7206         struct mlxsw_sp_vr *vr;
7207
7208         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
7209         if (!vr)
7210                 return NULL;
7211         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
7212
7213         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
7214                                             sizeof(rt->fib6_dst.addr),
7215                                             rt->fib6_dst.plen);
7216         if (!fib_node)
7217                 return NULL;
7218
7219         fib6_entry = container_of(fib_node->fib_entry,
7220                                   struct mlxsw_sp_fib6_entry, common);
7221         cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7222         if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
7223             rt->fib6_metric == cmp_rt->fib6_metric &&
7224             mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
7225                 return fib6_entry;
7226
7227         return NULL;
7228 }
7229
7230 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
7231 {
7232         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7233         struct mlxsw_sp_fib6_entry *fib6_replaced;
7234         struct fib6_info *rt, *rt_replaced;
7235
7236         if (!fib_node->fib_entry)
7237                 return true;
7238
7239         fib6_replaced = container_of(fib_node->fib_entry,
7240                                      struct mlxsw_sp_fib6_entry,
7241                                      common);
7242         rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7243         rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
7244         if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
7245             rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
7246                 return false;
7247
7248         return true;
7249 }
7250
7251 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
7252                                         struct fib6_info **rt_arr,
7253                                         unsigned int nrt6)
7254 {
7255         struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
7256         struct mlxsw_sp_fib_entry *replaced;
7257         struct mlxsw_sp_fib_node *fib_node;
7258         struct fib6_info *rt = rt_arr[0];
7259         int err;
7260
7261         if (rt->fib6_src.plen)
7262                 return -EINVAL;
7263
7264         if (mlxsw_sp_fib6_rt_should_ignore(rt))
7265                 return 0;
7266
7267         if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id))
7268                 return 0;
7269
7270         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7271                                          &rt->fib6_dst.addr,
7272                                          sizeof(rt->fib6_dst.addr),
7273                                          rt->fib6_dst.plen,
7274                                          MLXSW_SP_L3_PROTO_IPV6);
7275         if (IS_ERR(fib_node))
7276                 return PTR_ERR(fib_node);
7277
7278         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
7279                                                 nrt6);
7280         if (IS_ERR(fib6_entry)) {
7281                 err = PTR_ERR(fib6_entry);
7282                 goto err_fib6_entry_create;
7283         }
7284
7285         if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
7286                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7287                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7288                 return 0;
7289         }
7290
7291         replaced = fib_node->fib_entry;
7292         err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common);
7293         if (err)
7294                 goto err_fib_node_entry_link;
7295
7296         /* Nothing to replace */
7297         if (!replaced)
7298                 return 0;
7299
7300         mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
7301         fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
7302                                      common);
7303         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
7304
7305         return 0;
7306
7307 err_fib_node_entry_link:
7308         fib_node->fib_entry = replaced;
7309         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7310 err_fib6_entry_create:
7311         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7312         return err;
7313 }
7314
7315 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
7316                                        struct fib6_info **rt_arr,
7317                                        unsigned int nrt6)
7318 {
7319         struct mlxsw_sp_fib6_entry *fib6_entry;
7320         struct mlxsw_sp_fib_node *fib_node;
7321         struct fib6_info *rt = rt_arr[0];
7322         int err;
7323
7324         if (rt->fib6_src.plen)
7325                 return -EINVAL;
7326
7327         if (mlxsw_sp_fib6_rt_should_ignore(rt))
7328                 return 0;
7329
7330         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7331                                          &rt->fib6_dst.addr,
7332                                          sizeof(rt->fib6_dst.addr),
7333                                          rt->fib6_dst.plen,
7334                                          MLXSW_SP_L3_PROTO_IPV6);
7335         if (IS_ERR(fib_node))
7336                 return PTR_ERR(fib_node);
7337
7338         if (WARN_ON_ONCE(!fib_node->fib_entry)) {
7339                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7340                 return -EINVAL;
7341         }
7342
7343         fib6_entry = container_of(fib_node->fib_entry,
7344                                   struct mlxsw_sp_fib6_entry, common);
7345         err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr,
7346                                               nrt6);
7347         if (err)
7348                 goto err_fib6_entry_nexthop_add;
7349
7350         return 0;
7351
7352 err_fib6_entry_nexthop_add:
7353         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7354         return err;
7355 }
7356
7357 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
7358                                      struct fib6_info **rt_arr,
7359                                      unsigned int nrt6)
7360 {
7361         struct mlxsw_sp_fib6_entry *fib6_entry;
7362         struct mlxsw_sp_fib_node *fib_node;
7363         struct fib6_info *rt = rt_arr[0];
7364
7365         if (mlxsw_sp_fib6_rt_should_ignore(rt))
7366                 return;
7367
7368         /* Multipath routes are first added to the FIB trie and only then
7369          * notified. If we vetoed the addition, we will get a delete
7370          * notification for a route we do not have. Therefore, do not warn if
7371          * route was not found.
7372          */
7373         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
7374         if (!fib6_entry)
7375                 return;
7376
7377         /* If not all the nexthops are deleted, then only reduce the nexthop
7378          * group.
7379          */
7380         if (nrt6 != fib6_entry->nrt6) {
7381                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr,
7382                                                 nrt6);
7383                 return;
7384         }
7385
7386         fib_node = fib6_entry->common.fib_node;
7387
7388         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common);
7389         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7390         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7391 }
7392
7393 static struct mlxsw_sp_mr_table *
7394 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
7395 {
7396         if (family == RTNL_FAMILY_IPMR)
7397                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
7398         else
7399                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
7400 }
7401
7402 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
7403                                      struct mfc_entry_notifier_info *men_info,
7404                                      bool replace)
7405 {
7406         struct mlxsw_sp_mr_table *mrt;
7407         struct mlxsw_sp_vr *vr;
7408
7409         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
7410         if (IS_ERR(vr))
7411                 return PTR_ERR(vr);
7412
7413         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7414         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
7415 }
7416
7417 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
7418                                       struct mfc_entry_notifier_info *men_info)
7419 {
7420         struct mlxsw_sp_mr_table *mrt;
7421         struct mlxsw_sp_vr *vr;
7422
7423         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
7424         if (WARN_ON(!vr))
7425                 return;
7426
7427         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7428         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
7429         mlxsw_sp_vr_put(mlxsw_sp, vr);
7430 }
7431
7432 static int
7433 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
7434                               struct vif_entry_notifier_info *ven_info)
7435 {
7436         struct mlxsw_sp_mr_table *mrt;
7437         struct mlxsw_sp_rif *rif;
7438         struct mlxsw_sp_vr *vr;
7439
7440         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
7441         if (IS_ERR(vr))
7442                 return PTR_ERR(vr);
7443
7444         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7445         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
7446         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
7447                                    ven_info->vif_index,
7448                                    ven_info->vif_flags, rif);
7449 }
7450
7451 static void
7452 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
7453                               struct vif_entry_notifier_info *ven_info)
7454 {
7455         struct mlxsw_sp_mr_table *mrt;
7456         struct mlxsw_sp_vr *vr;
7457
7458         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
7459         if (WARN_ON(!vr))
7460                 return;
7461
7462         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7463         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
7464         mlxsw_sp_vr_put(mlxsw_sp, vr);
7465 }
7466
7467 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
7468                                      struct mlxsw_sp_fib_node *fib_node)
7469 {
7470         struct mlxsw_sp_fib4_entry *fib4_entry;
7471
7472         fib4_entry = container_of(fib_node->fib_entry,
7473                                   struct mlxsw_sp_fib4_entry, common);
7474         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7475         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
7476         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7477 }
7478
7479 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
7480                                      struct mlxsw_sp_fib_node *fib_node)
7481 {
7482         struct mlxsw_sp_fib6_entry *fib6_entry;
7483
7484         fib6_entry = container_of(fib_node->fib_entry,
7485                                   struct mlxsw_sp_fib6_entry, common);
7486         mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7487         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7488         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7489 }
7490
7491 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
7492                                     struct mlxsw_sp_fib_node *fib_node)
7493 {
7494         switch (fib_node->fib->proto) {
7495         case MLXSW_SP_L3_PROTO_IPV4:
7496                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
7497                 break;
7498         case MLXSW_SP_L3_PROTO_IPV6:
7499                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
7500                 break;
7501         }
7502 }
7503
7504 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
7505                                   struct mlxsw_sp_vr *vr,
7506                                   enum mlxsw_sp_l3proto proto)
7507 {
7508         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
7509         struct mlxsw_sp_fib_node *fib_node, *tmp;
7510
7511         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
7512                 bool do_break = &tmp->list == &fib->node_list;
7513
7514                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
7515                 if (do_break)
7516                         break;
7517         }
7518 }
7519
7520 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
7521 {
7522         int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
7523         int i, j;
7524
7525         for (i = 0; i < max_vrs; i++) {
7526                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
7527
7528                 if (!mlxsw_sp_vr_is_used(vr))
7529                         continue;
7530
7531                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
7532                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
7533                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
7534
7535                 /* If virtual router was only used for IPv4, then it's no
7536                  * longer used.
7537                  */
7538                 if (!mlxsw_sp_vr_is_used(vr))
7539                         continue;
7540                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
7541         }
7542 }
7543
7544 struct mlxsw_sp_fib6_event_work {
7545         struct fib6_info **rt_arr;
7546         unsigned int nrt6;
7547 };
7548
7549 struct mlxsw_sp_fib_event_work {
7550         struct work_struct work;
7551         netdevice_tracker dev_tracker;
7552         union {
7553                 struct mlxsw_sp_fib6_event_work fib6_work;
7554                 struct fib_entry_notifier_info fen_info;
7555                 struct fib_rule_notifier_info fr_info;
7556                 struct fib_nh_notifier_info fnh_info;
7557                 struct mfc_entry_notifier_info men_info;
7558                 struct vif_entry_notifier_info ven_info;
7559         };
7560         struct mlxsw_sp *mlxsw_sp;
7561         unsigned long event;
7562 };
7563
7564 static int
7565 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work,
7566                                struct fib6_entry_notifier_info *fen6_info)
7567 {
7568         struct fib6_info *rt = fen6_info->rt;
7569         struct fib6_info **rt_arr;
7570         struct fib6_info *iter;
7571         unsigned int nrt6;
7572         int i = 0;
7573
7574         nrt6 = fen6_info->nsiblings + 1;
7575
7576         rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
7577         if (!rt_arr)
7578                 return -ENOMEM;
7579
7580         fib6_work->rt_arr = rt_arr;
7581         fib6_work->nrt6 = nrt6;
7582
7583         rt_arr[0] = rt;
7584         fib6_info_hold(rt);
7585
7586         if (!fen6_info->nsiblings)
7587                 return 0;
7588
7589         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
7590                 if (i == fen6_info->nsiblings)
7591                         break;
7592
7593                 rt_arr[i + 1] = iter;
7594                 fib6_info_hold(iter);
7595                 i++;
7596         }
7597         WARN_ON_ONCE(i != fen6_info->nsiblings);
7598
7599         return 0;
7600 }
7601
7602 static void
7603 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work)
7604 {
7605         int i;
7606
7607         for (i = 0; i < fib6_work->nrt6; i++)
7608                 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]);
7609         kfree(fib6_work->rt_arr);
7610 }
7611
7612 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
7613 {
7614         struct mlxsw_sp_fib_event_work *fib_work =
7615                 container_of(work, struct mlxsw_sp_fib_event_work, work);
7616         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7617         int err;
7618
7619         mutex_lock(&mlxsw_sp->router->lock);
7620         mlxsw_sp_span_respin(mlxsw_sp);
7621
7622         switch (fib_work->event) {
7623         case FIB_EVENT_ENTRY_REPLACE:
7624                 err = mlxsw_sp_router_fib4_replace(mlxsw_sp,
7625                                                    &fib_work->fen_info);
7626                 if (err) {
7627                         dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7628                         mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
7629                                                               &fib_work->fen_info);
7630                 }
7631                 fib_info_put(fib_work->fen_info.fi);
7632                 break;
7633         case FIB_EVENT_ENTRY_DEL:
7634                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
7635                 fib_info_put(fib_work->fen_info.fi);
7636                 break;
7637         case FIB_EVENT_NH_ADD:
7638         case FIB_EVENT_NH_DEL:
7639                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
7640                                         fib_work->fnh_info.fib_nh);
7641                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
7642                 break;
7643         }
7644         mutex_unlock(&mlxsw_sp->router->lock);
7645         kfree(fib_work);
7646 }
7647
7648 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
7649 {
7650         struct mlxsw_sp_fib_event_work *fib_work =
7651                     container_of(work, struct mlxsw_sp_fib_event_work, work);
7652         struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work;
7653         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7654         int err;
7655
7656         mutex_lock(&mlxsw_sp->router->lock);
7657         mlxsw_sp_span_respin(mlxsw_sp);
7658
7659         switch (fib_work->event) {
7660         case FIB_EVENT_ENTRY_REPLACE:
7661                 err = mlxsw_sp_router_fib6_replace(mlxsw_sp,
7662                                                    fib6_work->rt_arr,
7663                                                    fib6_work->nrt6);
7664                 if (err) {
7665                         dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7666                         mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7667                                                               fib6_work->rt_arr,
7668                                                               fib6_work->nrt6);
7669                 }
7670                 mlxsw_sp_router_fib6_work_fini(fib6_work);
7671                 break;
7672         case FIB_EVENT_ENTRY_APPEND:
7673                 err = mlxsw_sp_router_fib6_append(mlxsw_sp,
7674                                                   fib6_work->rt_arr,
7675                                                   fib6_work->nrt6);
7676                 if (err) {
7677                         dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n");
7678                         mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7679                                                               fib6_work->rt_arr,
7680                                                               fib6_work->nrt6);
7681                 }
7682                 mlxsw_sp_router_fib6_work_fini(fib6_work);
7683                 break;
7684         case FIB_EVENT_ENTRY_DEL:
7685                 mlxsw_sp_router_fib6_del(mlxsw_sp,
7686                                          fib6_work->rt_arr,
7687                                          fib6_work->nrt6);
7688                 mlxsw_sp_router_fib6_work_fini(fib6_work);
7689                 break;
7690         }
7691         mutex_unlock(&mlxsw_sp->router->lock);
7692         kfree(fib_work);
7693 }
7694
7695 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
7696 {
7697         struct mlxsw_sp_fib_event_work *fib_work =
7698                 container_of(work, struct mlxsw_sp_fib_event_work, work);
7699         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7700         bool replace;
7701         int err;
7702
7703         rtnl_lock();
7704         mutex_lock(&mlxsw_sp->router->lock);
7705         switch (fib_work->event) {
7706         case FIB_EVENT_ENTRY_REPLACE:
7707         case FIB_EVENT_ENTRY_ADD:
7708                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
7709
7710                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info,
7711                                                 replace);
7712                 if (err)
7713                         dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n");
7714                 mr_cache_put(fib_work->men_info.mfc);
7715                 break;
7716         case FIB_EVENT_ENTRY_DEL:
7717                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info);
7718                 mr_cache_put(fib_work->men_info.mfc);
7719                 break;
7720         case FIB_EVENT_VIF_ADD:
7721                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
7722                                                     &fib_work->ven_info);
7723                 if (err)
7724                         dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n");
7725                 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker);
7726                 break;
7727         case FIB_EVENT_VIF_DEL:
7728                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
7729                                               &fib_work->ven_info);
7730                 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker);
7731                 break;
7732         }
7733         mutex_unlock(&mlxsw_sp->router->lock);
7734         rtnl_unlock();
7735         kfree(fib_work);
7736 }
7737
7738 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
7739                                        struct fib_notifier_info *info)
7740 {
7741         struct fib_entry_notifier_info *fen_info;
7742         struct fib_nh_notifier_info *fnh_info;
7743
7744         switch (fib_work->event) {
7745         case FIB_EVENT_ENTRY_REPLACE:
7746         case FIB_EVENT_ENTRY_DEL:
7747                 fen_info = container_of(info, struct fib_entry_notifier_info,
7748                                         info);
7749                 fib_work->fen_info = *fen_info;
7750                 /* Take reference on fib_info to prevent it from being
7751                  * freed while work is queued. Release it afterwards.
7752                  */
7753                 fib_info_hold(fib_work->fen_info.fi);
7754                 break;
7755         case FIB_EVENT_NH_ADD:
7756         case FIB_EVENT_NH_DEL:
7757                 fnh_info = container_of(info, struct fib_nh_notifier_info,
7758                                         info);
7759                 fib_work->fnh_info = *fnh_info;
7760                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
7761                 break;
7762         }
7763 }
7764
7765 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
7766                                       struct fib_notifier_info *info)
7767 {
7768         struct fib6_entry_notifier_info *fen6_info;
7769         int err;
7770
7771         switch (fib_work->event) {
7772         case FIB_EVENT_ENTRY_REPLACE:
7773         case FIB_EVENT_ENTRY_APPEND:
7774         case FIB_EVENT_ENTRY_DEL:
7775                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
7776                                          info);
7777                 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work,
7778                                                      fen6_info);
7779                 if (err)
7780                         return err;
7781                 break;
7782         }
7783
7784         return 0;
7785 }
7786
7787 static void
7788 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
7789                             struct fib_notifier_info *info)
7790 {
7791         switch (fib_work->event) {
7792         case FIB_EVENT_ENTRY_REPLACE:
7793         case FIB_EVENT_ENTRY_ADD:
7794         case FIB_EVENT_ENTRY_DEL:
7795                 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
7796                 mr_cache_hold(fib_work->men_info.mfc);
7797                 break;
7798         case FIB_EVENT_VIF_ADD:
7799         case FIB_EVENT_VIF_DEL:
7800                 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
7801                 netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker,
7802                             GFP_ATOMIC);
7803                 break;
7804         }
7805 }
7806
7807 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
7808                                           struct fib_notifier_info *info,
7809                                           struct mlxsw_sp *mlxsw_sp)
7810 {
7811         struct netlink_ext_ack *extack = info->extack;
7812         struct fib_rule_notifier_info *fr_info;
7813         struct fib_rule *rule;
7814         int err = 0;
7815
7816         /* nothing to do at the moment */
7817         if (event == FIB_EVENT_RULE_DEL)
7818                 return 0;
7819
7820         fr_info = container_of(info, struct fib_rule_notifier_info, info);
7821         rule = fr_info->rule;
7822
7823         /* Rule only affects locally generated traffic */
7824         if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
7825                 return 0;
7826
7827         switch (info->family) {
7828         case AF_INET:
7829                 if (!fib4_rule_default(rule) && !rule->l3mdev)
7830                         err = -EOPNOTSUPP;
7831                 break;
7832         case AF_INET6:
7833                 if (!fib6_rule_default(rule) && !rule->l3mdev)
7834                         err = -EOPNOTSUPP;
7835                 break;
7836         case RTNL_FAMILY_IPMR:
7837                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
7838                         err = -EOPNOTSUPP;
7839                 break;
7840         case RTNL_FAMILY_IP6MR:
7841                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
7842                         err = -EOPNOTSUPP;
7843                 break;
7844         }
7845
7846         if (err < 0)
7847                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
7848
7849         return err;
7850 }
7851
7852 /* Called with rcu_read_lock() */
7853 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
7854                                      unsigned long event, void *ptr)
7855 {
7856         struct mlxsw_sp_fib_event_work *fib_work;
7857         struct fib_notifier_info *info = ptr;
7858         struct mlxsw_sp_router *router;
7859         int err;
7860
7861         if ((info->family != AF_INET && info->family != AF_INET6 &&
7862              info->family != RTNL_FAMILY_IPMR &&
7863              info->family != RTNL_FAMILY_IP6MR))
7864                 return NOTIFY_DONE;
7865
7866         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7867
7868         switch (event) {
7869         case FIB_EVENT_RULE_ADD:
7870         case FIB_EVENT_RULE_DEL:
7871                 err = mlxsw_sp_router_fib_rule_event(event, info,
7872                                                      router->mlxsw_sp);
7873                 return notifier_from_errno(err);
7874         case FIB_EVENT_ENTRY_ADD:
7875         case FIB_EVENT_ENTRY_REPLACE:
7876         case FIB_EVENT_ENTRY_APPEND:
7877                 if (info->family == AF_INET) {
7878                         struct fib_entry_notifier_info *fen_info = ptr;
7879
7880                         if (fen_info->fi->fib_nh_is_v6) {
7881                                 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
7882                                 return notifier_from_errno(-EINVAL);
7883                         }
7884                 }
7885                 break;
7886         }
7887
7888         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
7889         if (!fib_work)
7890                 return NOTIFY_BAD;
7891
7892         fib_work->mlxsw_sp = router->mlxsw_sp;
7893         fib_work->event = event;
7894
7895         switch (info->family) {
7896         case AF_INET:
7897                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
7898                 mlxsw_sp_router_fib4_event(fib_work, info);
7899                 break;
7900         case AF_INET6:
7901                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
7902                 err = mlxsw_sp_router_fib6_event(fib_work, info);
7903                 if (err)
7904                         goto err_fib_event;
7905                 break;
7906         case RTNL_FAMILY_IP6MR:
7907         case RTNL_FAMILY_IPMR:
7908                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
7909                 mlxsw_sp_router_fibmr_event(fib_work, info);
7910                 break;
7911         }
7912
7913         mlxsw_core_schedule_work(&fib_work->work);
7914
7915         return NOTIFY_DONE;
7916
7917 err_fib_event:
7918         kfree(fib_work);
7919         return NOTIFY_BAD;
7920 }
7921
7922 static struct mlxsw_sp_rif *
7923 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
7924                          const struct net_device *dev)
7925 {
7926         int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7927         int i;
7928
7929         for (i = 0; i < max_rifs; i++)
7930                 if (mlxsw_sp->router->rifs[i] &&
7931                     mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev))
7932                         return mlxsw_sp->router->rifs[i];
7933
7934         return NULL;
7935 }
7936
7937 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
7938                          const struct net_device *dev)
7939 {
7940         struct mlxsw_sp_rif *rif;
7941
7942         mutex_lock(&mlxsw_sp->router->lock);
7943         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7944         mutex_unlock(&mlxsw_sp->router->lock);
7945
7946         return rif;
7947 }
7948
7949 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
7950 {
7951         struct mlxsw_sp_rif *rif;
7952         u16 vid = 0;
7953
7954         mutex_lock(&mlxsw_sp->router->lock);
7955         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7956         if (!rif)
7957                 goto out;
7958
7959         /* We only return the VID for VLAN RIFs. Otherwise we return an
7960          * invalid value (0).
7961          */
7962         if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
7963                 goto out;
7964
7965         vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7966
7967 out:
7968         mutex_unlock(&mlxsw_sp->router->lock);
7969         return vid;
7970 }
7971
7972 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
7973 {
7974         char ritr_pl[MLXSW_REG_RITR_LEN];
7975         int err;
7976
7977         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
7978         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7979         if (err)
7980                 return err;
7981
7982         mlxsw_reg_ritr_enable_set(ritr_pl, false);
7983         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7984 }
7985
7986 static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
7987                                          struct mlxsw_sp_rif *rif)
7988 {
7989         int err;
7990
7991         err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif);
7992         if (err)
7993                 return err;
7994
7995         err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif);
7996         if (err)
7997                 goto err_nexthop;
7998
7999         return 0;
8000
8001 err_nexthop:
8002         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8003         return err;
8004 }
8005
8006 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
8007                                           struct mlxsw_sp_rif *rif)
8008 {
8009         /* Signal to nexthop cleanup that the RIF is going away. */
8010         rif->crif->rif = NULL;
8011
8012         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
8013         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
8014         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8015 }
8016
8017 static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8018 {
8019         struct inet6_dev *inet6_dev;
8020         struct in_device *idev;
8021
8022         idev = __in_dev_get_rcu(dev);
8023         if (idev && idev->ifa_list)
8024                 return false;
8025
8026         inet6_dev = __in6_dev_get(dev);
8027         if (inet6_dev && !list_empty(&inet6_dev->addr_list))
8028                 return false;
8029
8030         return true;
8031 }
8032
8033 static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8034 {
8035         bool addr_list_empty;
8036
8037         rcu_read_lock();
8038         addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev);
8039         rcu_read_unlock();
8040
8041         return addr_list_empty;
8042 }
8043
8044 static bool
8045 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
8046                            unsigned long event)
8047 {
8048         bool addr_list_empty;
8049
8050         switch (event) {
8051         case NETDEV_UP:
8052                 return rif == NULL;
8053         case NETDEV_DOWN:
8054                 addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev);
8055
8056                 /* macvlans do not have a RIF, but rather piggy back on the
8057                  * RIF of their lower device.
8058                  */
8059                 if (netif_is_macvlan(dev) && addr_list_empty)
8060                         return true;
8061
8062                 if (rif && addr_list_empty &&
8063                     !netif_is_l3_slave(mlxsw_sp_rif_dev(rif)))
8064                         return true;
8065                 /* It is possible we already removed the RIF ourselves
8066                  * if it was assigned to a netdev that is now a bridge
8067                  * or LAG slave.
8068                  */
8069                 return false;
8070         }
8071
8072         return false;
8073 }
8074
8075 static enum mlxsw_sp_rif_type
8076 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
8077                       const struct net_device *dev)
8078 {
8079         enum mlxsw_sp_fid_type type;
8080
8081         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
8082                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
8083
8084         /* Otherwise RIF type is derived from the type of the underlying FID. */
8085         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
8086                 type = MLXSW_SP_FID_TYPE_8021Q;
8087         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
8088                 type = MLXSW_SP_FID_TYPE_8021Q;
8089         else if (netif_is_bridge_master(dev))
8090                 type = MLXSW_SP_FID_TYPE_8021D;
8091         else
8092                 type = MLXSW_SP_FID_TYPE_RFID;
8093
8094         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
8095 }
8096
8097 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index,
8098                                     u8 rif_entries)
8099 {
8100         *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table,
8101                                       rif_entries);
8102         if (*p_rif_index == 0)
8103                 return -ENOBUFS;
8104         *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET;
8105
8106         /* RIF indexes must be aligned to the allocation size. */
8107         WARN_ON_ONCE(*p_rif_index % rif_entries);
8108
8109         return 0;
8110 }
8111
8112 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8113                                     u8 rif_entries)
8114 {
8115         gen_pool_free(mlxsw_sp->router->rifs_table,
8116                       MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries);
8117 }
8118
8119 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
8120                                                u16 vr_id,
8121                                                struct mlxsw_sp_crif *crif)
8122 {
8123         struct net_device *l3_dev = crif ? crif->key.dev : NULL;
8124         struct mlxsw_sp_rif *rif;
8125
8126         rif = kzalloc(rif_size, GFP_KERNEL);
8127         if (!rif)
8128                 return NULL;
8129
8130         INIT_LIST_HEAD(&rif->neigh_list);
8131         if (l3_dev) {
8132                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
8133                 rif->mtu = l3_dev->mtu;
8134         }
8135         rif->vr_id = vr_id;
8136         rif->rif_index = rif_index;
8137         if (crif) {
8138                 rif->crif = crif;
8139                 crif->rif = rif;
8140         }
8141
8142         return rif;
8143 }
8144
8145 static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif)
8146 {
8147         WARN_ON(!list_empty(&rif->neigh_list));
8148
8149         if (rif->crif)
8150                 rif->crif->rif = NULL;
8151         kfree(rif);
8152 }
8153
8154 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
8155                                            u16 rif_index)
8156 {
8157         return mlxsw_sp->router->rifs[rif_index];
8158 }
8159
8160 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
8161 {
8162         return rif->rif_index;
8163 }
8164
8165 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8166 {
8167         return lb_rif->common.rif_index;
8168 }
8169
8170 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8171 {
8172         struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common);
8173         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
8174         struct mlxsw_sp_vr *ul_vr;
8175
8176         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
8177         if (WARN_ON(IS_ERR(ul_vr)))
8178                 return 0;
8179
8180         return ul_vr->id;
8181 }
8182
8183 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8184 {
8185         return lb_rif->ul_rif_id;
8186 }
8187
8188 static bool
8189 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif)
8190 {
8191         return mlxsw_sp_rif_counter_valid_get(rif,
8192                                               MLXSW_SP_RIF_COUNTER_EGRESS) &&
8193                mlxsw_sp_rif_counter_valid_get(rif,
8194                                               MLXSW_SP_RIF_COUNTER_INGRESS);
8195 }
8196
8197 static int
8198 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif)
8199 {
8200         int err;
8201
8202         err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8203         if (err)
8204                 return err;
8205
8206         /* Clear stale data. */
8207         err = mlxsw_sp_rif_counter_fetch_clear(rif,
8208                                                MLXSW_SP_RIF_COUNTER_INGRESS,
8209                                                NULL);
8210         if (err)
8211                 goto err_clear_ingress;
8212
8213         err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8214         if (err)
8215                 goto err_alloc_egress;
8216
8217         /* Clear stale data. */
8218         err = mlxsw_sp_rif_counter_fetch_clear(rif,
8219                                                MLXSW_SP_RIF_COUNTER_EGRESS,
8220                                                NULL);
8221         if (err)
8222                 goto err_clear_egress;
8223
8224         return 0;
8225
8226 err_clear_egress:
8227         mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8228 err_alloc_egress:
8229 err_clear_ingress:
8230         mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8231         return err;
8232 }
8233
8234 static void
8235 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif)
8236 {
8237         mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8238         mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8239 }
8240
8241 static void
8242 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif,
8243                                           struct netdev_notifier_offload_xstats_info *info)
8244 {
8245         if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8246                 return;
8247         netdev_offload_xstats_report_used(info->report_used);
8248 }
8249
8250 static int
8251 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif,
8252                                     struct rtnl_hw_stats64 *p_stats)
8253 {
8254         struct mlxsw_sp_rif_counter_set_basic ingress;
8255         struct mlxsw_sp_rif_counter_set_basic egress;
8256         int err;
8257
8258         err = mlxsw_sp_rif_counter_fetch_clear(rif,
8259                                                MLXSW_SP_RIF_COUNTER_INGRESS,
8260                                                &ingress);
8261         if (err)
8262                 return err;
8263
8264         err = mlxsw_sp_rif_counter_fetch_clear(rif,
8265                                                MLXSW_SP_RIF_COUNTER_EGRESS,
8266                                                &egress);
8267         if (err)
8268                 return err;
8269
8270 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX)              \
8271                 ((SET.good_unicast_ ## SFX) +           \
8272                  (SET.good_multicast_ ## SFX) +         \
8273                  (SET.good_broadcast_ ## SFX))
8274
8275         p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets);
8276         p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets);
8277         p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes);
8278         p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes);
8279         p_stats->rx_errors = ingress.error_packets;
8280         p_stats->tx_errors = egress.error_packets;
8281         p_stats->rx_dropped = ingress.discard_packets;
8282         p_stats->tx_dropped = egress.discard_packets;
8283         p_stats->multicast = ingress.good_multicast_packets +
8284                              ingress.good_broadcast_packets;
8285
8286 #undef MLXSW_SP_ROUTER_ALL_GOOD
8287
8288         return 0;
8289 }
8290
8291 static int
8292 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif,
8293                                            struct netdev_notifier_offload_xstats_info *info)
8294 {
8295         struct rtnl_hw_stats64 stats = {};
8296         int err;
8297
8298         if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8299                 return 0;
8300
8301         err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats);
8302         if (err)
8303                 return err;
8304
8305         netdev_offload_xstats_report_delta(info->report_delta, &stats);
8306         return 0;
8307 }
8308
8309 struct mlxsw_sp_router_hwstats_notify_work {
8310         struct work_struct work;
8311         struct net_device *dev;
8312         netdevice_tracker dev_tracker;
8313 };
8314
8315 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work)
8316 {
8317         struct mlxsw_sp_router_hwstats_notify_work *hws_work =
8318                 container_of(work, struct mlxsw_sp_router_hwstats_notify_work,
8319                              work);
8320
8321         rtnl_lock();
8322         rtnl_offload_xstats_notify(hws_work->dev);
8323         rtnl_unlock();
8324         netdev_put(hws_work->dev, &hws_work->dev_tracker);
8325         kfree(hws_work);
8326 }
8327
8328 static void
8329 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev)
8330 {
8331         struct mlxsw_sp_router_hwstats_notify_work *hws_work;
8332
8333         /* To collect notification payload, the core ends up sending another
8334          * notifier block message, which would deadlock on the attempt to
8335          * acquire the router lock again. Just postpone the notification until
8336          * later.
8337          */
8338
8339         hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL);
8340         if (!hws_work)
8341                 return;
8342
8343         INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work);
8344         netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL);
8345         hws_work->dev = dev;
8346         mlxsw_core_schedule_work(&hws_work->work);
8347 }
8348
8349 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
8350 {
8351         return mlxsw_sp_rif_dev(rif)->ifindex;
8352 }
8353
8354 bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif)
8355 {
8356         return !!mlxsw_sp_rif_dev(rif);
8357 }
8358
8359 bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif,
8360                          const struct net_device *dev)
8361 {
8362         return mlxsw_sp_rif_dev(rif) == dev;
8363 }
8364
8365 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif)
8366 {
8367         struct rtnl_hw_stats64 stats = {};
8368
8369         if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats))
8370                 netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif),
8371                                                  NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8372                                                  &stats);
8373 }
8374
8375 static struct mlxsw_sp_rif *
8376 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
8377                     const struct mlxsw_sp_rif_params *params,
8378                     struct netlink_ext_ack *extack)
8379 {
8380         u8 rif_entries = params->double_entry ? 2 : 1;
8381         u32 tb_id = l3mdev_fib_table(params->dev);
8382         const struct mlxsw_sp_rif_ops *ops;
8383         struct mlxsw_sp_fid *fid = NULL;
8384         enum mlxsw_sp_rif_type type;
8385         struct mlxsw_sp_crif *crif;
8386         struct mlxsw_sp_rif *rif;
8387         struct mlxsw_sp_vr *vr;
8388         u16 rif_index;
8389         int i, err;
8390
8391         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
8392         ops = mlxsw_sp->router->rif_ops_arr[type];
8393
8394         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
8395         if (IS_ERR(vr))
8396                 return ERR_CAST(vr);
8397         vr->rif_count++;
8398
8399         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
8400         if (err) {
8401                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8402                 goto err_rif_index_alloc;
8403         }
8404
8405         crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev);
8406         if (WARN_ON(!crif)) {
8407                 err = -ENOENT;
8408                 goto err_crif_lookup;
8409         }
8410
8411         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif);
8412         if (!rif) {
8413                 err = -ENOMEM;
8414                 goto err_rif_alloc;
8415         }
8416         netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL);
8417         mlxsw_sp->router->rifs[rif_index] = rif;
8418         rif->mlxsw_sp = mlxsw_sp;
8419         rif->ops = ops;
8420         rif->rif_entries = rif_entries;
8421
8422         if (ops->fid_get) {
8423                 fid = ops->fid_get(rif, params, extack);
8424                 if (IS_ERR(fid)) {
8425                         err = PTR_ERR(fid);
8426                         goto err_fid_get;
8427                 }
8428                 rif->fid = fid;
8429         }
8430
8431         if (ops->setup)
8432                 ops->setup(rif, params);
8433
8434         err = ops->configure(rif, extack);
8435         if (err)
8436                 goto err_configure;
8437
8438         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8439                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
8440                 if (err)
8441                         goto err_mr_rif_add;
8442         }
8443
8444         err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif);
8445         if (err)
8446                 goto err_rif_made_sync;
8447
8448         if (netdev_offload_xstats_enabled(params->dev,
8449                                           NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8450                 err = mlxsw_sp_router_port_l3_stats_enable(rif);
8451                 if (err)
8452                         goto err_stats_enable;
8453                 mlxsw_sp_router_hwstats_notify_schedule(params->dev);
8454         } else {
8455                 mlxsw_sp_rif_counters_alloc(rif);
8456         }
8457
8458         atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
8459         return rif;
8460
8461 err_stats_enable:
8462         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8463 err_rif_made_sync:
8464 err_mr_rif_add:
8465         for (i--; i >= 0; i--)
8466                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8467         ops->deconfigure(rif);
8468 err_configure:
8469         if (fid)
8470                 mlxsw_sp_fid_put(fid);
8471 err_fid_get:
8472         mlxsw_sp->router->rifs[rif_index] = NULL;
8473         netdev_put(params->dev, &rif->dev_tracker);
8474         mlxsw_sp_rif_free(rif);
8475 err_rif_alloc:
8476 err_crif_lookup:
8477         mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8478 err_rif_index_alloc:
8479         vr->rif_count--;
8480         mlxsw_sp_vr_put(mlxsw_sp, vr);
8481         return ERR_PTR(err);
8482 }
8483
8484 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
8485 {
8486         struct net_device *dev = mlxsw_sp_rif_dev(rif);
8487         const struct mlxsw_sp_rif_ops *ops = rif->ops;
8488         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8489         struct mlxsw_sp_crif *crif = rif->crif;
8490         struct mlxsw_sp_fid *fid = rif->fid;
8491         u8 rif_entries = rif->rif_entries;
8492         u16 rif_index = rif->rif_index;
8493         struct mlxsw_sp_vr *vr;
8494         int i;
8495
8496         atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
8497         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8498         vr = &mlxsw_sp->router->vrs[rif->vr_id];
8499
8500         if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8501                 mlxsw_sp_rif_push_l3_stats(rif);
8502                 mlxsw_sp_router_port_l3_stats_disable(rif);
8503                 mlxsw_sp_router_hwstats_notify_schedule(dev);
8504         } else {
8505                 mlxsw_sp_rif_counters_free(rif);
8506         }
8507
8508         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
8509                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8510         ops->deconfigure(rif);
8511         if (fid)
8512                 /* Loopback RIFs are not associated with a FID. */
8513                 mlxsw_sp_fid_put(fid);
8514         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
8515         netdev_put(dev, &rif->dev_tracker);
8516         mlxsw_sp_rif_free(rif);
8517         mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8518         vr->rif_count--;
8519         mlxsw_sp_vr_put(mlxsw_sp, vr);
8520
8521         if (crif->can_destroy)
8522                 mlxsw_sp_crif_free(crif);
8523 }
8524
8525 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
8526                                  struct net_device *dev)
8527 {
8528         struct mlxsw_sp_rif *rif;
8529
8530         mutex_lock(&mlxsw_sp->router->lock);
8531         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8532         if (!rif)
8533                 goto out;
8534         mlxsw_sp_rif_destroy(rif);
8535 out:
8536         mutex_unlock(&mlxsw_sp->router->lock);
8537 }
8538
8539 static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp,
8540                                             struct net_device *br_dev,
8541                                             u16 vid)
8542 {
8543         struct net_device *upper_dev;
8544         struct mlxsw_sp_crif *crif;
8545
8546         rcu_read_lock();
8547         upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid);
8548         rcu_read_unlock();
8549
8550         if (!upper_dev)
8551                 return;
8552
8553         crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev);
8554         if (!crif || !crif->rif)
8555                 return;
8556
8557         mlxsw_sp_rif_destroy(crif->rif);
8558 }
8559
8560 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
8561                                           struct net_device *l3_dev,
8562                                           int lower_pvid,
8563                                           unsigned long event,
8564                                           struct netlink_ext_ack *extack);
8565
8566 int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp,
8567                                     struct net_device *br_dev,
8568                                     u16 new_vid, bool is_pvid,
8569                                     struct netlink_ext_ack *extack)
8570 {
8571         struct mlxsw_sp_rif *old_rif;
8572         struct mlxsw_sp_rif *new_rif;
8573         struct net_device *upper_dev;
8574         u16 old_pvid = 0;
8575         u16 new_pvid;
8576         int err = 0;
8577
8578         mutex_lock(&mlxsw_sp->router->lock);
8579         old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev);
8580         if (old_rif) {
8581                 /* If the RIF on the bridge is not a VLAN RIF, we shouldn't have
8582                  * gotten a PVID notification.
8583                  */
8584                 if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN))
8585                         old_rif = NULL;
8586                 else
8587                         old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid);
8588         }
8589
8590         if (is_pvid)
8591                 new_pvid = new_vid;
8592         else if (old_pvid == new_vid)
8593                 new_pvid = 0;
8594         else
8595                 goto out;
8596
8597         if (old_pvid == new_pvid)
8598                 goto out;
8599
8600         if (new_pvid) {
8601                 struct mlxsw_sp_rif_params params = {
8602                         .dev = br_dev,
8603                         .vid = new_pvid,
8604                 };
8605
8606                 /* If there is a VLAN upper with the same VID as the new PVID,
8607                  * kill its RIF, if there is one.
8608                  */
8609                 mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid);
8610
8611                 if (mlxsw_sp_dev_addr_list_empty(br_dev))
8612                         goto out;
8613                 new_rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
8614                 if (IS_ERR(new_rif)) {
8615                         err = PTR_ERR(new_rif);
8616                         goto out;
8617                 }
8618
8619                 if (old_pvid)
8620                         mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif,
8621                                                      true);
8622         } else {
8623                 mlxsw_sp_rif_destroy(old_rif);
8624         }
8625
8626         if (old_pvid) {
8627                 rcu_read_lock();
8628                 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q),
8629                                                      old_pvid);
8630                 rcu_read_unlock();
8631                 if (upper_dev)
8632                         err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp,
8633                                                              upper_dev,
8634                                                              new_pvid,
8635                                                              NETDEV_UP, extack);
8636         }
8637
8638 out:
8639         mutex_unlock(&mlxsw_sp->router->lock);
8640         return err;
8641 }
8642
8643 static void
8644 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
8645                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8646 {
8647         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8648
8649         params->vid = mlxsw_sp_port_vlan->vid;
8650         params->lag = mlxsw_sp_port->lagged;
8651         if (params->lag)
8652                 params->lag_id = mlxsw_sp_port->lag_id;
8653         else
8654                 params->system_port = mlxsw_sp_port->local_port;
8655 }
8656
8657 static struct mlxsw_sp_rif_subport *
8658 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
8659 {
8660         return container_of(rif, struct mlxsw_sp_rif_subport, common);
8661 }
8662
8663 static struct mlxsw_sp_rif *
8664 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
8665                          const struct mlxsw_sp_rif_params *params,
8666                          struct netlink_ext_ack *extack)
8667 {
8668         struct mlxsw_sp_rif_subport *rif_subport;
8669         struct mlxsw_sp_rif *rif;
8670
8671         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
8672         if (!rif)
8673                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
8674
8675         rif_subport = mlxsw_sp_rif_subport_rif(rif);
8676         refcount_inc(&rif_subport->ref_count);
8677         return rif;
8678 }
8679
8680 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
8681 {
8682         struct mlxsw_sp_rif_subport *rif_subport;
8683
8684         rif_subport = mlxsw_sp_rif_subport_rif(rif);
8685         if (!refcount_dec_and_test(&rif_subport->ref_count))
8686                 return;
8687
8688         mlxsw_sp_rif_destroy(rif);
8689 }
8690
8691 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp,
8692                                                 struct mlxsw_sp_rif_mac_profile *profile,
8693                                                 struct netlink_ext_ack *extack)
8694 {
8695         u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
8696         struct mlxsw_sp_router *router = mlxsw_sp->router;
8697         int id;
8698
8699         id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0,
8700                        max_rif_mac_profiles, GFP_KERNEL);
8701
8702         if (id >= 0) {
8703                 profile->id = id;
8704                 return 0;
8705         }
8706
8707         if (id == -ENOSPC)
8708                 NL_SET_ERR_MSG_MOD(extack,
8709                                    "Exceeded number of supported router interface MAC profiles");
8710
8711         return id;
8712 }
8713
8714 static struct mlxsw_sp_rif_mac_profile *
8715 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile)
8716 {
8717         struct mlxsw_sp_rif_mac_profile *profile;
8718
8719         profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr,
8720                              mac_profile);
8721         WARN_ON(!profile);
8722         return profile;
8723 }
8724
8725 static struct mlxsw_sp_rif_mac_profile *
8726 mlxsw_sp_rif_mac_profile_alloc(const char *mac)
8727 {
8728         struct mlxsw_sp_rif_mac_profile *profile;
8729
8730         profile = kzalloc(sizeof(*profile), GFP_KERNEL);
8731         if (!profile)
8732                 return NULL;
8733
8734         ether_addr_copy(profile->mac_prefix, mac);
8735         refcount_set(&profile->ref_count, 1);
8736         return profile;
8737 }
8738
8739 static struct mlxsw_sp_rif_mac_profile *
8740 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac)
8741 {
8742         struct mlxsw_sp_router *router = mlxsw_sp->router;
8743         struct mlxsw_sp_rif_mac_profile *profile;
8744         int id;
8745
8746         idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) {
8747                 if (ether_addr_equal_masked(profile->mac_prefix, mac,
8748                                             mlxsw_sp->mac_mask))
8749                         return profile;
8750         }
8751
8752         return NULL;
8753 }
8754
8755 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv)
8756 {
8757         const struct mlxsw_sp *mlxsw_sp = priv;
8758
8759         return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count);
8760 }
8761
8762 static u64 mlxsw_sp_rifs_occ_get(void *priv)
8763 {
8764         const struct mlxsw_sp *mlxsw_sp = priv;
8765
8766         return atomic_read(&mlxsw_sp->router->rifs_count);
8767 }
8768
8769 static struct mlxsw_sp_rif_mac_profile *
8770 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac,
8771                                 struct netlink_ext_ack *extack)
8772 {
8773         struct mlxsw_sp_rif_mac_profile *profile;
8774         int err;
8775
8776         profile = mlxsw_sp_rif_mac_profile_alloc(mac);
8777         if (!profile)
8778                 return ERR_PTR(-ENOMEM);
8779
8780         err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack);
8781         if (err)
8782                 goto profile_index_alloc_err;
8783
8784         atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count);
8785         return profile;
8786
8787 profile_index_alloc_err:
8788         kfree(profile);
8789         return ERR_PTR(err);
8790 }
8791
8792 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp,
8793                                              u8 mac_profile)
8794 {
8795         struct mlxsw_sp_rif_mac_profile *profile;
8796
8797         atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count);
8798         profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile);
8799         kfree(profile);
8800 }
8801
8802 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp,
8803                                         const char *mac, u8 *p_mac_profile,
8804                                         struct netlink_ext_ack *extack)
8805 {
8806         struct mlxsw_sp_rif_mac_profile *profile;
8807
8808         profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac);
8809         if (profile) {
8810                 refcount_inc(&profile->ref_count);
8811                 goto out;
8812         }
8813
8814         profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack);
8815         if (IS_ERR(profile))
8816                 return PTR_ERR(profile);
8817
8818 out:
8819         *p_mac_profile = profile->id;
8820         return 0;
8821 }
8822
8823 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp,
8824                                          u8 mac_profile)
8825 {
8826         struct mlxsw_sp_rif_mac_profile *profile;
8827
8828         profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8829                            mac_profile);
8830         if (WARN_ON(!profile))
8831                 return;
8832
8833         if (!refcount_dec_and_test(&profile->ref_count))
8834                 return;
8835
8836         mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile);
8837 }
8838
8839 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif)
8840 {
8841         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8842         struct mlxsw_sp_rif_mac_profile *profile;
8843
8844         profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8845                            rif->mac_profile_id);
8846         if (WARN_ON(!profile))
8847                 return false;
8848
8849         return refcount_read(&profile->ref_count) > 1;
8850 }
8851
8852 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif,
8853                                          const char *new_mac)
8854 {
8855         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8856         struct mlxsw_sp_rif_mac_profile *profile;
8857
8858         profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8859                            rif->mac_profile_id);
8860         if (WARN_ON(!profile))
8861                 return -EINVAL;
8862
8863         ether_addr_copy(profile->mac_prefix, new_mac);
8864         return 0;
8865 }
8866
8867 static int
8868 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp,
8869                                  struct mlxsw_sp_rif *rif,
8870                                  const char *new_mac,
8871                                  struct netlink_ext_ack *extack)
8872 {
8873         u8 mac_profile;
8874         int err;
8875
8876         if (!mlxsw_sp_rif_mac_profile_is_shared(rif) &&
8877             !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac))
8878                 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac);
8879
8880         err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac,
8881                                            &mac_profile, extack);
8882         if (err)
8883                 return err;
8884
8885         mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id);
8886         rif->mac_profile_id = mac_profile;
8887         return 0;
8888 }
8889
8890 static int
8891 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8892                                  struct net_device *l3_dev,
8893                                  struct netlink_ext_ack *extack)
8894 {
8895         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8896         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
8897         struct mlxsw_sp_rif_params params;
8898         u16 vid = mlxsw_sp_port_vlan->vid;
8899         struct mlxsw_sp_rif *rif;
8900         struct mlxsw_sp_fid *fid;
8901         int err;
8902
8903         params = (struct mlxsw_sp_rif_params) {
8904                 .dev = l3_dev,
8905                 .vid = vid,
8906         };
8907
8908         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
8909         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
8910         if (IS_ERR(rif))
8911                 return PTR_ERR(rif);
8912
8913         /* FID was already created, just take a reference */
8914         fid = rif->ops->fid_get(rif, &params, extack);
8915         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
8916         if (err)
8917                 goto err_fid_port_vid_map;
8918
8919         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
8920         if (err)
8921                 goto err_port_vid_learning_set;
8922
8923         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
8924                                         BR_STATE_FORWARDING);
8925         if (err)
8926                 goto err_port_vid_stp_set;
8927
8928         mlxsw_sp_port_vlan->fid = fid;
8929
8930         return 0;
8931
8932 err_port_vid_stp_set:
8933         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8934 err_port_vid_learning_set:
8935         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8936 err_fid_port_vid_map:
8937         mlxsw_sp_fid_put(fid);
8938         mlxsw_sp_rif_subport_put(rif);
8939         return err;
8940 }
8941
8942 static void
8943 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8944 {
8945         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8946         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
8947         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
8948         u16 vid = mlxsw_sp_port_vlan->vid;
8949
8950         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
8951                 return;
8952
8953         mlxsw_sp_port_vlan->fid = NULL;
8954         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
8955         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8956         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8957         mlxsw_sp_fid_put(fid);
8958         mlxsw_sp_rif_subport_put(rif);
8959 }
8960
8961 static int
8962 mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8963                                         struct net_device *l3_dev,
8964                                         struct netlink_ext_ack *extack)
8965 {
8966         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8967
8968         lockdep_assert_held(&mlxsw_sp->router->lock);
8969
8970         if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev))
8971                 return 0;
8972
8973         return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
8974                                                 extack);
8975 }
8976
8977 void
8978 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8979 {
8980         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8981
8982         mutex_lock(&mlxsw_sp->router->lock);
8983         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
8984         mutex_unlock(&mlxsw_sp->router->lock);
8985 }
8986
8987 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
8988                                              struct net_device *port_dev,
8989                                              unsigned long event, u16 vid,
8990                                              struct netlink_ext_ack *extack)
8991 {
8992         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
8993         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
8994
8995         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
8996         if (WARN_ON(!mlxsw_sp_port_vlan))
8997                 return -EINVAL;
8998
8999         switch (event) {
9000         case NETDEV_UP:
9001                 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
9002                                                         l3_dev, extack);
9003         case NETDEV_DOWN:
9004                 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9005                 break;
9006         }
9007
9008         return 0;
9009 }
9010
9011 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
9012                                         unsigned long event, bool nomaster,
9013                                         struct netlink_ext_ack *extack)
9014 {
9015         if (!nomaster && (netif_is_any_bridge_port(port_dev) ||
9016                           netif_is_lag_port(port_dev)))
9017                 return 0;
9018
9019         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
9020                                                  MLXSW_SP_DEFAULT_VID, extack);
9021 }
9022
9023 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
9024                                          struct net_device *lag_dev,
9025                                          unsigned long event, u16 vid,
9026                                          struct netlink_ext_ack *extack)
9027 {
9028         struct net_device *port_dev;
9029         struct list_head *iter;
9030         int err;
9031
9032         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
9033                 if (mlxsw_sp_port_dev_check(port_dev)) {
9034                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
9035                                                                 port_dev,
9036                                                                 event, vid,
9037                                                                 extack);
9038                         if (err)
9039                                 return err;
9040                 }
9041         }
9042
9043         return 0;
9044 }
9045
9046 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
9047                                        unsigned long event, bool nomaster,
9048                                        struct netlink_ext_ack *extack)
9049 {
9050         if (!nomaster && netif_is_bridge_port(lag_dev))
9051                 return 0;
9052
9053         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
9054                                              MLXSW_SP_DEFAULT_VID, extack);
9055 }
9056
9057 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
9058                                           struct net_device *l3_dev,
9059                                           int lower_pvid,
9060                                           unsigned long event,
9061                                           struct netlink_ext_ack *extack)
9062 {
9063         struct mlxsw_sp_rif_params params = {
9064                 .dev = l3_dev,
9065         };
9066         struct mlxsw_sp_rif *rif;
9067         int err;
9068
9069         switch (event) {
9070         case NETDEV_UP:
9071                 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) {
9072                         u16 proto;
9073
9074                         br_vlan_get_proto(l3_dev, &proto);
9075                         if (proto == ETH_P_8021AD) {
9076                                 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
9077                                 return -EOPNOTSUPP;
9078                         }
9079                         err = br_vlan_get_pvid(l3_dev, &params.vid);
9080                         if (err)
9081                                 return err;
9082                         if (!params.vid)
9083                                 return 0;
9084                 } else if (is_vlan_dev(l3_dev)) {
9085                         params.vid = vlan_dev_vlan_id(l3_dev);
9086
9087                         /* If the VID matches PVID of the bridge below, the
9088                          * bridge owns the RIF for this VLAN. Don't do anything.
9089                          */
9090                         if ((int)params.vid == lower_pvid)
9091                                 return 0;
9092                 }
9093
9094                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
9095                 if (IS_ERR(rif))
9096                         return PTR_ERR(rif);
9097                 break;
9098         case NETDEV_DOWN:
9099                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9100                 mlxsw_sp_rif_destroy(rif);
9101                 break;
9102         }
9103
9104         return 0;
9105 }
9106
9107 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
9108                                         struct net_device *vlan_dev,
9109                                         unsigned long event, bool nomaster,
9110                                         struct netlink_ext_ack *extack)
9111 {
9112         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
9113         u16 vid = vlan_dev_vlan_id(vlan_dev);
9114         u16 lower_pvid;
9115         int err;
9116
9117         if (!nomaster && netif_is_bridge_port(vlan_dev))
9118                 return 0;
9119
9120         if (mlxsw_sp_port_dev_check(real_dev)) {
9121                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
9122                                                          event, vid, extack);
9123         } else if (netif_is_lag_master(real_dev)) {
9124                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
9125                                                      vid, extack);
9126         } else if (netif_is_bridge_master(real_dev) &&
9127                    br_vlan_enabled(real_dev)) {
9128                 err = br_vlan_get_pvid(real_dev, &lower_pvid);
9129                 if (err)
9130                         return err;
9131                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev,
9132                                                       lower_pvid, event,
9133                                                       extack);
9134         }
9135
9136         return 0;
9137 }
9138
9139 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
9140 {
9141         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
9142         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9143
9144         return ether_addr_equal_masked(mac, vrrp4, mask);
9145 }
9146
9147 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
9148 {
9149         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
9150         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9151
9152         return ether_addr_equal_masked(mac, vrrp6, mask);
9153 }
9154
9155 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9156                                 const u8 *mac, bool adding)
9157 {
9158         char ritr_pl[MLXSW_REG_RITR_LEN];
9159         u8 vrrp_id = adding ? mac[5] : 0;
9160         int err;
9161
9162         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
9163             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
9164                 return 0;
9165
9166         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
9167         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9168         if (err)
9169                 return err;
9170
9171         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
9172                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
9173         else
9174                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
9175
9176         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9177 }
9178
9179 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
9180                                     const struct net_device *macvlan_dev,
9181                                     struct netlink_ext_ack *extack)
9182 {
9183         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
9184         struct mlxsw_sp_rif *rif;
9185         int err;
9186
9187         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
9188         if (!rif)
9189                 return 0;
9190
9191         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9192                                   mlxsw_sp_fid_index(rif->fid), true);
9193         if (err)
9194                 return err;
9195
9196         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
9197                                    macvlan_dev->dev_addr, true);
9198         if (err)
9199                 goto err_rif_vrrp_add;
9200
9201         /* Make sure the bridge driver does not have this MAC pointing at
9202          * some other port.
9203          */
9204         if (rif->ops->fdb_del)
9205                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
9206
9207         return 0;
9208
9209 err_rif_vrrp_add:
9210         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9211                             mlxsw_sp_fid_index(rif->fid), false);
9212         return err;
9213 }
9214
9215 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9216                                        const struct net_device *macvlan_dev)
9217 {
9218         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
9219         struct mlxsw_sp_rif *rif;
9220
9221         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
9222         /* If we do not have a RIF, then we already took care of
9223          * removing the macvlan's MAC during RIF deletion.
9224          */
9225         if (!rif)
9226                 return;
9227         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
9228                              false);
9229         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9230                             mlxsw_sp_fid_index(rif->fid), false);
9231 }
9232
9233 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9234                               const struct net_device *macvlan_dev)
9235 {
9236         mutex_lock(&mlxsw_sp->router->lock);
9237         __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9238         mutex_unlock(&mlxsw_sp->router->lock);
9239 }
9240
9241 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
9242                                            struct net_device *macvlan_dev,
9243                                            unsigned long event,
9244                                            struct netlink_ext_ack *extack)
9245 {
9246         switch (event) {
9247         case NETDEV_UP:
9248                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
9249         case NETDEV_DOWN:
9250                 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9251                 break;
9252         }
9253
9254         return 0;
9255 }
9256
9257 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
9258                                      struct net_device *dev,
9259                                      unsigned long event, bool nomaster,
9260                                      struct netlink_ext_ack *extack)
9261 {
9262         if (mlxsw_sp_port_dev_check(dev))
9263                 return mlxsw_sp_inetaddr_port_event(dev, event, nomaster,
9264                                                     extack);
9265         else if (netif_is_lag_master(dev))
9266                 return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster,
9267                                                    extack);
9268         else if (netif_is_bridge_master(dev))
9269                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event,
9270                                                       extack);
9271         else if (is_vlan_dev(dev))
9272                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
9273                                                     nomaster, extack);
9274         else if (netif_is_macvlan(dev))
9275                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
9276                                                        extack);
9277         else
9278                 return 0;
9279 }
9280
9281 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
9282                                    unsigned long event, void *ptr)
9283 {
9284         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
9285         struct net_device *dev = ifa->ifa_dev->dev;
9286         struct mlxsw_sp_router *router;
9287         struct mlxsw_sp_rif *rif;
9288         int err = 0;
9289
9290         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
9291         if (event == NETDEV_UP)
9292                 return NOTIFY_DONE;
9293
9294         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
9295         mutex_lock(&router->lock);
9296         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
9297         if (!mlxsw_sp_rif_should_config(rif, dev, event))
9298                 goto out;
9299
9300         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false,
9301                                         NULL);
9302 out:
9303         mutex_unlock(&router->lock);
9304         return notifier_from_errno(err);
9305 }
9306
9307 static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
9308                                          unsigned long event, void *ptr)
9309 {
9310         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
9311         struct net_device *dev = ivi->ivi_dev->dev;
9312         struct mlxsw_sp *mlxsw_sp;
9313         struct mlxsw_sp_rif *rif;
9314         int err = 0;
9315
9316         mlxsw_sp = mlxsw_sp_lower_get(dev);
9317         if (!mlxsw_sp)
9318                 return NOTIFY_DONE;
9319
9320         mutex_lock(&mlxsw_sp->router->lock);
9321         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9322         if (!mlxsw_sp_rif_should_config(rif, dev, event))
9323                 goto out;
9324
9325         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false,
9326                                         ivi->extack);
9327 out:
9328         mutex_unlock(&mlxsw_sp->router->lock);
9329         return notifier_from_errno(err);
9330 }
9331
9332 struct mlxsw_sp_inet6addr_event_work {
9333         struct work_struct work;
9334         struct mlxsw_sp *mlxsw_sp;
9335         struct net_device *dev;
9336         netdevice_tracker dev_tracker;
9337         unsigned long event;
9338 };
9339
9340 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
9341 {
9342         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
9343                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
9344         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
9345         struct net_device *dev = inet6addr_work->dev;
9346         unsigned long event = inet6addr_work->event;
9347         struct mlxsw_sp_rif *rif;
9348
9349         rtnl_lock();
9350         mutex_lock(&mlxsw_sp->router->lock);
9351
9352         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9353         if (!mlxsw_sp_rif_should_config(rif, dev, event))
9354                 goto out;
9355
9356         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL);
9357 out:
9358         mutex_unlock(&mlxsw_sp->router->lock);
9359         rtnl_unlock();
9360         netdev_put(dev, &inet6addr_work->dev_tracker);
9361         kfree(inet6addr_work);
9362 }
9363
9364 /* Called with rcu_read_lock() */
9365 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
9366                                     unsigned long event, void *ptr)
9367 {
9368         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
9369         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
9370         struct net_device *dev = if6->idev->dev;
9371         struct mlxsw_sp_router *router;
9372
9373         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
9374         if (event == NETDEV_UP)
9375                 return NOTIFY_DONE;
9376
9377         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
9378         if (!inet6addr_work)
9379                 return NOTIFY_BAD;
9380
9381         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
9382         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
9383         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
9384         inet6addr_work->dev = dev;
9385         inet6addr_work->event = event;
9386         netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC);
9387         mlxsw_core_schedule_work(&inet6addr_work->work);
9388
9389         return NOTIFY_DONE;
9390 }
9391
9392 static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
9393                                           unsigned long event, void *ptr)
9394 {
9395         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
9396         struct net_device *dev = i6vi->i6vi_dev->dev;
9397         struct mlxsw_sp *mlxsw_sp;
9398         struct mlxsw_sp_rif *rif;
9399         int err = 0;
9400
9401         mlxsw_sp = mlxsw_sp_lower_get(dev);
9402         if (!mlxsw_sp)
9403                 return NOTIFY_DONE;
9404
9405         mutex_lock(&mlxsw_sp->router->lock);
9406         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9407         if (!mlxsw_sp_rif_should_config(rif, dev, event))
9408                 goto out;
9409
9410         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false,
9411                                         i6vi->extack);
9412 out:
9413         mutex_unlock(&mlxsw_sp->router->lock);
9414         return notifier_from_errno(err);
9415 }
9416
9417 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9418                              const char *mac, int mtu, u8 mac_profile)
9419 {
9420         char ritr_pl[MLXSW_REG_RITR_LEN];
9421         int err;
9422
9423         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
9424         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9425         if (err)
9426                 return err;
9427
9428         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
9429         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
9430         mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile);
9431         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
9432         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9433 }
9434
9435 static int
9436 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
9437                                   struct mlxsw_sp_rif *rif,
9438                                   struct netlink_ext_ack *extack)
9439 {
9440         struct net_device *dev = mlxsw_sp_rif_dev(rif);
9441         u8 old_mac_profile;
9442         u16 fid_index;
9443         int err;
9444
9445         fid_index = mlxsw_sp_fid_index(rif->fid);
9446
9447         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
9448         if (err)
9449                 return err;
9450
9451         old_mac_profile = rif->mac_profile_id;
9452         err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr,
9453                                                extack);
9454         if (err)
9455                 goto err_rif_mac_profile_replace;
9456
9457         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
9458                                 dev->mtu, rif->mac_profile_id);
9459         if (err)
9460                 goto err_rif_edit;
9461
9462         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
9463         if (err)
9464                 goto err_rif_fdb_op;
9465
9466         if (rif->mtu != dev->mtu) {
9467                 struct mlxsw_sp_vr *vr;
9468                 int i;
9469
9470                 /* The RIF is relevant only to its mr_table instance, as unlike
9471                  * unicast routing, in multicast routing a RIF cannot be shared
9472                  * between several multicast routing tables.
9473                  */
9474                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
9475                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
9476                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
9477                                                    rif, dev->mtu);
9478         }
9479
9480         ether_addr_copy(rif->addr, dev->dev_addr);
9481         rif->mtu = dev->mtu;
9482
9483         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
9484
9485         return 0;
9486
9487 err_rif_fdb_op:
9488         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu,
9489                           old_mac_profile);
9490 err_rif_edit:
9491         mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack);
9492 err_rif_mac_profile_replace:
9493         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
9494         return err;
9495 }
9496
9497 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
9498                             struct netdev_notifier_pre_changeaddr_info *info)
9499 {
9500         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9501         struct mlxsw_sp_rif_mac_profile *profile;
9502         struct netlink_ext_ack *extack;
9503         u8 max_rif_mac_profiles;
9504         u64 occ;
9505
9506         extack = netdev_notifier_info_to_extack(&info->info);
9507
9508         profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr);
9509         if (profile)
9510                 return 0;
9511
9512         max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
9513         occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp);
9514         if (occ < max_rif_mac_profiles)
9515                 return 0;
9516
9517         if (!mlxsw_sp_rif_mac_profile_is_shared(rif))
9518                 return 0;
9519
9520         NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles");
9521         return -ENOBUFS;
9522 }
9523
9524 static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp,
9525                                                   struct net_device *dev)
9526 {
9527         struct vlan_dev_priv *vlan;
9528
9529         if (netif_is_lag_master(dev) ||
9530             netif_is_bridge_master(dev) ||
9531             mlxsw_sp_port_dev_check(dev) ||
9532             mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) ||
9533             netif_is_l3_master(dev))
9534                 return true;
9535
9536         if (!is_vlan_dev(dev))
9537                 return false;
9538
9539         vlan = vlan_dev_priv(dev);
9540         return netif_is_lag_master(vlan->real_dev) ||
9541                netif_is_bridge_master(vlan->real_dev) ||
9542                mlxsw_sp_port_dev_check(vlan->real_dev);
9543 }
9544
9545 static struct mlxsw_sp_crif *
9546 mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev)
9547 {
9548         struct mlxsw_sp_crif *crif;
9549         int err;
9550
9551         if (WARN_ON(mlxsw_sp_crif_lookup(router, dev)))
9552                 return NULL;
9553
9554         crif = mlxsw_sp_crif_alloc(dev);
9555         if (!crif)
9556                 return ERR_PTR(-ENOMEM);
9557
9558         err = mlxsw_sp_crif_insert(router, crif);
9559         if (err)
9560                 goto err_netdev_insert;
9561
9562         return crif;
9563
9564 err_netdev_insert:
9565         mlxsw_sp_crif_free(crif);
9566         return ERR_PTR(err);
9567 }
9568
9569 static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router,
9570                                      struct mlxsw_sp_crif *crif)
9571 {
9572         struct mlxsw_sp_nexthop *nh, *tmp;
9573
9574         mlxsw_sp_crif_remove(router, crif);
9575
9576         list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node)
9577                 mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh);
9578
9579         if (crif->rif)
9580                 crif->can_destroy = true;
9581         else
9582                 mlxsw_sp_crif_free(crif);
9583 }
9584
9585 static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router,
9586                                        struct net_device *dev)
9587 {
9588         struct mlxsw_sp_crif *crif;
9589
9590         if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev))
9591                 return 0;
9592
9593         crif = mlxsw_sp_crif_register(router, dev);
9594         return PTR_ERR_OR_ZERO(crif);
9595 }
9596
9597 static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router,
9598                                           struct net_device *dev)
9599 {
9600         struct mlxsw_sp_crif *crif;
9601
9602         if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev))
9603                 return;
9604
9605         /* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts
9606          * the NETDEV_UNREGISTER message, so we can get here twice. If that's
9607          * what happened, the netdevice state is NETREG_UNREGISTERED. In that
9608          * case, we expect to have collected the CRIF already, and warn if it
9609          * still exists. Otherwise we expect the CRIF to exist.
9610          */
9611         crif = mlxsw_sp_crif_lookup(router, dev);
9612         if (dev->reg_state == NETREG_UNREGISTERED) {
9613                 if (!WARN_ON(crif))
9614                         return;
9615         }
9616         if (WARN_ON(!crif))
9617                 return;
9618
9619         mlxsw_sp_crif_unregister(router, crif);
9620 }
9621
9622 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event)
9623 {
9624         switch (event) {
9625         case NETDEV_OFFLOAD_XSTATS_ENABLE:
9626         case NETDEV_OFFLOAD_XSTATS_DISABLE:
9627         case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9628         case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9629                 return true;
9630         }
9631
9632         return false;
9633 }
9634
9635 static int
9636 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif,
9637                                         unsigned long event,
9638                                         struct netdev_notifier_offload_xstats_info *info)
9639 {
9640         switch (info->type) {
9641         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9642                 break;
9643         default:
9644                 return 0;
9645         }
9646
9647         switch (event) {
9648         case NETDEV_OFFLOAD_XSTATS_ENABLE:
9649                 return mlxsw_sp_router_port_l3_stats_enable(rif);
9650         case NETDEV_OFFLOAD_XSTATS_DISABLE:
9651                 mlxsw_sp_router_port_l3_stats_disable(rif);
9652                 return 0;
9653         case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9654                 mlxsw_sp_router_port_l3_stats_report_used(rif, info);
9655                 return 0;
9656         case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9657                 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info);
9658         }
9659
9660         WARN_ON_ONCE(1);
9661         return 0;
9662 }
9663
9664 static int
9665 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp,
9666                                       struct net_device *dev,
9667                                       unsigned long event,
9668                                       struct netdev_notifier_offload_xstats_info *info)
9669 {
9670         struct mlxsw_sp_rif *rif;
9671
9672         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9673         if (!rif)
9674                 return 0;
9675
9676         return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info);
9677 }
9678
9679 static bool mlxsw_sp_is_router_event(unsigned long event)
9680 {
9681         switch (event) {
9682         case NETDEV_PRE_CHANGEADDR:
9683         case NETDEV_CHANGEADDR:
9684         case NETDEV_CHANGEMTU:
9685                 return true;
9686         default:
9687                 return false;
9688         }
9689 }
9690
9691 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
9692                                                 unsigned long event, void *ptr)
9693 {
9694         struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
9695         struct mlxsw_sp *mlxsw_sp;
9696         struct mlxsw_sp_rif *rif;
9697
9698         mlxsw_sp = mlxsw_sp_lower_get(dev);
9699         if (!mlxsw_sp)
9700                 return 0;
9701
9702         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9703         if (!rif)
9704                 return 0;
9705
9706         switch (event) {
9707         case NETDEV_CHANGEMTU:
9708         case NETDEV_CHANGEADDR:
9709                 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack);
9710         case NETDEV_PRE_CHANGEADDR:
9711                 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
9712         default:
9713                 WARN_ON_ONCE(1);
9714                 break;
9715         }
9716
9717         return 0;
9718 }
9719
9720 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
9721                                   struct net_device *l3_dev,
9722                                   struct netlink_ext_ack *extack)
9723 {
9724         struct mlxsw_sp_rif *rif;
9725
9726         /* If netdev is already associated with a RIF, then we need to
9727          * destroy it and create a new one with the new virtual router ID.
9728          */
9729         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9730         if (rif)
9731                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false,
9732                                           extack);
9733
9734         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false,
9735                                          extack);
9736 }
9737
9738 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
9739                                     struct net_device *l3_dev)
9740 {
9741         struct mlxsw_sp_rif *rif;
9742
9743         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9744         if (!rif)
9745                 return;
9746         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL);
9747 }
9748
9749 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr)
9750 {
9751         struct netdev_notifier_changeupper_info *info = ptr;
9752
9753         if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER)
9754                 return false;
9755         return netif_is_l3_master(info->upper_dev);
9756 }
9757
9758 static int
9759 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
9760                              struct netdev_notifier_changeupper_info *info)
9761 {
9762         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
9763         int err = 0;
9764
9765         /* We do not create a RIF for a macvlan, but only use it to
9766          * direct more MAC addresses to the router.
9767          */
9768         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
9769                 return 0;
9770
9771         switch (event) {
9772         case NETDEV_PRECHANGEUPPER:
9773                 break;
9774         case NETDEV_CHANGEUPPER:
9775                 if (info->linking) {
9776                         struct netlink_ext_ack *extack;
9777
9778                         extack = netdev_notifier_info_to_extack(&info->info);
9779                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
9780                 } else {
9781                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
9782                 }
9783                 break;
9784         }
9785
9786         return err;
9787 }
9788
9789 struct mlxsw_sp_router_replay_inetaddr_up {
9790         struct mlxsw_sp *mlxsw_sp;
9791         struct netlink_ext_ack *extack;
9792         unsigned int done;
9793         bool deslavement;
9794 };
9795
9796 static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev,
9797                                               struct netdev_nested_priv *priv)
9798 {
9799         struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9800         bool nomaster = ctx->deslavement;
9801         struct mlxsw_sp_crif *crif;
9802         int err;
9803
9804         if (mlxsw_sp_dev_addr_list_empty(dev))
9805                 return 0;
9806
9807         crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev);
9808         if (!crif || crif->rif)
9809                 return 0;
9810
9811         if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP))
9812                 return 0;
9813
9814         err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP,
9815                                         nomaster, ctx->extack);
9816         if (err)
9817                 return err;
9818
9819         ctx->done++;
9820         return 0;
9821 }
9822
9823 static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev,
9824                                                 struct netdev_nested_priv *priv)
9825 {
9826         struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9827         bool nomaster = ctx->deslavement;
9828         struct mlxsw_sp_crif *crif;
9829
9830         if (!ctx->done)
9831                 return 0;
9832
9833         if (mlxsw_sp_dev_addr_list_empty(dev))
9834                 return 0;
9835
9836         crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev);
9837         if (!crif || !crif->rif)
9838                 return 0;
9839
9840         /* We are rolling back NETDEV_UP, so ask for that. */
9841         if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP))
9842                 return 0;
9843
9844         __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster,
9845                                   NULL);
9846
9847         ctx->done--;
9848         return 0;
9849 }
9850
9851 int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp,
9852                                           struct net_device *upper_dev,
9853                                           struct netlink_ext_ack *extack)
9854 {
9855         struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9856                 .mlxsw_sp = mlxsw_sp,
9857                 .extack = extack,
9858                 .deslavement = false,
9859         };
9860         struct netdev_nested_priv priv = {
9861                 .data = &ctx,
9862         };
9863         int err;
9864
9865         err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv);
9866         if (err)
9867                 return err;
9868
9869         err = netdev_walk_all_upper_dev_rcu(upper_dev,
9870                                             mlxsw_sp_router_replay_inetaddr_up,
9871                                             &priv);
9872         if (err)
9873                 goto err_replay_up;
9874
9875         return 0;
9876
9877 err_replay_up:
9878         netdev_walk_all_upper_dev_rcu(upper_dev,
9879                                       mlxsw_sp_router_unreplay_inetaddr_up,
9880                                       &priv);
9881         mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv);
9882         return err;
9883 }
9884
9885 void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp,
9886                                            struct net_device *dev)
9887 {
9888         struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9889                 .mlxsw_sp = mlxsw_sp,
9890                 .deslavement = true,
9891         };
9892         struct netdev_nested_priv priv = {
9893                 .data = &ctx,
9894         };
9895
9896         mlxsw_sp_router_replay_inetaddr_up(dev, &priv);
9897 }
9898
9899 static int
9900 mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port,
9901                                        u16 vid, struct net_device *dev,
9902                                        struct netlink_ext_ack *extack)
9903 {
9904         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9905
9906         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9907                                                             vid);
9908         if (WARN_ON(!mlxsw_sp_port_vlan))
9909                 return -EINVAL;
9910
9911         return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan,
9912                                                        dev, extack);
9913 }
9914
9915 static void
9916 mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
9917                                struct net_device *dev)
9918 {
9919         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9920
9921         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9922                                                             vid);
9923         if (WARN_ON(!mlxsw_sp_port_vlan))
9924                 return;
9925
9926         __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9927 }
9928
9929 static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9930                                            struct net_device *lag_dev,
9931                                            struct netlink_ext_ack *extack)
9932 {
9933         u16 default_vid = MLXSW_SP_DEFAULT_VID;
9934         struct net_device *upper_dev;
9935         struct list_head *iter;
9936         int done = 0;
9937         u16 vid;
9938         int err;
9939
9940         err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid,
9941                                                      lag_dev, extack);
9942         if (err)
9943                 return err;
9944
9945         netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9946                 if (!is_vlan_dev(upper_dev))
9947                         continue;
9948
9949                 vid = vlan_dev_vlan_id(upper_dev);
9950                 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid,
9951                                                              upper_dev, extack);
9952                 if (err)
9953                         goto err_router_join_dev;
9954
9955                 ++done;
9956         }
9957
9958         return 0;
9959
9960 err_router_join_dev:
9961         netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9962                 if (!is_vlan_dev(upper_dev))
9963                         continue;
9964                 if (!done--)
9965                         break;
9966
9967                 vid = vlan_dev_vlan_id(upper_dev);
9968                 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev);
9969         }
9970
9971         mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev);
9972         return err;
9973 }
9974
9975 static void
9976 __mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9977                                  struct net_device *lag_dev)
9978 {
9979         u16 default_vid = MLXSW_SP_DEFAULT_VID;
9980         struct net_device *upper_dev;
9981         struct list_head *iter;
9982         u16 vid;
9983
9984         netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9985                 if (!is_vlan_dev(upper_dev))
9986                         continue;
9987
9988                 vid = vlan_dev_vlan_id(upper_dev);
9989                 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev);
9990         }
9991
9992         mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev);
9993 }
9994
9995 int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9996                                   struct net_device *lag_dev,
9997                                   struct netlink_ext_ack *extack)
9998 {
9999         int err;
10000
10001         mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10002         err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack);
10003         mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10004
10005         return err;
10006 }
10007
10008 void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
10009                                     struct net_device *lag_dev)
10010 {
10011         mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10012         __mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev);
10013         mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10014 }
10015
10016 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb,
10017                                            unsigned long event, void *ptr)
10018 {
10019         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
10020         struct mlxsw_sp_router *router;
10021         struct mlxsw_sp *mlxsw_sp;
10022         int err = 0;
10023
10024         router = container_of(nb, struct mlxsw_sp_router, netdevice_nb);
10025         mlxsw_sp = router->mlxsw_sp;
10026
10027         mutex_lock(&mlxsw_sp->router->lock);
10028
10029         if (event == NETDEV_REGISTER) {
10030                 err = mlxsw_sp_netdevice_register(router, dev);
10031                 if (err)
10032                         /* No need to roll this back, UNREGISTER will collect it
10033                          * anyhow.
10034                          */
10035                         goto out;
10036         }
10037
10038         if (mlxsw_sp_is_offload_xstats_event(event))
10039                 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev,
10040                                                             event, ptr);
10041         else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev))
10042                 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev,
10043                                                        event, ptr);
10044         else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev))
10045                 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev,
10046                                                        event, ptr);
10047         else if (mlxsw_sp_is_router_event(event))
10048                 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr);
10049         else if (mlxsw_sp_is_vrf_event(event, ptr))
10050                 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr);
10051
10052         if (event == NETDEV_UNREGISTER)
10053                 mlxsw_sp_netdevice_unregister(router, dev);
10054
10055 out:
10056         mutex_unlock(&mlxsw_sp->router->lock);
10057
10058         return notifier_from_errno(err);
10059 }
10060
10061 struct mlxsw_sp_macvlan_replay {
10062         struct mlxsw_sp *mlxsw_sp;
10063         struct netlink_ext_ack *extack;
10064 };
10065
10066 static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev,
10067                                          struct netdev_nested_priv *priv)
10068 {
10069         const struct mlxsw_sp_macvlan_replay *rms = priv->data;
10070         struct netlink_ext_ack *extack = rms->extack;
10071         struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp;
10072
10073         if (!netif_is_macvlan(dev))
10074                 return 0;
10075
10076         return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack);
10077 }
10078
10079 static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif,
10080                                    struct netlink_ext_ack *extack)
10081 {
10082         struct mlxsw_sp_macvlan_replay rms = {
10083                 .mlxsw_sp = rif->mlxsw_sp,
10084                 .extack = extack,
10085         };
10086         struct netdev_nested_priv priv = {
10087                 .data = &rms,
10088         };
10089
10090         return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif),
10091                                              mlxsw_sp_macvlan_replay_upper,
10092                                              &priv);
10093 }
10094
10095 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
10096                                         struct netdev_nested_priv *priv)
10097 {
10098         struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
10099
10100         if (!netif_is_macvlan(dev))
10101                 return 0;
10102
10103         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10104                                    mlxsw_sp_fid_index(rif->fid), false);
10105 }
10106
10107 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
10108 {
10109         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10110         struct netdev_nested_priv priv = {
10111                 .data = (void *)rif,
10112         };
10113
10114         if (!netif_is_macvlan_port(dev))
10115                 return 0;
10116
10117         return netdev_walk_all_upper_dev_rcu(dev,
10118                                              __mlxsw_sp_rif_macvlan_flush, &priv);
10119 }
10120
10121 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
10122                                        const struct mlxsw_sp_rif_params *params)
10123 {
10124         struct mlxsw_sp_rif_subport *rif_subport;
10125
10126         rif_subport = mlxsw_sp_rif_subport_rif(rif);
10127         refcount_set(&rif_subport->ref_count, 1);
10128         rif_subport->vid = params->vid;
10129         rif_subport->lag = params->lag;
10130         if (params->lag)
10131                 rif_subport->lag_id = params->lag_id;
10132         else
10133                 rif_subport->system_port = params->system_port;
10134 }
10135
10136 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
10137 {
10138         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10139         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10140         struct mlxsw_sp_rif_subport *rif_subport;
10141         char ritr_pl[MLXSW_REG_RITR_LEN];
10142         u16 efid;
10143
10144         rif_subport = mlxsw_sp_rif_subport_rif(rif);
10145         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
10146                             rif->rif_index, rif->vr_id, dev->mtu);
10147         mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr);
10148         mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
10149         efid = mlxsw_sp_fid_index(rif->fid);
10150         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
10151                                   rif_subport->lag ? rif_subport->lag_id :
10152                                                      rif_subport->system_port,
10153                                   efid, 0);
10154         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10155 }
10156
10157 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif,
10158                                           struct netlink_ext_ack *extack)
10159 {
10160         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10161         u8 mac_profile;
10162         int err;
10163
10164         err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr,
10165                                            &mac_profile, extack);
10166         if (err)
10167                 return err;
10168         rif->mac_profile_id = mac_profile;
10169
10170         err = mlxsw_sp_rif_subport_op(rif, true);
10171         if (err)
10172                 goto err_rif_subport_op;
10173
10174         err = mlxsw_sp_macvlan_replay(rif, extack);
10175         if (err)
10176                 goto err_macvlan_replay;
10177
10178         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10179                                   mlxsw_sp_fid_index(rif->fid), true);
10180         if (err)
10181                 goto err_rif_fdb_op;
10182
10183         err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10184         if (err)
10185                 goto err_fid_rif_set;
10186
10187         return 0;
10188
10189 err_fid_rif_set:
10190         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10191                             mlxsw_sp_fid_index(rif->fid), false);
10192 err_rif_fdb_op:
10193         mlxsw_sp_rif_macvlan_flush(rif);
10194 err_macvlan_replay:
10195         mlxsw_sp_rif_subport_op(rif, false);
10196 err_rif_subport_op:
10197         mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile);
10198         return err;
10199 }
10200
10201 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
10202 {
10203         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10204         struct mlxsw_sp_fid *fid = rif->fid;
10205
10206         mlxsw_sp_fid_rif_unset(fid);
10207         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10208                             mlxsw_sp_fid_index(fid), false);
10209         mlxsw_sp_rif_macvlan_flush(rif);
10210         mlxsw_sp_rif_subport_op(rif, false);
10211         mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10212 }
10213
10214 static struct mlxsw_sp_fid *
10215 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
10216                              const struct mlxsw_sp_rif_params *params,
10217                              struct netlink_ext_ack *extack)
10218 {
10219         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
10220 }
10221
10222 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
10223         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
10224         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
10225         .setup                  = mlxsw_sp_rif_subport_setup,
10226         .configure              = mlxsw_sp_rif_subport_configure,
10227         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
10228         .fid_get                = mlxsw_sp_rif_subport_fid_get,
10229 };
10230
10231 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable)
10232 {
10233         enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF;
10234         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10235         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10236         char ritr_pl[MLXSW_REG_RITR_LEN];
10237
10238         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
10239                             dev->mtu);
10240         mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr);
10241         mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
10242         mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid);
10243
10244         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10245 }
10246
10247 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
10248 {
10249         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
10250 }
10251
10252 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif,
10253                                       struct netlink_ext_ack *extack)
10254 {
10255         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10256         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10257         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
10258         u8 mac_profile;
10259         int err;
10260
10261         err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
10262                                            &mac_profile, extack);
10263         if (err)
10264                 return err;
10265         rif->mac_profile_id = mac_profile;
10266
10267         err = mlxsw_sp_rif_fid_op(rif, fid_index, true);
10268         if (err)
10269                 goto err_rif_fid_op;
10270
10271         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10272                                      mlxsw_sp_router_port(mlxsw_sp), true);
10273         if (err)
10274                 goto err_fid_mc_flood_set;
10275
10276         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10277                                      mlxsw_sp_router_port(mlxsw_sp), true);
10278         if (err)
10279                 goto err_fid_bc_flood_set;
10280
10281         err = mlxsw_sp_macvlan_replay(rif, extack);
10282         if (err)
10283                 goto err_macvlan_replay;
10284
10285         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10286                                   mlxsw_sp_fid_index(rif->fid), true);
10287         if (err)
10288                 goto err_rif_fdb_op;
10289
10290         err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10291         if (err)
10292                 goto err_fid_rif_set;
10293
10294         return 0;
10295
10296 err_fid_rif_set:
10297         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10298                             mlxsw_sp_fid_index(rif->fid), false);
10299 err_rif_fdb_op:
10300         mlxsw_sp_rif_macvlan_flush(rif);
10301 err_macvlan_replay:
10302         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10303                                mlxsw_sp_router_port(mlxsw_sp), false);
10304 err_fid_bc_flood_set:
10305         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10306                                mlxsw_sp_router_port(mlxsw_sp), false);
10307 err_fid_mc_flood_set:
10308         mlxsw_sp_rif_fid_op(rif, fid_index, false);
10309 err_rif_fid_op:
10310         mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10311         return err;
10312 }
10313
10314 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
10315 {
10316         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10317         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
10318         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10319         struct mlxsw_sp_fid *fid = rif->fid;
10320
10321         mlxsw_sp_fid_rif_unset(fid);
10322         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10323                             mlxsw_sp_fid_index(fid), false);
10324         mlxsw_sp_rif_macvlan_flush(rif);
10325         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10326                                mlxsw_sp_router_port(mlxsw_sp), false);
10327         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10328                                mlxsw_sp_router_port(mlxsw_sp), false);
10329         mlxsw_sp_rif_fid_op(rif, fid_index, false);
10330         mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10331 }
10332
10333 static struct mlxsw_sp_fid *
10334 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
10335                          const struct mlxsw_sp_rif_params *params,
10336                          struct netlink_ext_ack *extack)
10337 {
10338         int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif);
10339
10340         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex);
10341 }
10342
10343 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10344 {
10345         struct switchdev_notifier_fdb_info info = {};
10346         struct net_device *dev;
10347
10348         dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0);
10349         if (!dev)
10350                 return;
10351
10352         info.addr = mac;
10353         info.vid = 0;
10354         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
10355                                  NULL);
10356 }
10357
10358 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
10359         .type                   = MLXSW_SP_RIF_TYPE_FID,
10360         .rif_size               = sizeof(struct mlxsw_sp_rif),
10361         .configure              = mlxsw_sp_rif_fid_configure,
10362         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
10363         .fid_get                = mlxsw_sp_rif_fid_fid_get,
10364         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
10365 };
10366
10367 static struct mlxsw_sp_fid *
10368 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
10369                           const struct mlxsw_sp_rif_params *params,
10370                           struct netlink_ext_ack *extack)
10371 {
10372         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10373         struct net_device *br_dev;
10374
10375         if (WARN_ON(!params->vid))
10376                 return ERR_PTR(-EINVAL);
10377
10378         if (is_vlan_dev(dev)) {
10379                 br_dev = vlan_dev_real_dev(dev);
10380                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
10381                         return ERR_PTR(-EINVAL);
10382         }
10383
10384         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid);
10385 }
10386
10387 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10388 {
10389         struct net_device *rif_dev = mlxsw_sp_rif_dev(rif);
10390         struct switchdev_notifier_fdb_info info = {};
10391         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10392         struct net_device *br_dev;
10393         struct net_device *dev;
10394
10395         br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev;
10396         dev = br_fdb_find_port(br_dev, mac, vid);
10397         if (!dev)
10398                 return;
10399
10400         info.addr = mac;
10401         info.vid = vid;
10402         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
10403                                  NULL);
10404 }
10405
10406 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid,
10407                                 bool enable)
10408 {
10409         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10410         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10411         char ritr_pl[MLXSW_REG_RITR_LEN];
10412
10413         mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id,
10414                                     dev->mtu, dev->dev_addr,
10415                                     rif->mac_profile_id, vid, efid);
10416
10417         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10418 }
10419
10420 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid,
10421                                        struct netlink_ext_ack *extack)
10422 {
10423         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10424         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10425         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10426         u8 mac_profile;
10427         int err;
10428
10429         err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
10430                                            &mac_profile, extack);
10431         if (err)
10432                 return err;
10433         rif->mac_profile_id = mac_profile;
10434
10435         err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true);
10436         if (err)
10437                 goto err_rif_vlan_fid_op;
10438
10439         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10440                                      mlxsw_sp_router_port(mlxsw_sp), true);
10441         if (err)
10442                 goto err_fid_mc_flood_set;
10443
10444         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10445                                      mlxsw_sp_router_port(mlxsw_sp), true);
10446         if (err)
10447                 goto err_fid_bc_flood_set;
10448
10449         err = mlxsw_sp_macvlan_replay(rif, extack);
10450         if (err)
10451                 goto err_macvlan_replay;
10452
10453         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10454                                   mlxsw_sp_fid_index(rif->fid), true);
10455         if (err)
10456                 goto err_rif_fdb_op;
10457
10458         err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10459         if (err)
10460                 goto err_fid_rif_set;
10461
10462         return 0;
10463
10464 err_fid_rif_set:
10465         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10466                             mlxsw_sp_fid_index(rif->fid), false);
10467 err_rif_fdb_op:
10468         mlxsw_sp_rif_macvlan_flush(rif);
10469 err_macvlan_replay:
10470         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10471                                mlxsw_sp_router_port(mlxsw_sp), false);
10472 err_fid_bc_flood_set:
10473         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10474                                mlxsw_sp_router_port(mlxsw_sp), false);
10475 err_fid_mc_flood_set:
10476         mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
10477 err_rif_vlan_fid_op:
10478         mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10479         return err;
10480 }
10481
10482 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
10483 {
10484         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10485         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10486         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10487
10488         mlxsw_sp_fid_rif_unset(rif->fid);
10489         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10490                             mlxsw_sp_fid_index(rif->fid), false);
10491         mlxsw_sp_rif_macvlan_flush(rif);
10492         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10493                                mlxsw_sp_router_port(mlxsw_sp), false);
10494         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10495                                mlxsw_sp_router_port(mlxsw_sp), false);
10496         mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
10497         mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10498 }
10499
10500 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10501                                         struct netlink_ext_ack *extack)
10502 {
10503         return mlxsw_sp_rif_vlan_configure(rif, 0, extack);
10504 }
10505
10506 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = {
10507         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
10508         .rif_size               = sizeof(struct mlxsw_sp_rif),
10509         .configure              = mlxsw_sp1_rif_vlan_configure,
10510         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
10511         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
10512         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
10513 };
10514
10515 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10516                                         struct netlink_ext_ack *extack)
10517 {
10518         u16 efid = mlxsw_sp_fid_index(rif->fid);
10519
10520         return mlxsw_sp_rif_vlan_configure(rif, efid, extack);
10521 }
10522
10523 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = {
10524         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
10525         .rif_size               = sizeof(struct mlxsw_sp_rif),
10526         .configure              = mlxsw_sp2_rif_vlan_configure,
10527         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
10528         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
10529         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
10530 };
10531
10532 static struct mlxsw_sp_rif_ipip_lb *
10533 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
10534 {
10535         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
10536 }
10537
10538 static void
10539 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
10540                            const struct mlxsw_sp_rif_params *params)
10541 {
10542         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
10543         struct mlxsw_sp_rif_ipip_lb *rif_lb;
10544
10545         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
10546                                  common);
10547         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
10548         rif_lb->lb_config = params_lb->lb_config;
10549 }
10550
10551 static int
10552 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10553                                 struct netlink_ext_ack *extack)
10554 {
10555         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10556         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10557         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
10558         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10559         struct mlxsw_sp_vr *ul_vr;
10560         int err;
10561
10562         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack);
10563         if (IS_ERR(ul_vr))
10564                 return PTR_ERR(ul_vr);
10565
10566         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
10567         if (err)
10568                 goto err_loopback_op;
10569
10570         lb_rif->ul_vr_id = ul_vr->id;
10571         lb_rif->ul_rif_id = 0;
10572         ++ul_vr->rif_count;
10573         return 0;
10574
10575 err_loopback_op:
10576         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
10577         return err;
10578 }
10579
10580 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10581 {
10582         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10583         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10584         struct mlxsw_sp_vr *ul_vr;
10585
10586         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
10587         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
10588
10589         --ul_vr->rif_count;
10590         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
10591 }
10592
10593 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
10594         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
10595         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
10596         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
10597         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
10598         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
10599 };
10600
10601 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
10602         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
10603         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp1_rif_vlan_ops,
10604         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
10605         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
10606 };
10607
10608 static int
10609 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
10610 {
10611         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10612         char ritr_pl[MLXSW_REG_RITR_LEN];
10613
10614         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
10615                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
10616         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
10617                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
10618
10619         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10620 }
10621
10622 static struct mlxsw_sp_rif *
10623 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
10624                        struct mlxsw_sp_crif *ul_crif,
10625                        struct netlink_ext_ack *extack)
10626 {
10627         struct mlxsw_sp_rif *ul_rif;
10628         u8 rif_entries = 1;
10629         u16 rif_index;
10630         int err;
10631
10632         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
10633         if (err) {
10634                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
10635                 return ERR_PTR(err);
10636         }
10637
10638         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id,
10639                                     ul_crif);
10640         if (!ul_rif) {
10641                 err = -ENOMEM;
10642                 goto err_rif_alloc;
10643         }
10644
10645         mlxsw_sp->router->rifs[rif_index] = ul_rif;
10646         ul_rif->mlxsw_sp = mlxsw_sp;
10647         ul_rif->rif_entries = rif_entries;
10648         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
10649         if (err)
10650                 goto ul_rif_op_err;
10651
10652         atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
10653         return ul_rif;
10654
10655 ul_rif_op_err:
10656         mlxsw_sp->router->rifs[rif_index] = NULL;
10657         mlxsw_sp_rif_free(ul_rif);
10658 err_rif_alloc:
10659         mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10660         return ERR_PTR(err);
10661 }
10662
10663 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
10664 {
10665         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10666         u8 rif_entries = ul_rif->rif_entries;
10667         u16 rif_index = ul_rif->rif_index;
10668
10669         atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
10670         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
10671         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
10672         mlxsw_sp_rif_free(ul_rif);
10673         mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10674 }
10675
10676 static struct mlxsw_sp_rif *
10677 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
10678                     struct mlxsw_sp_crif *ul_crif,
10679                     struct netlink_ext_ack *extack)
10680 {
10681         struct mlxsw_sp_vr *vr;
10682         int err;
10683
10684         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
10685         if (IS_ERR(vr))
10686                 return ERR_CAST(vr);
10687
10688         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
10689                 return vr->ul_rif;
10690
10691         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack);
10692         if (IS_ERR(vr->ul_rif)) {
10693                 err = PTR_ERR(vr->ul_rif);
10694                 goto err_ul_rif_create;
10695         }
10696
10697         vr->rif_count++;
10698         refcount_set(&vr->ul_rif_refcnt, 1);
10699
10700         return vr->ul_rif;
10701
10702 err_ul_rif_create:
10703         mlxsw_sp_vr_put(mlxsw_sp, vr);
10704         return ERR_PTR(err);
10705 }
10706
10707 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
10708 {
10709         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10710         struct mlxsw_sp_vr *vr;
10711
10712         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
10713
10714         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
10715                 return;
10716
10717         vr->rif_count--;
10718         mlxsw_sp_ul_rif_destroy(ul_rif);
10719         mlxsw_sp_vr_put(mlxsw_sp, vr);
10720 }
10721
10722 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
10723                                u16 *ul_rif_index)
10724 {
10725         struct mlxsw_sp_rif *ul_rif;
10726         int err = 0;
10727
10728         mutex_lock(&mlxsw_sp->router->lock);
10729         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL);
10730         if (IS_ERR(ul_rif)) {
10731                 err = PTR_ERR(ul_rif);
10732                 goto out;
10733         }
10734         *ul_rif_index = ul_rif->rif_index;
10735 out:
10736         mutex_unlock(&mlxsw_sp->router->lock);
10737         return err;
10738 }
10739
10740 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
10741 {
10742         struct mlxsw_sp_rif *ul_rif;
10743
10744         mutex_lock(&mlxsw_sp->router->lock);
10745         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
10746         if (WARN_ON(!ul_rif))
10747                 goto out;
10748
10749         mlxsw_sp_ul_rif_put(ul_rif);
10750 out:
10751         mutex_unlock(&mlxsw_sp->router->lock);
10752 }
10753
10754 static int
10755 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10756                                 struct netlink_ext_ack *extack)
10757 {
10758         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10759         struct net_device *dev = mlxsw_sp_rif_dev(rif);
10760         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
10761         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10762         struct mlxsw_sp_rif *ul_rif;
10763         int err;
10764
10765         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack);
10766         if (IS_ERR(ul_rif))
10767                 return PTR_ERR(ul_rif);
10768
10769         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
10770         if (err)
10771                 goto err_loopback_op;
10772
10773         lb_rif->ul_vr_id = 0;
10774         lb_rif->ul_rif_id = ul_rif->rif_index;
10775
10776         return 0;
10777
10778 err_loopback_op:
10779         mlxsw_sp_ul_rif_put(ul_rif);
10780         return err;
10781 }
10782
10783 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10784 {
10785         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10786         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10787         struct mlxsw_sp_rif *ul_rif;
10788
10789         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
10790         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
10791         mlxsw_sp_ul_rif_put(ul_rif);
10792 }
10793
10794 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
10795         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
10796         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
10797         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
10798         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
10799         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
10800 };
10801
10802 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
10803         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
10804         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp2_rif_vlan_ops,
10805         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
10806         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
10807 };
10808
10809 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp)
10810 {
10811         struct gen_pool *rifs_table;
10812         int err;
10813
10814         rifs_table = gen_pool_create(0, -1);
10815         if (!rifs_table)
10816                 return -ENOMEM;
10817
10818         gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align,
10819                           NULL);
10820
10821         err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET,
10822                            MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1);
10823         if (err)
10824                 goto err_gen_pool_add;
10825
10826         mlxsw_sp->router->rifs_table = rifs_table;
10827
10828         return 0;
10829
10830 err_gen_pool_add:
10831         gen_pool_destroy(rifs_table);
10832         return err;
10833 }
10834
10835 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp)
10836 {
10837         gen_pool_destroy(mlxsw_sp->router->rifs_table);
10838 }
10839
10840 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
10841 {
10842         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10843         struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10844         struct mlxsw_core *core = mlxsw_sp->core;
10845         int err;
10846
10847         if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES))
10848                 return -EIO;
10849         mlxsw_sp->router->max_rif_mac_profile =
10850                 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES);
10851
10852         mlxsw_sp->router->rifs = kcalloc(max_rifs,
10853                                          sizeof(struct mlxsw_sp_rif *),
10854                                          GFP_KERNEL);
10855         if (!mlxsw_sp->router->rifs)
10856                 return -ENOMEM;
10857
10858         err = mlxsw_sp_rifs_table_init(mlxsw_sp);
10859         if (err)
10860                 goto err_rifs_table_init;
10861
10862         idr_init(&mlxsw_sp->router->rif_mac_profiles_idr);
10863         atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0);
10864         atomic_set(&mlxsw_sp->router->rifs_count, 0);
10865         devl_resource_occ_get_register(devlink,
10866                                        MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
10867                                        mlxsw_sp_rif_mac_profiles_occ_get,
10868                                        mlxsw_sp);
10869         devl_resource_occ_get_register(devlink,
10870                                        MLXSW_SP_RESOURCE_RIFS,
10871                                        mlxsw_sp_rifs_occ_get,
10872                                        mlxsw_sp);
10873
10874         return 0;
10875
10876 err_rifs_table_init:
10877         kfree(mlxsw_sp->router->rifs);
10878         return err;
10879 }
10880
10881 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
10882 {
10883         int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10884         struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10885         int i;
10886
10887         WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count));
10888         for (i = 0; i < max_rifs; i++)
10889                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
10890
10891         devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS);
10892         devl_resource_occ_get_unregister(devlink,
10893                                          MLXSW_SP_RESOURCE_RIF_MAC_PROFILES);
10894         WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr));
10895         idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr);
10896         mlxsw_sp_rifs_table_fini(mlxsw_sp);
10897         kfree(mlxsw_sp->router->rifs);
10898 }
10899
10900 static int
10901 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
10902 {
10903         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
10904
10905         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
10906         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
10907 }
10908
10909 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
10910 {
10911         int err;
10912
10913         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
10914
10915         err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
10916         if (err)
10917                 return err;
10918         err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
10919         if (err)
10920                 return err;
10921
10922         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
10923 }
10924
10925 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp)
10926 {
10927         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr;
10928         return mlxsw_sp_ipips_init(mlxsw_sp);
10929 }
10930
10931 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp)
10932 {
10933         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr;
10934         return mlxsw_sp_ipips_init(mlxsw_sp);
10935 }
10936
10937 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
10938 {
10939         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
10940 }
10941
10942 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
10943 {
10944         struct mlxsw_sp_router *router;
10945
10946         /* Flush pending FIB notifications and then flush the device's
10947          * table before requesting another dump. The FIB notification
10948          * block is unregistered, so no need to take RTNL.
10949          */
10950         mlxsw_core_flush_owq();
10951         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
10952         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
10953 }
10954
10955 #ifdef CONFIG_IP_ROUTE_MULTIPATH
10956 struct mlxsw_sp_mp_hash_config {
10957         DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT);
10958         DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT);
10959         DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT);
10960         DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT);
10961         bool inc_parsing_depth;
10962 };
10963
10964 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \
10965         bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1)
10966
10967 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \
10968         bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1)
10969
10970 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \
10971         bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr)
10972
10973 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config)
10974 {
10975         unsigned long *inner_headers = config->inner_headers;
10976         unsigned long *inner_fields = config->inner_fields;
10977
10978         /* IPv4 inner */
10979         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
10980         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
10981         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
10982         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
10983         /* IPv6 inner */
10984         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
10985         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
10986         MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
10987         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
10988         MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
10989         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
10990         MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
10991         MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
10992 }
10993
10994 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
10995 {
10996         unsigned long *headers = config->headers;
10997         unsigned long *fields = config->fields;
10998
10999         MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11000         MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11001         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11002         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11003 }
11004
11005 static void
11006 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config,
11007                               u32 hash_fields)
11008 {
11009         unsigned long *inner_headers = config->inner_headers;
11010         unsigned long *inner_fields = config->inner_fields;
11011
11012         /* IPv4 Inner */
11013         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
11014         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
11015         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
11016                 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
11017         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
11018                 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
11019         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11020                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL);
11021         /* IPv6 inner */
11022         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
11023         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
11024         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) {
11025                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
11026                 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
11027         }
11028         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) {
11029                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
11030                 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
11031         }
11032         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11033                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
11034         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
11035                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
11036         /* L4 inner */
11037         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4);
11038         MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6);
11039         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
11040                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT);
11041         if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
11042                 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT);
11043 }
11044
11045 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp,
11046                                    struct mlxsw_sp_mp_hash_config *config)
11047 {
11048         struct net *net = mlxsw_sp_net(mlxsw_sp);
11049         unsigned long *headers = config->headers;
11050         unsigned long *fields = config->fields;
11051         u32 hash_fields;
11052
11053         switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) {
11054         case 0:
11055                 mlxsw_sp_mp4_hash_outer_addr(config);
11056                 break;
11057         case 1:
11058                 mlxsw_sp_mp4_hash_outer_addr(config);
11059                 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11060                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11061                 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11062                 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11063                 break;
11064         case 2:
11065                 /* Outer */
11066                 mlxsw_sp_mp4_hash_outer_addr(config);
11067                 /* Inner */
11068                 mlxsw_sp_mp_hash_inner_l3(config);
11069                 break;
11070         case 3:
11071                 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields);
11072                 /* Outer */
11073                 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11074                 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11075                 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11076                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
11077                         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11078                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
11079                         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11080                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11081                         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11082                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11083                         MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11084                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11085                         MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11086                 /* Inner */
11087                 mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11088                 break;
11089         }
11090 }
11091
11092 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
11093 {
11094         unsigned long *headers = config->headers;
11095         unsigned long *fields = config->fields;
11096
11097         MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11098         MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11099         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11100         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11101         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11102         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11103 }
11104
11105 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp,
11106                                    struct mlxsw_sp_mp_hash_config *config)
11107 {
11108         u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp));
11109         unsigned long *headers = config->headers;
11110         unsigned long *fields = config->fields;
11111
11112         switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) {
11113         case 0:
11114                 mlxsw_sp_mp6_hash_outer_addr(config);
11115                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11116                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11117                 break;
11118         case 1:
11119                 mlxsw_sp_mp6_hash_outer_addr(config);
11120                 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11121                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11122                 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11123                 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11124                 break;
11125         case 2:
11126                 /* Outer */
11127                 mlxsw_sp_mp6_hash_outer_addr(config);
11128                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11129                 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11130                 /* Inner */
11131                 mlxsw_sp_mp_hash_inner_l3(config);
11132                 config->inc_parsing_depth = true;
11133                 break;
11134         case 3:
11135                 /* Outer */
11136                 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11137                 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11138                 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11139                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) {
11140                         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11141                         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11142                 }
11143                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) {
11144                         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11145                         MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11146                 }
11147                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11148                         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11149                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
11150                         MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11151                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11152                         MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11153                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11154                         MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11155                 /* Inner */
11156                 mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11157                 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)
11158                         config->inc_parsing_depth = true;
11159                 break;
11160         }
11161 }
11162
11163 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp,
11164                                                  bool old_inc_parsing_depth,
11165                                                  bool new_inc_parsing_depth)
11166 {
11167         int err;
11168
11169         if (!old_inc_parsing_depth && new_inc_parsing_depth) {
11170                 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp);
11171                 if (err)
11172                         return err;
11173                 mlxsw_sp->router->inc_parsing_depth = true;
11174         } else if (old_inc_parsing_depth && !new_inc_parsing_depth) {
11175                 mlxsw_sp_parsing_depth_dec(mlxsw_sp);
11176                 mlxsw_sp->router->inc_parsing_depth = false;
11177         }
11178
11179         return 0;
11180 }
11181
11182 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11183 {
11184         bool old_inc_parsing_depth, new_inc_parsing_depth;
11185         struct mlxsw_sp_mp_hash_config config = {};
11186         char recr2_pl[MLXSW_REG_RECR2_LEN];
11187         unsigned long bit;
11188         u32 seed;
11189         int err;
11190
11191         seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
11192         mlxsw_reg_recr2_pack(recr2_pl, seed);
11193         mlxsw_sp_mp4_hash_init(mlxsw_sp, &config);
11194         mlxsw_sp_mp6_hash_init(mlxsw_sp, &config);
11195
11196         old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11197         new_inc_parsing_depth = config.inc_parsing_depth;
11198         err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp,
11199                                                     old_inc_parsing_depth,
11200                                                     new_inc_parsing_depth);
11201         if (err)
11202                 return err;
11203
11204         for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT)
11205                 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1);
11206         for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT)
11207                 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1);
11208         for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT)
11209                 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1);
11210         for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT)
11211                 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1);
11212
11213         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
11214         if (err)
11215                 goto err_reg_write;
11216
11217         return 0;
11218
11219 err_reg_write:
11220         mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth,
11221                                               old_inc_parsing_depth);
11222         return err;
11223 }
11224
11225 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11226 {
11227         bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11228
11229         mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth,
11230                                               false);
11231 }
11232 #else
11233 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11234 {
11235         return 0;
11236 }
11237
11238 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11239 {
11240 }
11241 #endif
11242
11243 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
11244 {
11245         char rdpm_pl[MLXSW_REG_RDPM_LEN];
11246         unsigned int i;
11247
11248         MLXSW_REG_ZERO(rdpm, rdpm_pl);
11249
11250         /* HW is determining switch priority based on DSCP-bits, but the
11251          * kernel is still doing that based on the ToS. Since there's a
11252          * mismatch in bits we need to make sure to translate the right
11253          * value ToS would observe, skipping the 2 least-significant ECN bits.
11254          */
11255         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
11256                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
11257
11258         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
11259 }
11260
11261 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
11262 {
11263         struct net *net = mlxsw_sp_net(mlxsw_sp);
11264         char rgcr_pl[MLXSW_REG_RGCR_LEN];
11265         u64 max_rifs;
11266         bool usp;
11267
11268         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
11269                 return -EIO;
11270         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
11271         usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority);
11272
11273         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
11274         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
11275         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
11276         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
11277 }
11278
11279 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11280 {
11281         char rgcr_pl[MLXSW_REG_RGCR_LEN];
11282
11283         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
11284         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
11285 }
11286
11287 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp,
11288                                 struct netlink_ext_ack *extack)
11289 {
11290         struct mlxsw_sp_router *router = mlxsw_sp->router;
11291         struct mlxsw_sp_rif *lb_rif;
11292         int err;
11293
11294         router->lb_crif = mlxsw_sp_crif_alloc(NULL);
11295         if (!router->lb_crif)
11296                 return -ENOMEM;
11297
11298         /* Create a generic loopback RIF associated with the main table
11299          * (default VRF). Any table can be used, but the main table exists
11300          * anyway, so we do not waste resources. Loopback RIFs are usually
11301          * created with a NULL CRIF, but this RIF is used as a fallback RIF
11302          * for blackhole nexthops, and nexthops expect to have a valid CRIF.
11303          */
11304         lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif,
11305                                      extack);
11306         if (IS_ERR(lb_rif)) {
11307                 err = PTR_ERR(lb_rif);
11308                 goto err_ul_rif_get;
11309         }
11310
11311         return 0;
11312
11313 err_ul_rif_get:
11314         mlxsw_sp_crif_free(router->lb_crif);
11315         return err;
11316 }
11317
11318 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
11319 {
11320         mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif);
11321         mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif);
11322 }
11323
11324 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp)
11325 {
11326         size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges);
11327
11328         mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr;
11329         mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges;
11330         mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11331
11332         return 0;
11333 }
11334
11335 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = {
11336         .init = mlxsw_sp1_router_init,
11337         .ipips_init = mlxsw_sp1_ipips_init,
11338 };
11339
11340 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp)
11341 {
11342         size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges);
11343
11344         mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr;
11345         mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges;
11346         mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11347
11348         return 0;
11349 }
11350
11351 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = {
11352         .init = mlxsw_sp2_router_init,
11353         .ipips_init = mlxsw_sp2_ipips_init,
11354 };
11355
11356 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
11357                          struct netlink_ext_ack *extack)
11358 {
11359         struct mlxsw_sp_router *router;
11360         struct notifier_block *nb;
11361         int err;
11362
11363         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
11364         if (!router)
11365                 return -ENOMEM;
11366         mutex_init(&router->lock);
11367         mlxsw_sp->router = router;
11368         router->mlxsw_sp = mlxsw_sp;
11369
11370         err = mlxsw_sp->router_ops->init(mlxsw_sp);
11371         if (err)
11372                 goto err_router_ops_init;
11373
11374         INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list);
11375         INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw,
11376                           mlxsw_sp_nh_grp_activity_work);
11377         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
11378         err = __mlxsw_sp_router_init(mlxsw_sp);
11379         if (err)
11380                 goto err_router_init;
11381
11382         err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp);
11383         if (err)
11384                 goto err_ipips_init;
11385
11386         err = rhashtable_init(&mlxsw_sp->router->crif_ht,
11387                               &mlxsw_sp_crif_ht_params);
11388         if (err)
11389                 goto err_crif_ht_init;
11390
11391         err = mlxsw_sp_rifs_init(mlxsw_sp);
11392         if (err)
11393                 goto err_rifs_init;
11394
11395         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
11396                               &mlxsw_sp_nexthop_ht_params);
11397         if (err)
11398                 goto err_nexthop_ht_init;
11399
11400         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
11401                               &mlxsw_sp_nexthop_group_ht_params);
11402         if (err)
11403                 goto err_nexthop_group_ht_init;
11404
11405         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
11406         err = mlxsw_sp_lpm_init(mlxsw_sp);
11407         if (err)
11408                 goto err_lpm_init;
11409
11410         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
11411         if (err)
11412                 goto err_mr_init;
11413
11414         err = mlxsw_sp_vrs_init(mlxsw_sp);
11415         if (err)
11416                 goto err_vrs_init;
11417
11418         err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack);
11419         if (err)
11420                 goto err_lb_rif_init;
11421
11422         err = mlxsw_sp_neigh_init(mlxsw_sp);
11423         if (err)
11424                 goto err_neigh_init;
11425
11426         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
11427         if (err)
11428                 goto err_mp_hash_init;
11429
11430         err = mlxsw_sp_dscp_init(mlxsw_sp);
11431         if (err)
11432                 goto err_dscp_init;
11433
11434         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
11435         err = register_inetaddr_notifier(&router->inetaddr_nb);
11436         if (err)
11437                 goto err_register_inetaddr_notifier;
11438
11439         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
11440         err = register_inet6addr_notifier(&router->inet6addr_nb);
11441         if (err)
11442                 goto err_register_inet6addr_notifier;
11443
11444         router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event;
11445         err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11446         if (err)
11447                 goto err_register_inetaddr_valid_notifier;
11448
11449         nb = &router->inet6addr_valid_nb;
11450         nb->notifier_call = mlxsw_sp_inet6addr_valid_event;
11451         err = register_inet6addr_validator_notifier(nb);
11452         if (err)
11453                 goto err_register_inet6addr_valid_notifier;
11454
11455         mlxsw_sp->router->netevent_nb.notifier_call =
11456                 mlxsw_sp_router_netevent_event;
11457         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
11458         if (err)
11459                 goto err_register_netevent_notifier;
11460
11461         mlxsw_sp->router->nexthop_nb.notifier_call =
11462                 mlxsw_sp_nexthop_obj_event;
11463         err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11464                                         &mlxsw_sp->router->nexthop_nb,
11465                                         extack);
11466         if (err)
11467                 goto err_register_nexthop_notifier;
11468
11469         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
11470         err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
11471                                     &mlxsw_sp->router->fib_nb,
11472                                     mlxsw_sp_router_fib_dump_flush, extack);
11473         if (err)
11474                 goto err_register_fib_notifier;
11475
11476         mlxsw_sp->router->netdevice_nb.notifier_call =
11477                 mlxsw_sp_router_netdevice_event;
11478         err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
11479                                               &mlxsw_sp->router->netdevice_nb);
11480         if (err)
11481                 goto err_register_netdev_notifier;
11482
11483         return 0;
11484
11485 err_register_netdev_notifier:
11486         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
11487                                 &mlxsw_sp->router->fib_nb);
11488 err_register_fib_notifier:
11489         unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11490                                     &mlxsw_sp->router->nexthop_nb);
11491 err_register_nexthop_notifier:
11492         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
11493 err_register_netevent_notifier:
11494         unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb);
11495 err_register_inet6addr_valid_notifier:
11496         unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11497 err_register_inetaddr_valid_notifier:
11498         unregister_inet6addr_notifier(&router->inet6addr_nb);
11499 err_register_inet6addr_notifier:
11500         unregister_inetaddr_notifier(&router->inetaddr_nb);
11501 err_register_inetaddr_notifier:
11502         mlxsw_core_flush_owq();
11503 err_dscp_init:
11504         mlxsw_sp_mp_hash_fini(mlxsw_sp);
11505 err_mp_hash_init:
11506         mlxsw_sp_neigh_fini(mlxsw_sp);
11507 err_neigh_init:
11508         mlxsw_sp_lb_rif_fini(mlxsw_sp);
11509 err_lb_rif_init:
11510         mlxsw_sp_vrs_fini(mlxsw_sp);
11511 err_vrs_init:
11512         mlxsw_sp_mr_fini(mlxsw_sp);
11513 err_mr_init:
11514         mlxsw_sp_lpm_fini(mlxsw_sp);
11515 err_lpm_init:
11516         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
11517 err_nexthop_group_ht_init:
11518         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
11519 err_nexthop_ht_init:
11520         mlxsw_sp_rifs_fini(mlxsw_sp);
11521 err_rifs_init:
11522         rhashtable_destroy(&mlxsw_sp->router->crif_ht);
11523 err_crif_ht_init:
11524         mlxsw_sp_ipips_fini(mlxsw_sp);
11525 err_ipips_init:
11526         __mlxsw_sp_router_fini(mlxsw_sp);
11527 err_router_init:
11528         cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
11529 err_router_ops_init:
11530         mutex_destroy(&mlxsw_sp->router->lock);
11531         kfree(mlxsw_sp->router);
11532         return err;
11533 }
11534
11535 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11536 {
11537         struct mlxsw_sp_router *router = mlxsw_sp->router;
11538
11539         unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
11540                                           &router->netdevice_nb);
11541         unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb);
11542         unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11543                                     &router->nexthop_nb);
11544         unregister_netevent_notifier(&router->netevent_nb);
11545         unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb);
11546         unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11547         unregister_inet6addr_notifier(&router->inet6addr_nb);
11548         unregister_inetaddr_notifier(&router->inetaddr_nb);
11549         mlxsw_core_flush_owq();
11550         mlxsw_sp_mp_hash_fini(mlxsw_sp);
11551         mlxsw_sp_neigh_fini(mlxsw_sp);
11552         mlxsw_sp_lb_rif_fini(mlxsw_sp);
11553         mlxsw_sp_vrs_fini(mlxsw_sp);
11554         mlxsw_sp_mr_fini(mlxsw_sp);
11555         mlxsw_sp_lpm_fini(mlxsw_sp);
11556         rhashtable_destroy(&router->nexthop_group_ht);
11557         rhashtable_destroy(&router->nexthop_ht);
11558         mlxsw_sp_rifs_fini(mlxsw_sp);
11559         rhashtable_destroy(&mlxsw_sp->router->crif_ht);
11560         mlxsw_sp_ipips_fini(mlxsw_sp);
11561         __mlxsw_sp_router_fini(mlxsw_sp);
11562         cancel_delayed_work_sync(&router->nh_grp_activity_dw);
11563         mutex_destroy(&router->lock);
11564         kfree(router);
11565 }