1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/rhashtable.h>
7 #include <linux/bitops.h>
9 #include <linux/notifier.h>
10 #include <linux/inetdevice.h>
11 #include <linux/netdevice.h>
12 #include <linux/if_bridge.h>
13 #include <linux/socket.h>
14 #include <linux/route.h>
15 #include <linux/gcd.h>
16 #include <linux/if_macvlan.h>
17 #include <linux/refcount.h>
18 #include <linux/jhash.h>
19 #include <linux/net_namespace.h>
20 #include <linux/mutex.h>
21 #include <net/netevent.h>
22 #include <net/neighbour.h>
24 #include <net/ip_fib.h>
25 #include <net/ip6_fib.h>
26 #include <net/nexthop.h>
27 #include <net/fib_rules.h>
28 #include <net/ip_tunnels.h>
29 #include <net/l3mdev.h>
30 #include <net/addrconf.h>
31 #include <net/ndisc.h>
33 #include <net/fib_notifier.h>
34 #include <net/switchdev.h>
39 #include "spectrum_cnt.h"
40 #include "spectrum_dpipe.h"
41 #include "spectrum_ipip.h"
42 #include "spectrum_mr.h"
43 #include "spectrum_mr_tcam.h"
44 #include "spectrum_router.h"
45 #include "spectrum_span.h"
49 struct mlxsw_sp_lpm_tree;
50 struct mlxsw_sp_rif_ops;
53 struct list_head nexthop_list;
54 struct list_head neigh_list;
55 struct net_device *dev; /* NULL for underlay RIF */
56 struct mlxsw_sp_fid *fid;
57 unsigned char addr[ETH_ALEN];
61 const struct mlxsw_sp_rif_ops *ops;
62 struct mlxsw_sp *mlxsw_sp;
64 unsigned int counter_ingress;
65 bool counter_ingress_valid;
66 unsigned int counter_egress;
67 bool counter_egress_valid;
70 struct mlxsw_sp_rif_params {
71 struct net_device *dev;
80 struct mlxsw_sp_rif_subport {
81 struct mlxsw_sp_rif common;
91 struct mlxsw_sp_rif_ipip_lb {
92 struct mlxsw_sp_rif common;
93 struct mlxsw_sp_rif_ipip_lb_config lb_config;
94 u16 ul_vr_id; /* Reserved for Spectrum-2. */
95 u16 ul_rif_id; /* Reserved for Spectrum. */
98 struct mlxsw_sp_rif_params_ipip_lb {
99 struct mlxsw_sp_rif_params common;
100 struct mlxsw_sp_rif_ipip_lb_config lb_config;
103 struct mlxsw_sp_rif_ops {
104 enum mlxsw_sp_rif_type type;
107 void (*setup)(struct mlxsw_sp_rif *rif,
108 const struct mlxsw_sp_rif_params *params);
109 int (*configure)(struct mlxsw_sp_rif *rif);
110 void (*deconfigure)(struct mlxsw_sp_rif *rif);
111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
112 struct netlink_ext_ack *extack);
113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
116 static struct mlxsw_sp_rif *
117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
118 const struct net_device *dev);
119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
122 struct mlxsw_sp_lpm_tree *lpm_tree);
123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
124 const struct mlxsw_sp_fib *fib,
126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
127 const struct mlxsw_sp_fib *fib);
129 static unsigned int *
130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
131 enum mlxsw_sp_rif_counter_dir dir)
134 case MLXSW_SP_RIF_COUNTER_EGRESS:
135 return &rif->counter_egress;
136 case MLXSW_SP_RIF_COUNTER_INGRESS:
137 return &rif->counter_ingress;
143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
144 enum mlxsw_sp_rif_counter_dir dir)
147 case MLXSW_SP_RIF_COUNTER_EGRESS:
148 return rif->counter_egress_valid;
149 case MLXSW_SP_RIF_COUNTER_INGRESS:
150 return rif->counter_ingress_valid;
156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
157 enum mlxsw_sp_rif_counter_dir dir,
161 case MLXSW_SP_RIF_COUNTER_EGRESS:
162 rif->counter_egress_valid = valid;
164 case MLXSW_SP_RIF_COUNTER_INGRESS:
165 rif->counter_ingress_valid = valid;
170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
171 unsigned int counter_index, bool enable,
172 enum mlxsw_sp_rif_counter_dir dir)
174 char ritr_pl[MLXSW_REG_RITR_LEN];
175 bool is_egress = false;
178 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
180 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
181 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
185 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
187 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
191 struct mlxsw_sp_rif *rif,
192 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
194 char ricnt_pl[MLXSW_REG_RICNT_LEN];
195 unsigned int *p_counter_index;
199 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
203 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
204 if (!p_counter_index)
206 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
207 MLXSW_REG_RICNT_OPCODE_NOP);
208 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
211 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
216 unsigned int counter_index)
218 char ricnt_pl[MLXSW_REG_RICNT_LEN];
220 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
221 MLXSW_REG_RICNT_OPCODE_CLEAR);
222 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
226 struct mlxsw_sp_rif *rif,
227 enum mlxsw_sp_rif_counter_dir dir)
229 unsigned int *p_counter_index;
232 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
233 if (!p_counter_index)
235 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
240 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
242 goto err_counter_clear;
244 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
245 *p_counter_index, true, dir);
247 goto err_counter_edit;
248 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
253 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
259 struct mlxsw_sp_rif *rif,
260 enum mlxsw_sp_rif_counter_dir dir)
262 unsigned int *p_counter_index;
264 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
267 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
268 if (WARN_ON(!p_counter_index))
270 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
271 *p_counter_index, false, dir);
272 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
274 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
279 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
280 struct devlink *devlink;
282 devlink = priv_to_devlink(mlxsw_sp->core);
283 if (!devlink_dpipe_table_counter_enabled(devlink,
284 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
286 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
291 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
293 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
298 struct mlxsw_sp_prefix_usage {
299 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
303 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
307 struct mlxsw_sp_prefix_usage *prefix_usage2)
309 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
314 struct mlxsw_sp_prefix_usage *prefix_usage2)
316 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
321 unsigned char prefix_len)
323 set_bit(prefix_len, prefix_usage->b);
327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
328 unsigned char prefix_len)
330 clear_bit(prefix_len, prefix_usage->b);
333 struct mlxsw_sp_fib_key {
334 unsigned char addr[sizeof(struct in6_addr)];
335 unsigned char prefix_len;
338 enum mlxsw_sp_fib_entry_type {
339 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
340 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
341 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
342 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
343 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
345 /* This is a special case of local delivery, where a packet should be
346 * decapsulated on reception. Note that there is no corresponding ENCAP,
347 * because that's a type of next hop, not of FIB entry. (There can be
348 * several next hops in a REMOTE entry, and some of them may be
349 * encapsulating entries.)
351 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
352 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
355 struct mlxsw_sp_nexthop_group_info;
356 struct mlxsw_sp_nexthop_group;
357 struct mlxsw_sp_fib_entry;
359 struct mlxsw_sp_fib_node {
360 struct mlxsw_sp_fib_entry *fib_entry;
361 struct list_head list;
362 struct rhash_head ht_node;
363 struct mlxsw_sp_fib *fib;
364 struct mlxsw_sp_fib_key key;
367 struct mlxsw_sp_fib_entry_decap {
368 struct mlxsw_sp_ipip_entry *ipip_entry;
372 static struct mlxsw_sp_fib_entry_priv *
373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
375 struct mlxsw_sp_fib_entry_priv *priv;
377 if (!ll_ops->fib_entry_priv_size)
378 /* No need to have priv */
381 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
383 return ERR_PTR(-ENOMEM);
384 refcount_set(&priv->refcnt, 1);
389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
396 refcount_inc(&priv->refcnt);
399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
401 if (!priv || !refcount_dec_and_test(&priv->refcnt))
403 mlxsw_sp_fib_entry_priv_destroy(priv);
406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
407 struct mlxsw_sp_fib_entry_priv *priv)
411 mlxsw_sp_fib_entry_priv_hold(priv);
412 list_add(&priv->list, &op_ctx->fib_entry_priv_list);
415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
417 struct mlxsw_sp_fib_entry_priv *priv, *tmp;
419 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
420 mlxsw_sp_fib_entry_priv_put(priv);
421 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
424 struct mlxsw_sp_fib_entry {
425 struct mlxsw_sp_fib_node *fib_node;
426 enum mlxsw_sp_fib_entry_type type;
427 struct list_head nexthop_group_node;
428 struct mlxsw_sp_nexthop_group *nh_group;
429 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
430 struct mlxsw_sp_fib_entry_priv *priv;
433 struct mlxsw_sp_fib4_entry {
434 struct mlxsw_sp_fib_entry common;
441 struct mlxsw_sp_fib6_entry {
442 struct mlxsw_sp_fib_entry common;
443 struct list_head rt6_list;
447 struct mlxsw_sp_rt6 {
448 struct list_head list;
449 struct fib6_info *rt;
452 struct mlxsw_sp_lpm_tree {
454 unsigned int ref_count;
455 enum mlxsw_sp_l3proto proto;
456 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
457 struct mlxsw_sp_prefix_usage prefix_usage;
460 struct mlxsw_sp_fib {
461 struct rhashtable ht;
462 struct list_head node_list;
463 struct mlxsw_sp_vr *vr;
464 struct mlxsw_sp_lpm_tree *lpm_tree;
465 enum mlxsw_sp_l3proto proto;
466 const struct mlxsw_sp_router_ll_ops *ll_ops;
470 u16 id; /* virtual router ID */
471 u32 tb_id; /* kernel fib table id */
472 unsigned int rif_count;
473 struct mlxsw_sp_fib *fib4;
474 struct mlxsw_sp_fib *fib6;
475 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
476 struct mlxsw_sp_rif *ul_rif;
477 refcount_t ul_rif_refcnt;
480 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
481 enum mlxsw_sp_l3proto proto)
486 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
488 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
489 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
492 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
494 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
495 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
498 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
500 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
501 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
504 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
506 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
507 struct mlxsw_sp_vr *vr,
508 enum mlxsw_sp_l3proto proto)
510 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
511 struct mlxsw_sp_lpm_tree *lpm_tree;
512 struct mlxsw_sp_fib *fib;
515 err = ll_ops->init(mlxsw_sp, vr->id, proto);
519 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
520 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
522 return ERR_PTR(-ENOMEM);
523 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
525 goto err_rhashtable_init;
526 INIT_LIST_HEAD(&fib->node_list);
529 fib->lpm_tree = lpm_tree;
530 fib->ll_ops = ll_ops;
531 mlxsw_sp_lpm_tree_hold(lpm_tree);
532 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
534 goto err_lpm_tree_bind;
538 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
544 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
545 struct mlxsw_sp_fib *fib)
547 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
548 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
549 WARN_ON(!list_empty(&fib->node_list));
550 rhashtable_destroy(&fib->ht);
554 static struct mlxsw_sp_lpm_tree *
555 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
557 static struct mlxsw_sp_lpm_tree *lpm_tree;
560 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
561 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
562 if (lpm_tree->ref_count == 0)
568 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
569 const struct mlxsw_sp_router_ll_ops *ll_ops,
570 struct mlxsw_sp_lpm_tree *lpm_tree)
572 char xralta_pl[MLXSW_REG_XRALTA_LEN];
574 mlxsw_reg_xralta_pack(xralta_pl, true,
575 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
577 return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
580 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
581 const struct mlxsw_sp_router_ll_ops *ll_ops,
582 struct mlxsw_sp_lpm_tree *lpm_tree)
584 char xralta_pl[MLXSW_REG_XRALTA_LEN];
586 mlxsw_reg_xralta_pack(xralta_pl, false,
587 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
589 ll_ops->ralta_write(mlxsw_sp, xralta_pl);
593 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
594 const struct mlxsw_sp_router_ll_ops *ll_ops,
595 struct mlxsw_sp_prefix_usage *prefix_usage,
596 struct mlxsw_sp_lpm_tree *lpm_tree)
598 char xralst_pl[MLXSW_REG_XRALST_LEN];
601 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
603 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
606 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
610 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
611 MLXSW_REG_RALST_BIN_NO_CHILD);
612 last_prefix = prefix;
614 return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
617 static struct mlxsw_sp_lpm_tree *
618 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
619 const struct mlxsw_sp_router_ll_ops *ll_ops,
620 struct mlxsw_sp_prefix_usage *prefix_usage,
621 enum mlxsw_sp_l3proto proto)
623 struct mlxsw_sp_lpm_tree *lpm_tree;
626 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
628 return ERR_PTR(-EBUSY);
629 lpm_tree->proto = proto;
630 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
634 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
636 goto err_left_struct_set;
637 memcpy(&lpm_tree->prefix_usage, prefix_usage,
638 sizeof(lpm_tree->prefix_usage));
639 memset(&lpm_tree->prefix_ref_count, 0,
640 sizeof(lpm_tree->prefix_ref_count));
641 lpm_tree->ref_count = 1;
645 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
649 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
650 const struct mlxsw_sp_router_ll_ops *ll_ops,
651 struct mlxsw_sp_lpm_tree *lpm_tree)
653 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
656 static struct mlxsw_sp_lpm_tree *
657 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
658 struct mlxsw_sp_prefix_usage *prefix_usage,
659 enum mlxsw_sp_l3proto proto)
661 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
662 struct mlxsw_sp_lpm_tree *lpm_tree;
665 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
666 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
667 if (lpm_tree->ref_count != 0 &&
668 lpm_tree->proto == proto &&
669 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
671 mlxsw_sp_lpm_tree_hold(lpm_tree);
675 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
678 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
680 lpm_tree->ref_count++;
683 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
684 struct mlxsw_sp_lpm_tree *lpm_tree)
686 const struct mlxsw_sp_router_ll_ops *ll_ops =
687 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
689 if (--lpm_tree->ref_count == 0)
690 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
693 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
695 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
697 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
698 struct mlxsw_sp_lpm_tree *lpm_tree;
702 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
705 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
706 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
707 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
708 sizeof(struct mlxsw_sp_lpm_tree),
710 if (!mlxsw_sp->router->lpm.trees)
713 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
714 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
715 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
718 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
719 MLXSW_SP_L3_PROTO_IPV4);
720 if (IS_ERR(lpm_tree)) {
721 err = PTR_ERR(lpm_tree);
722 goto err_ipv4_tree_get;
724 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
726 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
727 MLXSW_SP_L3_PROTO_IPV6);
728 if (IS_ERR(lpm_tree)) {
729 err = PTR_ERR(lpm_tree);
730 goto err_ipv6_tree_get;
732 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
737 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
738 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
740 kfree(mlxsw_sp->router->lpm.trees);
744 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
746 struct mlxsw_sp_lpm_tree *lpm_tree;
748 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
749 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
751 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
752 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
754 kfree(mlxsw_sp->router->lpm.trees);
757 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
759 return !!vr->fib4 || !!vr->fib6 ||
760 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
761 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
764 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
766 struct mlxsw_sp_vr *vr;
769 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
770 vr = &mlxsw_sp->router->vrs[i];
771 if (!mlxsw_sp_vr_is_used(vr))
777 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
778 const struct mlxsw_sp_fib *fib, u8 tree_id)
780 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
782 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
783 (enum mlxsw_reg_ralxx_protocol) fib->proto,
785 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
788 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
789 const struct mlxsw_sp_fib *fib)
791 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
793 /* Bind to tree 0 which is default */
794 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
795 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
796 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
799 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
801 /* For our purpose, squash main, default and local tables into one */
802 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
803 tb_id = RT_TABLE_MAIN;
807 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
810 struct mlxsw_sp_vr *vr;
813 tb_id = mlxsw_sp_fix_tb_id(tb_id);
815 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
816 vr = &mlxsw_sp->router->vrs[i];
817 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
823 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
826 struct mlxsw_sp_vr *vr;
829 mutex_lock(&mlxsw_sp->router->lock);
830 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
837 mutex_unlock(&mlxsw_sp->router->lock);
841 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
842 enum mlxsw_sp_l3proto proto)
845 case MLXSW_SP_L3_PROTO_IPV4:
847 case MLXSW_SP_L3_PROTO_IPV6:
853 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
855 struct netlink_ext_ack *extack)
857 struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
858 struct mlxsw_sp_fib *fib4;
859 struct mlxsw_sp_fib *fib6;
860 struct mlxsw_sp_vr *vr;
863 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
865 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
866 return ERR_PTR(-EBUSY);
868 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
870 return ERR_CAST(fib4);
871 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
874 goto err_fib6_create;
876 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
877 MLXSW_SP_L3_PROTO_IPV4);
878 if (IS_ERR(mr4_table)) {
879 err = PTR_ERR(mr4_table);
880 goto err_mr4_table_create;
882 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
883 MLXSW_SP_L3_PROTO_IPV6);
884 if (IS_ERR(mr6_table)) {
885 err = PTR_ERR(mr6_table);
886 goto err_mr6_table_create;
891 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
892 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
896 err_mr6_table_create:
897 mlxsw_sp_mr_table_destroy(mr4_table);
898 err_mr4_table_create:
899 mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
901 mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
905 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
906 struct mlxsw_sp_vr *vr)
908 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
909 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
910 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
911 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
912 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
914 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
918 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
919 struct netlink_ext_ack *extack)
921 struct mlxsw_sp_vr *vr;
923 tb_id = mlxsw_sp_fix_tb_id(tb_id);
924 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
926 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
930 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
932 if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
933 list_empty(&vr->fib6->node_list) &&
934 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
935 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
936 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
940 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
941 enum mlxsw_sp_l3proto proto, u8 tree_id)
943 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
945 if (!mlxsw_sp_vr_is_used(vr))
947 if (fib->lpm_tree->id == tree_id)
952 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
953 struct mlxsw_sp_fib *fib,
954 struct mlxsw_sp_lpm_tree *new_tree)
956 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
959 fib->lpm_tree = new_tree;
960 mlxsw_sp_lpm_tree_hold(new_tree);
961 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
964 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
968 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
969 fib->lpm_tree = old_tree;
973 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
974 struct mlxsw_sp_fib *fib,
975 struct mlxsw_sp_lpm_tree *new_tree)
977 enum mlxsw_sp_l3proto proto = fib->proto;
978 struct mlxsw_sp_lpm_tree *old_tree;
979 u8 old_id, new_id = new_tree->id;
980 struct mlxsw_sp_vr *vr;
983 old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
984 old_id = old_tree->id;
986 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
987 vr = &mlxsw_sp->router->vrs[i];
988 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
990 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
991 mlxsw_sp_vr_fib(vr, proto),
994 goto err_tree_replace;
997 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
998 sizeof(new_tree->prefix_ref_count));
999 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
1000 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
1005 for (i--; i >= 0; i--) {
1006 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
1008 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1009 mlxsw_sp_vr_fib(vr, proto),
1015 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1017 struct mlxsw_sp_vr *vr;
1021 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1024 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1025 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1027 if (!mlxsw_sp->router->vrs)
1030 for (i = 0; i < max_vrs; i++) {
1031 vr = &mlxsw_sp->router->vrs[i];
1038 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1040 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1042 /* At this stage we're guaranteed not to have new incoming
1043 * FIB notifications and the work queue is free from FIBs
1044 * sitting on top of mlxsw netdevs. However, we can still
1045 * have other FIBs queued. Flush the queue before flushing
1046 * the device's tables. No need for locks, as we're the only
1049 mlxsw_core_flush_owq();
1050 mlxsw_sp_router_fib_flush(mlxsw_sp);
1051 kfree(mlxsw_sp->router->vrs);
1054 static struct net_device *
1055 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
1057 struct ip_tunnel *tun = netdev_priv(ol_dev);
1058 struct net *net = dev_net(ol_dev);
1060 return dev_get_by_index_rcu(net, tun->parms.link);
1063 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1065 struct net_device *d;
1069 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1071 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1073 tb_id = RT_TABLE_MAIN;
1079 static struct mlxsw_sp_rif *
1080 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1081 const struct mlxsw_sp_rif_params *params,
1082 struct netlink_ext_ack *extack);
1084 static struct mlxsw_sp_rif_ipip_lb *
1085 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1086 enum mlxsw_sp_ipip_type ipipt,
1087 struct net_device *ol_dev,
1088 struct netlink_ext_ack *extack)
1090 struct mlxsw_sp_rif_params_ipip_lb lb_params;
1091 const struct mlxsw_sp_ipip_ops *ipip_ops;
1092 struct mlxsw_sp_rif *rif;
1094 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1095 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1096 .common.dev = ol_dev,
1097 .common.lag = false,
1098 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1101 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1103 return ERR_CAST(rif);
1104 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1107 static struct mlxsw_sp_ipip_entry *
1108 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1109 enum mlxsw_sp_ipip_type ipipt,
1110 struct net_device *ol_dev)
1112 const struct mlxsw_sp_ipip_ops *ipip_ops;
1113 struct mlxsw_sp_ipip_entry *ipip_entry;
1114 struct mlxsw_sp_ipip_entry *ret = NULL;
1116 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1117 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1119 return ERR_PTR(-ENOMEM);
1121 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1123 if (IS_ERR(ipip_entry->ol_lb)) {
1124 ret = ERR_CAST(ipip_entry->ol_lb);
1125 goto err_ol_ipip_lb_create;
1128 ipip_entry->ipipt = ipipt;
1129 ipip_entry->ol_dev = ol_dev;
1131 switch (ipip_ops->ul_proto) {
1132 case MLXSW_SP_L3_PROTO_IPV4:
1133 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1135 case MLXSW_SP_L3_PROTO_IPV6:
1142 err_ol_ipip_lb_create:
1148 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1150 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1155 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1156 const enum mlxsw_sp_l3proto ul_proto,
1157 union mlxsw_sp_l3addr saddr,
1159 struct mlxsw_sp_ipip_entry *ipip_entry)
1161 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1162 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1163 union mlxsw_sp_l3addr tun_saddr;
1165 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1168 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1169 return tun_ul_tb_id == ul_tb_id &&
1170 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1174 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1175 struct mlxsw_sp_fib_entry *fib_entry,
1176 struct mlxsw_sp_ipip_entry *ipip_entry)
1181 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1186 ipip_entry->decap_fib_entry = fib_entry;
1187 fib_entry->decap.ipip_entry = ipip_entry;
1188 fib_entry->decap.tunnel_index = tunnel_index;
1192 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1193 struct mlxsw_sp_fib_entry *fib_entry)
1195 /* Unlink this node from the IPIP entry that it's the decap entry of. */
1196 fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1197 fib_entry->decap.ipip_entry = NULL;
1198 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1199 1, fib_entry->decap.tunnel_index);
1202 static struct mlxsw_sp_fib_node *
1203 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1204 size_t addr_len, unsigned char prefix_len);
1205 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1206 struct mlxsw_sp_fib_entry *fib_entry);
1209 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1210 struct mlxsw_sp_ipip_entry *ipip_entry)
1212 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1214 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1215 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1217 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1221 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1222 struct mlxsw_sp_ipip_entry *ipip_entry,
1223 struct mlxsw_sp_fib_entry *decap_fib_entry)
1225 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1228 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1230 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1231 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1234 static struct mlxsw_sp_fib_entry *
1235 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1236 enum mlxsw_sp_l3proto proto,
1237 const union mlxsw_sp_l3addr *addr,
1238 enum mlxsw_sp_fib_entry_type type)
1240 struct mlxsw_sp_fib_node *fib_node;
1241 unsigned char addr_prefix_len;
1242 struct mlxsw_sp_fib *fib;
1243 struct mlxsw_sp_vr *vr;
1248 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1251 fib = mlxsw_sp_vr_fib(vr, proto);
1254 case MLXSW_SP_L3_PROTO_IPV4:
1255 addr4 = be32_to_cpu(addr->addr4);
1258 addr_prefix_len = 32;
1260 case MLXSW_SP_L3_PROTO_IPV6:
1266 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1268 if (!fib_node || fib_node->fib_entry->type != type)
1271 return fib_node->fib_entry;
1274 /* Given an IPIP entry, find the corresponding decap route. */
1275 static struct mlxsw_sp_fib_entry *
1276 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1277 struct mlxsw_sp_ipip_entry *ipip_entry)
1279 static struct mlxsw_sp_fib_node *fib_node;
1280 const struct mlxsw_sp_ipip_ops *ipip_ops;
1281 unsigned char saddr_prefix_len;
1282 union mlxsw_sp_l3addr saddr;
1283 struct mlxsw_sp_fib *ul_fib;
1284 struct mlxsw_sp_vr *ul_vr;
1290 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1292 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1293 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1297 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1298 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1299 ipip_entry->ol_dev);
1301 switch (ipip_ops->ul_proto) {
1302 case MLXSW_SP_L3_PROTO_IPV4:
1303 saddr4 = be32_to_cpu(saddr.addr4);
1306 saddr_prefix_len = 32;
1313 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1316 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1319 return fib_node->fib_entry;
1322 static struct mlxsw_sp_ipip_entry *
1323 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1324 enum mlxsw_sp_ipip_type ipipt,
1325 struct net_device *ol_dev)
1327 struct mlxsw_sp_ipip_entry *ipip_entry;
1329 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1330 if (IS_ERR(ipip_entry))
1333 list_add_tail(&ipip_entry->ipip_list_node,
1334 &mlxsw_sp->router->ipip_list);
1340 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1341 struct mlxsw_sp_ipip_entry *ipip_entry)
1343 list_del(&ipip_entry->ipip_list_node);
1344 mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1348 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1349 const struct net_device *ul_dev,
1350 enum mlxsw_sp_l3proto ul_proto,
1351 union mlxsw_sp_l3addr ul_dip,
1352 struct mlxsw_sp_ipip_entry *ipip_entry)
1354 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1355 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1357 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1360 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1361 ul_tb_id, ipip_entry);
1364 /* Given decap parameters, find the corresponding IPIP entry. */
1365 static struct mlxsw_sp_ipip_entry *
1366 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1367 enum mlxsw_sp_l3proto ul_proto,
1368 union mlxsw_sp_l3addr ul_dip)
1370 struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1371 struct net_device *ul_dev;
1375 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1379 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1381 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1395 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1396 const struct net_device *dev,
1397 enum mlxsw_sp_ipip_type *p_type)
1399 struct mlxsw_sp_router *router = mlxsw_sp->router;
1400 const struct mlxsw_sp_ipip_ops *ipip_ops;
1401 enum mlxsw_sp_ipip_type ipipt;
1403 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1404 ipip_ops = router->ipip_ops_arr[ipipt];
1405 if (dev->type == ipip_ops->dev_type) {
1414 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1415 const struct net_device *dev)
1417 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1420 static struct mlxsw_sp_ipip_entry *
1421 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1422 const struct net_device *ol_dev)
1424 struct mlxsw_sp_ipip_entry *ipip_entry;
1426 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1428 if (ipip_entry->ol_dev == ol_dev)
1434 static struct mlxsw_sp_ipip_entry *
1435 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1436 const struct net_device *ul_dev,
1437 struct mlxsw_sp_ipip_entry *start)
1439 struct mlxsw_sp_ipip_entry *ipip_entry;
1441 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1443 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1445 struct net_device *ol_dev = ipip_entry->ol_dev;
1446 struct net_device *ipip_ul_dev;
1449 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1452 if (ipip_ul_dev == ul_dev)
1459 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1460 const struct net_device *dev)
1464 mutex_lock(&mlxsw_sp->router->lock);
1465 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1466 mutex_unlock(&mlxsw_sp->router->lock);
1471 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1472 const struct net_device *ol_dev,
1473 enum mlxsw_sp_ipip_type ipipt)
1475 const struct mlxsw_sp_ipip_ops *ops
1476 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1478 return ops->can_offload(mlxsw_sp, ol_dev);
1481 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1482 struct net_device *ol_dev)
1484 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1485 struct mlxsw_sp_ipip_entry *ipip_entry;
1486 enum mlxsw_sp_l3proto ul_proto;
1487 union mlxsw_sp_l3addr saddr;
1490 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1491 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1492 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1493 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1494 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1495 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1498 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1500 if (IS_ERR(ipip_entry))
1501 return PTR_ERR(ipip_entry);
1508 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1509 struct net_device *ol_dev)
1511 struct mlxsw_sp_ipip_entry *ipip_entry;
1513 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1515 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1519 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1520 struct mlxsw_sp_ipip_entry *ipip_entry)
1522 struct mlxsw_sp_fib_entry *decap_fib_entry;
1524 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1525 if (decap_fib_entry)
1526 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1531 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1532 u16 ul_rif_id, bool enable)
1534 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1535 struct mlxsw_sp_rif *rif = &lb_rif->common;
1536 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1537 char ritr_pl[MLXSW_REG_RITR_LEN];
1540 switch (lb_cf.ul_protocol) {
1541 case MLXSW_SP_L3_PROTO_IPV4:
1542 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1543 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1544 rif->rif_index, rif->vr_id, rif->dev->mtu);
1545 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1546 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1547 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1550 case MLXSW_SP_L3_PROTO_IPV6:
1551 return -EAFNOSUPPORT;
1554 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1557 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1558 struct net_device *ol_dev)
1560 struct mlxsw_sp_ipip_entry *ipip_entry;
1561 struct mlxsw_sp_rif_ipip_lb *lb_rif;
1564 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1566 lb_rif = ipip_entry->ol_lb;
1567 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1568 lb_rif->ul_rif_id, true);
1571 lb_rif->common.mtu = ol_dev->mtu;
1578 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1579 struct net_device *ol_dev)
1581 struct mlxsw_sp_ipip_entry *ipip_entry;
1583 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1585 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1589 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1590 struct mlxsw_sp_ipip_entry *ipip_entry)
1592 if (ipip_entry->decap_fib_entry)
1593 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1596 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1597 struct net_device *ol_dev)
1599 struct mlxsw_sp_ipip_entry *ipip_entry;
1601 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1603 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1606 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1607 struct mlxsw_sp_rif *old_rif,
1608 struct mlxsw_sp_rif *new_rif);
1610 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1611 struct mlxsw_sp_ipip_entry *ipip_entry,
1613 struct netlink_ext_ack *extack)
1615 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1616 struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1618 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1622 if (IS_ERR(new_lb_rif))
1623 return PTR_ERR(new_lb_rif);
1624 ipip_entry->ol_lb = new_lb_rif;
1627 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1628 &new_lb_rif->common);
1630 mlxsw_sp_rif_destroy(&old_lb_rif->common);
1635 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1636 struct mlxsw_sp_rif *rif);
1639 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1640 * @mlxsw_sp: mlxsw_sp.
1641 * @ipip_entry: IPIP entry.
1642 * @recreate_loopback: Recreates the associated loopback RIF.
1643 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1644 * relevant when recreate_loopback is true.
1645 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1646 * is only relevant when recreate_loopback is false.
1649 * Return: Non-zero value on failure.
1651 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1652 struct mlxsw_sp_ipip_entry *ipip_entry,
1653 bool recreate_loopback,
1655 bool update_nexthops,
1656 struct netlink_ext_ack *extack)
1660 /* RIFs can't be edited, so to update loopback, we need to destroy and
1661 * recreate it. That creates a window of opportunity where RALUE and
1662 * RATR registers end up referencing a RIF that's already gone. RATRs
1663 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1664 * of RALUE, demote the decap route back.
1666 if (ipip_entry->decap_fib_entry)
1667 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1669 if (recreate_loopback) {
1670 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1671 keep_encap, extack);
1674 } else if (update_nexthops) {
1675 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1676 &ipip_entry->ol_lb->common);
1679 if (ipip_entry->ol_dev->flags & IFF_UP)
1680 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1685 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1686 struct net_device *ol_dev,
1687 struct netlink_ext_ack *extack)
1689 struct mlxsw_sp_ipip_entry *ipip_entry =
1690 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1695 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1696 true, false, false, extack);
1700 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1701 struct mlxsw_sp_ipip_entry *ipip_entry,
1702 struct net_device *ul_dev,
1704 struct netlink_ext_ack *extack)
1706 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1707 enum mlxsw_sp_l3proto ul_proto;
1708 union mlxsw_sp_l3addr saddr;
1710 /* Moving underlay to a different VRF might cause local address
1711 * conflict, and the conflicting tunnels need to be demoted.
1713 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1714 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1715 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1718 *demote_this = true;
1722 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1723 true, true, false, extack);
1727 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1728 struct mlxsw_sp_ipip_entry *ipip_entry,
1729 struct net_device *ul_dev)
1731 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1732 false, false, true, NULL);
1736 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1737 struct mlxsw_sp_ipip_entry *ipip_entry,
1738 struct net_device *ul_dev)
1740 /* A down underlay device causes encapsulated packets to not be
1741 * forwarded, but decap still works. So refresh next hops without
1742 * touching anything else.
1744 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1745 false, false, true, NULL);
1749 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1750 struct net_device *ol_dev,
1751 struct netlink_ext_ack *extack)
1753 const struct mlxsw_sp_ipip_ops *ipip_ops;
1754 struct mlxsw_sp_ipip_entry *ipip_entry;
1757 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1759 /* A change might make a tunnel eligible for offloading, but
1760 * that is currently not implemented. What falls to slow path
1765 /* A change might make a tunnel not eligible for offloading. */
1766 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1767 ipip_entry->ipipt)) {
1768 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1772 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1773 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1777 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1778 struct mlxsw_sp_ipip_entry *ipip_entry)
1780 struct net_device *ol_dev = ipip_entry->ol_dev;
1782 if (ol_dev->flags & IFF_UP)
1783 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1784 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1787 /* The configuration where several tunnels have the same local address in the
1788 * same underlay table needs special treatment in the HW. That is currently not
1789 * implemented in the driver. This function finds and demotes the first tunnel
1790 * with a given source address, except the one passed in in the argument
1794 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1795 enum mlxsw_sp_l3proto ul_proto,
1796 union mlxsw_sp_l3addr saddr,
1798 const struct mlxsw_sp_ipip_entry *except)
1800 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1802 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1804 if (ipip_entry != except &&
1805 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1806 ul_tb_id, ipip_entry)) {
1807 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1815 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1816 struct net_device *ul_dev)
1818 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1820 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1822 struct net_device *ol_dev = ipip_entry->ol_dev;
1823 struct net_device *ipip_ul_dev;
1826 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1828 if (ipip_ul_dev == ul_dev)
1829 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1833 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1834 struct net_device *ol_dev,
1835 unsigned long event,
1836 struct netdev_notifier_info *info)
1838 struct netdev_notifier_changeupper_info *chup;
1839 struct netlink_ext_ack *extack;
1842 mutex_lock(&mlxsw_sp->router->lock);
1844 case NETDEV_REGISTER:
1845 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1847 case NETDEV_UNREGISTER:
1848 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1851 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1854 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1856 case NETDEV_CHANGEUPPER:
1857 chup = container_of(info, typeof(*chup), info);
1858 extack = info->extack;
1859 if (netif_is_l3_master(chup->upper_dev))
1860 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1865 extack = info->extack;
1866 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1869 case NETDEV_CHANGEMTU:
1870 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1873 mutex_unlock(&mlxsw_sp->router->lock);
1878 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1879 struct mlxsw_sp_ipip_entry *ipip_entry,
1880 struct net_device *ul_dev,
1882 unsigned long event,
1883 struct netdev_notifier_info *info)
1885 struct netdev_notifier_changeupper_info *chup;
1886 struct netlink_ext_ack *extack;
1889 case NETDEV_CHANGEUPPER:
1890 chup = container_of(info, typeof(*chup), info);
1891 extack = info->extack;
1892 if (netif_is_l3_master(chup->upper_dev))
1893 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1901 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1904 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1912 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1913 struct net_device *ul_dev,
1914 unsigned long event,
1915 struct netdev_notifier_info *info)
1917 struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1920 mutex_lock(&mlxsw_sp->router->lock);
1921 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1924 struct mlxsw_sp_ipip_entry *prev;
1925 bool demote_this = false;
1927 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1928 ul_dev, &demote_this,
1931 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1937 if (list_is_first(&ipip_entry->ipip_list_node,
1938 &mlxsw_sp->router->ipip_list))
1941 /* This can't be cached from previous iteration,
1942 * because that entry could be gone now.
1944 prev = list_prev_entry(ipip_entry,
1946 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1950 mutex_unlock(&mlxsw_sp->router->lock);
1955 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1956 enum mlxsw_sp_l3proto ul_proto,
1957 const union mlxsw_sp_l3addr *ul_sip,
1960 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1961 struct mlxsw_sp_router *router = mlxsw_sp->router;
1962 struct mlxsw_sp_fib_entry *fib_entry;
1965 mutex_lock(&mlxsw_sp->router->lock);
1967 if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1972 router->nve_decap_config.ul_tb_id = ul_tb_id;
1973 router->nve_decap_config.tunnel_index = tunnel_index;
1974 router->nve_decap_config.ul_proto = ul_proto;
1975 router->nve_decap_config.ul_sip = *ul_sip;
1976 router->nve_decap_config.valid = true;
1978 /* It is valid to create a tunnel with a local IP and only later
1979 * assign this IP address to a local interface
1981 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1987 fib_entry->decap.tunnel_index = tunnel_index;
1988 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1990 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1992 goto err_fib_entry_update;
1996 err_fib_entry_update:
1997 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1998 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2000 mutex_unlock(&mlxsw_sp->router->lock);
2004 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2005 enum mlxsw_sp_l3proto ul_proto,
2006 const union mlxsw_sp_l3addr *ul_sip)
2008 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2009 struct mlxsw_sp_router *router = mlxsw_sp->router;
2010 struct mlxsw_sp_fib_entry *fib_entry;
2012 mutex_lock(&mlxsw_sp->router->lock);
2014 if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2017 router->nve_decap_config.valid = false;
2019 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2025 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2026 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2028 mutex_unlock(&mlxsw_sp->router->lock);
2031 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2033 enum mlxsw_sp_l3proto ul_proto,
2034 const union mlxsw_sp_l3addr *ul_sip)
2036 struct mlxsw_sp_router *router = mlxsw_sp->router;
2038 return router->nve_decap_config.valid &&
2039 router->nve_decap_config.ul_tb_id == ul_tb_id &&
2040 router->nve_decap_config.ul_proto == ul_proto &&
2041 !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2045 struct mlxsw_sp_neigh_key {
2046 struct neighbour *n;
2049 struct mlxsw_sp_neigh_entry {
2050 struct list_head rif_list_node;
2051 struct rhash_head ht_node;
2052 struct mlxsw_sp_neigh_key key;
2055 unsigned char ha[ETH_ALEN];
2056 struct list_head nexthop_list; /* list of nexthops using
2059 struct list_head nexthop_neighs_list_node;
2060 unsigned int counter_index;
2064 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2065 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2066 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2067 .key_len = sizeof(struct mlxsw_sp_neigh_key),
2070 struct mlxsw_sp_neigh_entry *
2071 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2072 struct mlxsw_sp_neigh_entry *neigh_entry)
2075 if (list_empty(&rif->neigh_list))
2078 return list_first_entry(&rif->neigh_list,
2079 typeof(*neigh_entry),
2082 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2084 return list_next_entry(neigh_entry, rif_list_node);
2087 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2089 return neigh_entry->key.n->tbl->family;
2093 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2095 return neigh_entry->ha;
2098 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2100 struct neighbour *n;
2102 n = neigh_entry->key.n;
2103 return ntohl(*((__be32 *) n->primary_key));
2107 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2109 struct neighbour *n;
2111 n = neigh_entry->key.n;
2112 return (struct in6_addr *) &n->primary_key;
2115 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2116 struct mlxsw_sp_neigh_entry *neigh_entry,
2119 if (!neigh_entry->counter_valid)
2122 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2126 static struct mlxsw_sp_neigh_entry *
2127 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2130 struct mlxsw_sp_neigh_entry *neigh_entry;
2132 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2136 neigh_entry->key.n = n;
2137 neigh_entry->rif = rif;
2138 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2143 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2149 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2150 struct mlxsw_sp_neigh_entry *neigh_entry)
2152 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2153 &neigh_entry->ht_node,
2154 mlxsw_sp_neigh_ht_params);
2158 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2159 struct mlxsw_sp_neigh_entry *neigh_entry)
2161 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2162 &neigh_entry->ht_node,
2163 mlxsw_sp_neigh_ht_params);
2167 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2168 struct mlxsw_sp_neigh_entry *neigh_entry)
2170 struct devlink *devlink;
2171 const char *table_name;
2173 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2175 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2178 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2185 devlink = priv_to_devlink(mlxsw_sp->core);
2186 return devlink_dpipe_table_counter_enabled(devlink, table_name);
2190 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2191 struct mlxsw_sp_neigh_entry *neigh_entry)
2193 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2196 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2199 neigh_entry->counter_valid = true;
2203 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2204 struct mlxsw_sp_neigh_entry *neigh_entry)
2206 if (!neigh_entry->counter_valid)
2208 mlxsw_sp_flow_counter_free(mlxsw_sp,
2209 neigh_entry->counter_index);
2210 neigh_entry->counter_valid = false;
2213 static struct mlxsw_sp_neigh_entry *
2214 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2216 struct mlxsw_sp_neigh_entry *neigh_entry;
2217 struct mlxsw_sp_rif *rif;
2220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2222 return ERR_PTR(-EINVAL);
2224 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2226 return ERR_PTR(-ENOMEM);
2228 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2230 goto err_neigh_entry_insert;
2232 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2233 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2237 err_neigh_entry_insert:
2238 mlxsw_sp_neigh_entry_free(neigh_entry);
2239 return ERR_PTR(err);
2243 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2244 struct mlxsw_sp_neigh_entry *neigh_entry)
2246 list_del(&neigh_entry->rif_list_node);
2247 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2248 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2249 mlxsw_sp_neigh_entry_free(neigh_entry);
2252 static struct mlxsw_sp_neigh_entry *
2253 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2255 struct mlxsw_sp_neigh_key key;
2258 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2259 &key, mlxsw_sp_neigh_ht_params);
2263 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2265 unsigned long interval;
2267 #if IS_ENABLED(CONFIG_IPV6)
2268 interval = min_t(unsigned long,
2269 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2270 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2272 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2274 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2277 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2281 struct net_device *dev;
2282 struct neighbour *n;
2287 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2289 if (!mlxsw_sp->router->rifs[rif]) {
2290 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2295 dev = mlxsw_sp->router->rifs[rif]->dev;
2296 n = neigh_lookup(&arp_tbl, &dipn, dev);
2300 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2301 neigh_event_send(n, NULL);
2305 #if IS_ENABLED(CONFIG_IPV6)
2306 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2310 struct net_device *dev;
2311 struct neighbour *n;
2312 struct in6_addr dip;
2315 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2318 if (!mlxsw_sp->router->rifs[rif]) {
2319 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2323 dev = mlxsw_sp->router->rifs[rif]->dev;
2324 n = neigh_lookup(&nd_tbl, &dip, dev);
2328 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2329 neigh_event_send(n, NULL);
2333 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2340 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2347 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2349 /* Hardware starts counting at 0, so add 1. */
2352 /* Each record consists of several neighbour entries. */
2353 for (i = 0; i < num_entries; i++) {
2356 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2357 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2363 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2367 /* One record contains one entry. */
2368 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2372 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2373 char *rauhtd_pl, int rec_index)
2375 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2376 case MLXSW_REG_RAUHTD_TYPE_IPV4:
2377 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2380 case MLXSW_REG_RAUHTD_TYPE_IPV6:
2381 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2387 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2389 u8 num_rec, last_rec_index, num_entries;
2391 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2392 last_rec_index = num_rec - 1;
2394 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2396 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2397 MLXSW_REG_RAUHTD_TYPE_IPV6)
2400 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2402 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2408 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2410 enum mlxsw_reg_rauhtd_type type)
2415 /* Ensure the RIF we read from the device does not change mid-dump. */
2416 mutex_lock(&mlxsw_sp->router->lock);
2418 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2419 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2422 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2425 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2426 for (i = 0; i < num_rec; i++)
2427 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2429 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2430 mutex_unlock(&mlxsw_sp->router->lock);
2435 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2437 enum mlxsw_reg_rauhtd_type type;
2441 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2445 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2446 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2450 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2451 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2457 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2459 struct mlxsw_sp_neigh_entry *neigh_entry;
2461 mutex_lock(&mlxsw_sp->router->lock);
2462 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2463 nexthop_neighs_list_node)
2464 /* If this neigh have nexthops, make the kernel think this neigh
2465 * is active regardless of the traffic.
2467 neigh_event_send(neigh_entry->key.n, NULL);
2468 mutex_unlock(&mlxsw_sp->router->lock);
2472 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2474 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2476 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2477 msecs_to_jiffies(interval));
2480 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2482 struct mlxsw_sp_router *router;
2485 router = container_of(work, struct mlxsw_sp_router,
2486 neighs_update.dw.work);
2487 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2489 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2491 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2493 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2496 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2498 struct mlxsw_sp_neigh_entry *neigh_entry;
2499 struct mlxsw_sp_router *router;
2501 router = container_of(work, struct mlxsw_sp_router,
2502 nexthop_probe_dw.work);
2503 /* Iterate over nexthop neighbours, find those who are unresolved and
2504 * send arp on them. This solves the chicken-egg problem when
2505 * the nexthop wouldn't get offloaded until the neighbor is resolved
2506 * but it wouldn't get resolved ever in case traffic is flowing in HW
2507 * using different nexthop.
2509 mutex_lock(&router->lock);
2510 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2511 nexthop_neighs_list_node)
2512 if (!neigh_entry->connected)
2513 neigh_event_send(neigh_entry->key.n, NULL);
2514 mutex_unlock(&router->lock);
2516 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2517 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2521 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2522 struct mlxsw_sp_neigh_entry *neigh_entry,
2523 bool removing, bool dead);
2525 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2527 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2528 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2532 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2533 struct mlxsw_sp_neigh_entry *neigh_entry,
2534 enum mlxsw_reg_rauht_op op)
2536 struct neighbour *n = neigh_entry->key.n;
2537 u32 dip = ntohl(*((__be32 *) n->primary_key));
2538 char rauht_pl[MLXSW_REG_RAUHT_LEN];
2540 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2542 if (neigh_entry->counter_valid)
2543 mlxsw_reg_rauht_pack_counter(rauht_pl,
2544 neigh_entry->counter_index);
2545 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2549 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2550 struct mlxsw_sp_neigh_entry *neigh_entry,
2551 enum mlxsw_reg_rauht_op op)
2553 struct neighbour *n = neigh_entry->key.n;
2554 char rauht_pl[MLXSW_REG_RAUHT_LEN];
2555 const char *dip = n->primary_key;
2557 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2559 if (neigh_entry->counter_valid)
2560 mlxsw_reg_rauht_pack_counter(rauht_pl,
2561 neigh_entry->counter_index);
2562 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2565 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2567 struct neighbour *n = neigh_entry->key.n;
2569 /* Packets with a link-local destination address are trapped
2570 * after LPM lookup and never reach the neighbour table, so
2571 * there is no need to program such neighbours to the device.
2573 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2574 IPV6_ADDR_LINKLOCAL)
2580 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2581 struct mlxsw_sp_neigh_entry *neigh_entry,
2584 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2587 if (!adding && !neigh_entry->connected)
2589 neigh_entry->connected = adding;
2590 if (neigh_entry->key.n->tbl->family == AF_INET) {
2591 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2595 } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2596 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2598 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2608 neigh_entry->key.n->flags |= NTF_OFFLOADED;
2610 neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2614 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2615 struct mlxsw_sp_neigh_entry *neigh_entry,
2619 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2621 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2622 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2625 struct mlxsw_sp_netevent_work {
2626 struct work_struct work;
2627 struct mlxsw_sp *mlxsw_sp;
2628 struct neighbour *n;
2631 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2633 struct mlxsw_sp_netevent_work *net_work =
2634 container_of(work, struct mlxsw_sp_netevent_work, work);
2635 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2636 struct mlxsw_sp_neigh_entry *neigh_entry;
2637 struct neighbour *n = net_work->n;
2638 unsigned char ha[ETH_ALEN];
2639 bool entry_connected;
2642 /* If these parameters are changed after we release the lock,
2643 * then we are guaranteed to receive another event letting us
2646 read_lock_bh(&n->lock);
2647 memcpy(ha, n->ha, ETH_ALEN);
2648 nud_state = n->nud_state;
2650 read_unlock_bh(&n->lock);
2652 mutex_lock(&mlxsw_sp->router->lock);
2653 mlxsw_sp_span_respin(mlxsw_sp);
2655 entry_connected = nud_state & NUD_VALID && !dead;
2656 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2657 if (!entry_connected && !neigh_entry)
2660 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2661 if (IS_ERR(neigh_entry))
2665 if (neigh_entry->connected && entry_connected &&
2666 !memcmp(neigh_entry->ha, ha, ETH_ALEN))
2669 memcpy(neigh_entry->ha, ha, ETH_ALEN);
2670 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2671 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2674 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2675 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2678 mutex_unlock(&mlxsw_sp->router->lock);
2683 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2685 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2687 struct mlxsw_sp_netevent_work *net_work =
2688 container_of(work, struct mlxsw_sp_netevent_work, work);
2689 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2691 mlxsw_sp_mp_hash_init(mlxsw_sp);
2695 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2697 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2699 struct mlxsw_sp_netevent_work *net_work =
2700 container_of(work, struct mlxsw_sp_netevent_work, work);
2701 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2703 __mlxsw_sp_router_init(mlxsw_sp);
2707 static int mlxsw_sp_router_schedule_work(struct net *net,
2708 struct notifier_block *nb,
2709 void (*cb)(struct work_struct *))
2711 struct mlxsw_sp_netevent_work *net_work;
2712 struct mlxsw_sp_router *router;
2714 router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2715 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2718 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2722 INIT_WORK(&net_work->work, cb);
2723 net_work->mlxsw_sp = router->mlxsw_sp;
2724 mlxsw_core_schedule_work(&net_work->work);
2728 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2729 unsigned long event, void *ptr)
2731 struct mlxsw_sp_netevent_work *net_work;
2732 struct mlxsw_sp_port *mlxsw_sp_port;
2733 struct mlxsw_sp *mlxsw_sp;
2734 unsigned long interval;
2735 struct neigh_parms *p;
2736 struct neighbour *n;
2739 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2742 /* We don't care about changes in the default table. */
2743 if (!p->dev || (p->tbl->family != AF_INET &&
2744 p->tbl->family != AF_INET6))
2747 /* We are in atomic context and can't take RTNL mutex,
2748 * so use RCU variant to walk the device chain.
2750 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2754 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2755 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2756 mlxsw_sp->router->neighs_update.interval = interval;
2758 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2760 case NETEVENT_NEIGH_UPDATE:
2763 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2766 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2770 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2772 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2776 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2777 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2780 /* Take a reference to ensure the neighbour won't be
2781 * destructed until we drop the reference in delayed
2785 mlxsw_core_schedule_work(&net_work->work);
2786 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2788 case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2789 case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2790 return mlxsw_sp_router_schedule_work(ptr, nb,
2791 mlxsw_sp_router_mp_hash_event_work);
2793 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2794 return mlxsw_sp_router_schedule_work(ptr, nb,
2795 mlxsw_sp_router_update_priority_work);
2801 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2805 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2806 &mlxsw_sp_neigh_ht_params);
2810 /* Initialize the polling interval according to the default
2813 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2815 /* Create the delayed works for the activity_update */
2816 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2817 mlxsw_sp_router_neighs_update_work);
2818 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2819 mlxsw_sp_router_probe_unresolved_nexthops);
2820 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2821 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2825 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2827 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2828 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2829 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2832 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2833 struct mlxsw_sp_rif *rif)
2835 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2837 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2839 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2840 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2844 enum mlxsw_sp_nexthop_type {
2845 MLXSW_SP_NEXTHOP_TYPE_ETH,
2846 MLXSW_SP_NEXTHOP_TYPE_IPIP,
2849 enum mlxsw_sp_nexthop_action {
2850 /* Nexthop forwards packets to an egress RIF */
2851 MLXSW_SP_NEXTHOP_ACTION_FORWARD,
2852 /* Nexthop discards packets */
2853 MLXSW_SP_NEXTHOP_ACTION_DISCARD,
2854 /* Nexthop traps packets */
2855 MLXSW_SP_NEXTHOP_ACTION_TRAP,
2858 struct mlxsw_sp_nexthop_key {
2859 struct fib_nh *fib_nh;
2862 struct mlxsw_sp_nexthop {
2863 struct list_head neigh_list_node; /* member of neigh entry list */
2864 struct list_head rif_list_node;
2865 struct list_head router_list_node;
2866 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2867 * this nexthop belongs to
2869 struct rhash_head ht_node;
2870 struct neigh_table *neigh_tbl;
2871 struct mlxsw_sp_nexthop_key key;
2872 unsigned char gw_addr[sizeof(struct in6_addr)];
2876 int num_adj_entries;
2877 struct mlxsw_sp_rif *rif;
2878 u8 should_offload:1, /* set indicates this nexthop should be written
2879 * to the adjacency table.
2881 offloaded:1, /* set indicates this nexthop was written to the
2884 update:1; /* set indicates this nexthop should be updated in the
2885 * adjacency table (f.e., its MAC changed).
2887 enum mlxsw_sp_nexthop_action action;
2888 enum mlxsw_sp_nexthop_type type;
2890 struct mlxsw_sp_neigh_entry *neigh_entry;
2891 struct mlxsw_sp_ipip_entry *ipip_entry;
2893 unsigned int counter_index;
2897 enum mlxsw_sp_nexthop_group_type {
2898 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2899 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2900 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
2903 struct mlxsw_sp_nexthop_group_info {
2904 struct mlxsw_sp_nexthop_group *nh_grp;
2908 int sum_norm_weight;
2909 u8 adj_index_valid:1,
2910 gateway:1; /* routes using the group use a gateway */
2911 struct mlxsw_sp_nexthop nexthops[0];
2912 #define nh_rif nexthops[0].rif
2915 struct mlxsw_sp_nexthop_group_vr_key {
2917 enum mlxsw_sp_l3proto proto;
2920 struct mlxsw_sp_nexthop_group_vr_entry {
2921 struct list_head list; /* member in vr_list */
2922 struct rhash_head ht_node; /* member in vr_ht */
2923 refcount_t ref_count;
2924 struct mlxsw_sp_nexthop_group_vr_key key;
2927 struct mlxsw_sp_nexthop_group {
2928 struct rhash_head ht_node;
2929 struct list_head fib_list; /* list of fib entries that use this group */
2932 struct fib_info *fi;
2938 struct mlxsw_sp_nexthop_group_info *nhgi;
2939 struct list_head vr_list;
2940 struct rhashtable vr_ht;
2941 enum mlxsw_sp_nexthop_group_type type;
2945 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2946 struct mlxsw_sp_nexthop *nh)
2948 struct devlink *devlink;
2950 devlink = priv_to_devlink(mlxsw_sp->core);
2951 if (!devlink_dpipe_table_counter_enabled(devlink,
2952 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2955 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2958 nh->counter_valid = true;
2961 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2962 struct mlxsw_sp_nexthop *nh)
2964 if (!nh->counter_valid)
2966 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2967 nh->counter_valid = false;
2970 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2971 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2973 if (!nh->counter_valid)
2976 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2980 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2981 struct mlxsw_sp_nexthop *nh)
2984 if (list_empty(&router->nexthop_list))
2987 return list_first_entry(&router->nexthop_list,
2988 typeof(*nh), router_list_node);
2990 if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2992 return list_next_entry(nh, router_list_node);
2995 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
2997 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3000 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3002 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3003 !mlxsw_sp_nexthop_is_forward(nh))
3005 return nh->neigh_entry->ha;
3008 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3009 u32 *p_adj_size, u32 *p_adj_hash_index)
3011 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3012 u32 adj_hash_index = 0;
3015 if (!nh->offloaded || !nhgi->adj_index_valid)
3018 *p_adj_index = nhgi->adj_index;
3019 *p_adj_size = nhgi->ecmp_size;
3021 for (i = 0; i < nhgi->count; i++) {
3022 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3026 if (nh_iter->offloaded)
3027 adj_hash_index += nh_iter->num_adj_entries;
3030 *p_adj_hash_index = adj_hash_index;
3034 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3039 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3041 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3044 for (i = 0; i < nhgi->count; i++) {
3045 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3047 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3053 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3054 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3055 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3056 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3057 .automatic_shrinking = true,
3060 static struct mlxsw_sp_nexthop_group_vr_entry *
3061 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3062 const struct mlxsw_sp_fib *fib)
3064 struct mlxsw_sp_nexthop_group_vr_key key;
3066 memset(&key, 0, sizeof(key));
3067 key.vr_id = fib->vr->id;
3068 key.proto = fib->proto;
3069 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
3070 mlxsw_sp_nexthop_group_vr_ht_params);
3074 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3075 const struct mlxsw_sp_fib *fib)
3077 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3080 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
3084 vr_entry->key.vr_id = fib->vr->id;
3085 vr_entry->key.proto = fib->proto;
3086 refcount_set(&vr_entry->ref_count, 1);
3088 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3089 mlxsw_sp_nexthop_group_vr_ht_params);
3091 goto err_hashtable_insert;
3093 list_add(&vr_entry->list, &nh_grp->vr_list);
3097 err_hashtable_insert:
3103 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3104 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3106 list_del(&vr_entry->list);
3107 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3108 mlxsw_sp_nexthop_group_vr_ht_params);
3113 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3114 const struct mlxsw_sp_fib *fib)
3116 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3118 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3120 refcount_inc(&vr_entry->ref_count);
3124 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3128 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3129 const struct mlxsw_sp_fib *fib)
3131 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3133 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3134 if (WARN_ON_ONCE(!vr_entry))
3137 if (!refcount_dec_and_test(&vr_entry->ref_count))
3140 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3143 struct mlxsw_sp_nexthop_group_cmp_arg {
3144 enum mlxsw_sp_nexthop_group_type type;
3146 struct fib_info *fi;
3147 struct mlxsw_sp_fib6_entry *fib6_entry;
3153 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3154 const struct in6_addr *gw, int ifindex,
3159 for (i = 0; i < nh_grp->nhgi->count; i++) {
3160 const struct mlxsw_sp_nexthop *nh;
3162 nh = &nh_grp->nhgi->nexthops[i];
3163 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3164 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3172 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3173 const struct mlxsw_sp_fib6_entry *fib6_entry)
3175 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3177 if (nh_grp->nhgi->count != fib6_entry->nrt6)
3180 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3181 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3182 struct in6_addr *gw;
3183 int ifindex, weight;
3185 ifindex = fib6_nh->fib_nh_dev->ifindex;
3186 weight = fib6_nh->fib_nh_weight;
3187 gw = &fib6_nh->fib_nh_gw6;
3188 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3197 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3199 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3200 const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3202 if (nh_grp->type != cmp_arg->type)
3205 switch (cmp_arg->type) {
3206 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3207 return cmp_arg->fi != nh_grp->ipv4.fi;
3208 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3209 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3210 cmp_arg->fib6_entry);
3211 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3212 return cmp_arg->id != nh_grp->obj.id;
3219 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3221 const struct mlxsw_sp_nexthop_group *nh_grp = data;
3222 const struct mlxsw_sp_nexthop *nh;
3223 struct fib_info *fi;
3227 switch (nh_grp->type) {
3228 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3229 fi = nh_grp->ipv4.fi;
3230 return jhash(&fi, sizeof(fi), seed);
3231 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3232 val = nh_grp->nhgi->count;
3233 for (i = 0; i < nh_grp->nhgi->count; i++) {
3234 nh = &nh_grp->nhgi->nexthops[i];
3235 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3236 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3238 return jhash(&val, sizeof(val), seed);
3239 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3240 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
3248 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3250 unsigned int val = fib6_entry->nrt6;
3251 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3253 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3254 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3255 struct net_device *dev = fib6_nh->fib_nh_dev;
3256 struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3258 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3259 val ^= jhash(gw, sizeof(*gw), seed);
3262 return jhash(&val, sizeof(val), seed);
3266 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3268 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3270 switch (cmp_arg->type) {
3271 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3272 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3273 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3274 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3275 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3276 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
3283 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3284 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3285 .hashfn = mlxsw_sp_nexthop_group_hash,
3286 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
3287 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
3290 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3291 struct mlxsw_sp_nexthop_group *nh_grp)
3293 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3294 !nh_grp->nhgi->gateway)
3297 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3299 mlxsw_sp_nexthop_group_ht_params);
3302 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3303 struct mlxsw_sp_nexthop_group *nh_grp)
3305 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3306 !nh_grp->nhgi->gateway)
3309 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3311 mlxsw_sp_nexthop_group_ht_params);
3314 static struct mlxsw_sp_nexthop_group *
3315 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3316 struct fib_info *fi)
3318 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3320 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3322 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3324 mlxsw_sp_nexthop_group_ht_params);
3327 static struct mlxsw_sp_nexthop_group *
3328 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3329 struct mlxsw_sp_fib6_entry *fib6_entry)
3331 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3333 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3334 cmp_arg.fib6_entry = fib6_entry;
3335 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3337 mlxsw_sp_nexthop_group_ht_params);
3340 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3341 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3342 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3343 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3346 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3347 struct mlxsw_sp_nexthop *nh)
3349 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3350 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3353 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3354 struct mlxsw_sp_nexthop *nh)
3356 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3357 mlxsw_sp_nexthop_ht_params);
3360 static struct mlxsw_sp_nexthop *
3361 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3362 struct mlxsw_sp_nexthop_key key)
3364 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3365 mlxsw_sp_nexthop_ht_params);
3368 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3369 enum mlxsw_sp_l3proto proto,
3371 u32 adj_index, u16 ecmp_size,
3375 char raleu_pl[MLXSW_REG_RALEU_LEN];
3377 mlxsw_reg_raleu_pack(raleu_pl,
3378 (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
3379 adj_index, ecmp_size, new_adj_index,
3381 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3384 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3385 struct mlxsw_sp_nexthop_group *nh_grp,
3386 u32 old_adj_index, u16 old_ecmp_size)
3388 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3389 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3392 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3393 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3394 vr_entry->key.proto,
3395 vr_entry->key.vr_id,
3401 goto err_mass_update_vr;
3406 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3407 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
3408 vr_entry->key.vr_id,
3411 old_adj_index, old_ecmp_size);
3415 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3417 struct mlxsw_sp_nexthop *nh)
3419 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3420 char ratr_pl[MLXSW_REG_RATR_LEN];
3423 rif_index = nh->rif ? nh->rif->rif_index :
3424 mlxsw_sp->router->lb_rif_index;
3425 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3426 true, MLXSW_REG_RATR_TYPE_ETHERNET,
3427 adj_index, rif_index);
3428 switch (nh->action) {
3429 case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3430 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3432 case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3433 mlxsw_reg_ratr_trap_action_set(ratr_pl,
3434 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3436 case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3437 mlxsw_reg_ratr_trap_action_set(ratr_pl,
3438 MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3439 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
3445 if (nh->counter_valid)
3446 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3448 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3450 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3453 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3454 struct mlxsw_sp_nexthop *nh)
3458 for (i = 0; i < nh->num_adj_entries; i++) {
3461 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
3470 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3472 struct mlxsw_sp_nexthop *nh)
3474 const struct mlxsw_sp_ipip_ops *ipip_ops;
3476 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3477 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3480 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3482 struct mlxsw_sp_nexthop *nh)
3486 for (i = 0; i < nh->num_adj_entries; i++) {
3489 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3498 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3499 struct mlxsw_sp_nexthop *nh)
3501 /* When action is discard or trap, the nexthop must be
3502 * programmed as an Ethernet nexthop.
3504 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3505 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3506 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3507 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh);
3509 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh);
3513 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3514 struct mlxsw_sp_nexthop_group_info *nhgi,
3517 u32 adj_index = nhgi->adj_index; /* base */
3518 struct mlxsw_sp_nexthop *nh;
3521 for (i = 0; i < nhgi->count; i++) {
3522 nh = &nhgi->nexthops[i];
3524 if (!nh->should_offload) {
3529 if (nh->update || reallocate) {
3532 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh);
3538 adj_index += nh->num_adj_entries;
3544 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3545 struct mlxsw_sp_nexthop_group *nh_grp)
3547 struct mlxsw_sp_fib_entry *fib_entry;
3550 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3551 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3558 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3560 /* Valid sizes for an adjacency group are:
3561 * 1-64, 512, 1024, 2048 and 4096.
3563 if (*p_adj_grp_size <= 64)
3565 else if (*p_adj_grp_size <= 512)
3566 *p_adj_grp_size = 512;
3567 else if (*p_adj_grp_size <= 1024)
3568 *p_adj_grp_size = 1024;
3569 else if (*p_adj_grp_size <= 2048)
3570 *p_adj_grp_size = 2048;
3572 *p_adj_grp_size = 4096;
3575 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3576 unsigned int alloc_size)
3578 if (alloc_size >= 4096)
3579 *p_adj_grp_size = 4096;
3580 else if (alloc_size >= 2048)
3581 *p_adj_grp_size = 2048;
3582 else if (alloc_size >= 1024)
3583 *p_adj_grp_size = 1024;
3584 else if (alloc_size >= 512)
3585 *p_adj_grp_size = 512;
3588 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3589 u16 *p_adj_grp_size)
3591 unsigned int alloc_size;
3594 /* Round up the requested group size to the next size supported
3595 * by the device and make sure the request can be satisfied.
3597 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3598 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3599 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3600 *p_adj_grp_size, &alloc_size);
3603 /* It is possible the allocation results in more allocated
3604 * entries than requested. Try to use as much of them as
3607 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3613 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3615 int i, g = 0, sum_norm_weight = 0;
3616 struct mlxsw_sp_nexthop *nh;
3618 for (i = 0; i < nhgi->count; i++) {
3619 nh = &nhgi->nexthops[i];
3621 if (!nh->should_offload)
3624 g = gcd(nh->nh_weight, g);
3629 for (i = 0; i < nhgi->count; i++) {
3630 nh = &nhgi->nexthops[i];
3632 if (!nh->should_offload)
3634 nh->norm_nh_weight = nh->nh_weight / g;
3635 sum_norm_weight += nh->norm_nh_weight;
3638 nhgi->sum_norm_weight = sum_norm_weight;
3642 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3644 int i, weight = 0, lower_bound = 0;
3645 int total = nhgi->sum_norm_weight;
3646 u16 ecmp_size = nhgi->ecmp_size;
3648 for (i = 0; i < nhgi->count; i++) {
3649 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3652 if (!nh->should_offload)
3654 weight += nh->norm_nh_weight;
3655 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3656 nh->num_adj_entries = upper_bound - lower_bound;
3657 lower_bound = upper_bound;
3661 static struct mlxsw_sp_nexthop *
3662 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3663 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3666 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3667 struct mlxsw_sp_nexthop_group *nh_grp)
3671 for (i = 0; i < nh_grp->nhgi->count; i++) {
3672 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3675 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3677 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3682 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3683 struct mlxsw_sp_fib6_entry *fib6_entry)
3685 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3687 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3688 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3689 struct mlxsw_sp_nexthop *nh;
3691 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3692 if (nh && nh->offloaded)
3693 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3695 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3700 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3701 struct mlxsw_sp_nexthop_group *nh_grp)
3703 struct mlxsw_sp_fib6_entry *fib6_entry;
3705 /* Unfortunately, in IPv6 the route and the nexthop are described by
3706 * the same struct, so we need to iterate over all the routes using the
3707 * nexthop group and set / clear the offload indication for them.
3709 list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3710 common.nexthop_group_node)
3711 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3715 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3716 struct mlxsw_sp_nexthop_group *nh_grp)
3718 /* Do not update the flags if the nexthop group is being destroyed
3720 * 1. The nexthop objects is being deleted, in which case the flags are
3722 * 2. The nexthop group was replaced by a newer group, in which case
3723 * the flags of the nexthop object were already updated based on the
3726 if (nh_grp->can_destroy)
3729 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3730 nh_grp->nhgi->adj_index_valid, false);
3734 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3735 struct mlxsw_sp_nexthop_group *nh_grp)
3737 switch (nh_grp->type) {
3738 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3739 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3741 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3742 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3744 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3745 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
3751 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3752 struct mlxsw_sp_nexthop_group *nh_grp)
3754 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3755 u16 ecmp_size, old_ecmp_size;
3756 struct mlxsw_sp_nexthop *nh;
3757 bool offload_change = false;
3759 bool old_adj_index_valid;
3760 int i, err2, err = 0;
3764 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3766 for (i = 0; i < nhgi->count; i++) {
3767 nh = &nhgi->nexthops[i];
3769 if (nh->should_offload != nh->offloaded) {
3770 offload_change = true;
3771 if (nh->should_offload)
3775 if (!offload_change) {
3776 /* Nothing was added or removed, so no need to reallocate. Just
3777 * update MAC on existing adjacency indexes.
3779 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3781 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3786 mlxsw_sp_nexthop_group_normalize(nhgi);
3787 if (!nhgi->sum_norm_weight)
3788 /* No neigh of this group is connected so we just set
3789 * the trap and let everthing flow through kernel.
3793 ecmp_size = nhgi->sum_norm_weight;
3794 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3796 /* No valid allocation size available. */
3799 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3800 ecmp_size, &adj_index);
3802 /* We ran out of KVD linear space, just set the
3803 * trap and let everything flow through kernel.
3805 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3808 old_adj_index_valid = nhgi->adj_index_valid;
3809 old_adj_index = nhgi->adj_index;
3810 old_ecmp_size = nhgi->ecmp_size;
3811 nhgi->adj_index_valid = 1;
3812 nhgi->adj_index = adj_index;
3813 nhgi->ecmp_size = ecmp_size;
3814 mlxsw_sp_nexthop_group_rebalance(nhgi);
3815 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3817 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3821 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3823 if (!old_adj_index_valid) {
3824 /* The trap was set for fib entries, so we have to call
3825 * fib entry update to unset it and use adjacency index.
3827 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3829 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3835 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3836 old_adj_index, old_ecmp_size);
3837 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3838 old_ecmp_size, old_adj_index);
3840 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3847 old_adj_index_valid = nhgi->adj_index_valid;
3848 nhgi->adj_index_valid = 0;
3849 for (i = 0; i < nhgi->count; i++) {
3850 nh = &nhgi->nexthops[i];
3853 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3855 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3856 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3857 if (old_adj_index_valid)
3858 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3859 nhgi->ecmp_size, nhgi->adj_index);
3863 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3867 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3868 nh->should_offload = 1;
3870 nh->should_offload = 0;
3876 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3877 struct mlxsw_sp_neigh_entry *neigh_entry)
3879 struct neighbour *n, *old_n = neigh_entry->key.n;
3880 struct mlxsw_sp_nexthop *nh;
3881 bool entry_connected;
3885 nh = list_first_entry(&neigh_entry->nexthop_list,
3886 struct mlxsw_sp_nexthop, neigh_list_node);
3888 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3890 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3893 neigh_event_send(n, NULL);
3896 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3897 neigh_entry->key.n = n;
3898 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3900 goto err_neigh_entry_insert;
3902 read_lock_bh(&n->lock);
3903 nud_state = n->nud_state;
3905 read_unlock_bh(&n->lock);
3906 entry_connected = nud_state & NUD_VALID && !dead;
3908 list_for_each_entry(nh, &neigh_entry->nexthop_list,
3910 neigh_release(old_n);
3912 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
3913 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3920 err_neigh_entry_insert:
3921 neigh_entry->key.n = old_n;
3922 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3928 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3929 struct mlxsw_sp_neigh_entry *neigh_entry,
3930 bool removing, bool dead)
3932 struct mlxsw_sp_nexthop *nh;
3934 if (list_empty(&neigh_entry->nexthop_list))
3940 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
3943 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
3947 list_for_each_entry(nh, &neigh_entry->nexthop_list,
3949 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3950 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
3954 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3955 struct mlxsw_sp_rif *rif)
3961 list_add(&nh->rif_list_node, &rif->nexthop_list);
3964 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3969 list_del(&nh->rif_list_node);
3973 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3974 struct mlxsw_sp_nexthop *nh)
3976 struct mlxsw_sp_neigh_entry *neigh_entry;
3977 struct neighbour *n;
3981 if (!nh->nhgi->gateway || nh->neigh_entry)
3984 /* Take a reference of neigh here ensuring that neigh would
3985 * not be destructed before the nexthop entry is finished.
3986 * The reference is taken either in neigh_lookup() or
3987 * in neigh_create() in case n is not found.
3989 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3991 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3994 neigh_event_send(n, NULL);
3996 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3998 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3999 if (IS_ERR(neigh_entry)) {
4001 goto err_neigh_entry_create;
4005 /* If that is the first nexthop connected to that neigh, add to
4006 * nexthop_neighs_list
4008 if (list_empty(&neigh_entry->nexthop_list))
4009 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
4010 &mlxsw_sp->router->nexthop_neighs_list);
4012 nh->neigh_entry = neigh_entry;
4013 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
4014 read_lock_bh(&n->lock);
4015 nud_state = n->nud_state;
4017 read_unlock_bh(&n->lock);
4018 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
4022 err_neigh_entry_create:
4027 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4028 struct mlxsw_sp_nexthop *nh)
4030 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4031 struct neighbour *n;
4035 n = neigh_entry->key.n;
4037 __mlxsw_sp_nexthop_neigh_update(nh, true);
4038 list_del(&nh->neigh_list_node);
4039 nh->neigh_entry = NULL;
4041 /* If that is the last nexthop connected to that neigh, remove from
4042 * nexthop_neighs_list
4044 if (list_empty(&neigh_entry->nexthop_list))
4045 list_del(&neigh_entry->nexthop_neighs_list_node);
4047 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
4048 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4053 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4055 struct net_device *ul_dev;
4059 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4060 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4066 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4067 struct mlxsw_sp_nexthop *nh,
4068 struct mlxsw_sp_ipip_entry *ipip_entry)
4072 if (!nh->nhgi->gateway || nh->ipip_entry)
4075 nh->ipip_entry = ipip_entry;
4076 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
4077 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4078 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
4081 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4082 struct mlxsw_sp_nexthop *nh)
4084 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4089 __mlxsw_sp_nexthop_neigh_update(nh, true);
4090 nh->ipip_entry = NULL;
4093 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4094 const struct fib_nh *fib_nh,
4095 enum mlxsw_sp_ipip_type *p_ipipt)
4097 struct net_device *dev = fib_nh->fib_nh_dev;
4100 fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4101 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
4104 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4105 struct mlxsw_sp_nexthop *nh,
4106 const struct net_device *dev)
4108 const struct mlxsw_sp_ipip_ops *ipip_ops;
4109 struct mlxsw_sp_ipip_entry *ipip_entry;
4110 struct mlxsw_sp_rif *rif;
4113 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4115 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4116 if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4117 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4118 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4123 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4124 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4128 mlxsw_sp_nexthop_rif_init(nh, rif);
4129 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4131 goto err_neigh_init;
4136 mlxsw_sp_nexthop_rif_fini(nh);
4140 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4141 struct mlxsw_sp_nexthop *nh)
4144 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4145 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4146 mlxsw_sp_nexthop_rif_fini(nh);
4148 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4149 mlxsw_sp_nexthop_rif_fini(nh);
4150 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4155 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4156 struct mlxsw_sp_nexthop_group *nh_grp,
4157 struct mlxsw_sp_nexthop *nh,
4158 struct fib_nh *fib_nh)
4160 struct net_device *dev = fib_nh->fib_nh_dev;
4161 struct in_device *in_dev;
4164 nh->nhgi = nh_grp->nhgi;
4165 nh->key.fib_nh = fib_nh;
4166 #ifdef CONFIG_IP_ROUTE_MULTIPATH
4167 nh->nh_weight = fib_nh->fib_nh_weight;
4171 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4172 nh->neigh_tbl = &arp_tbl;
4173 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4177 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4178 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4182 nh->ifindex = dev->ifindex;
4185 in_dev = __in_dev_get_rcu(dev);
4186 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4187 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4193 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4195 goto err_nexthop_neigh_init;
4199 err_nexthop_neigh_init:
4200 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4204 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4205 struct mlxsw_sp_nexthop *nh)
4207 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4208 list_del(&nh->router_list_node);
4209 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4210 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4213 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4214 unsigned long event, struct fib_nh *fib_nh)
4216 struct mlxsw_sp_nexthop_key key;
4217 struct mlxsw_sp_nexthop *nh;
4219 if (mlxsw_sp->router->aborted)
4222 key.fib_nh = fib_nh;
4223 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4228 case FIB_EVENT_NH_ADD:
4229 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4231 case FIB_EVENT_NH_DEL:
4232 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4236 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4239 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4240 struct mlxsw_sp_rif *rif)
4242 struct mlxsw_sp_nexthop *nh;
4245 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4247 case MLXSW_SP_NEXTHOP_TYPE_ETH:
4250 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4251 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4258 __mlxsw_sp_nexthop_neigh_update(nh, removing);
4259 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4263 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4264 struct mlxsw_sp_rif *old_rif,
4265 struct mlxsw_sp_rif *new_rif)
4267 struct mlxsw_sp_nexthop *nh;
4269 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4270 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4272 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4275 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4276 struct mlxsw_sp_rif *rif)
4278 struct mlxsw_sp_nexthop *nh, *tmp;
4280 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4281 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4282 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4287 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4288 const struct nh_notifier_single_info *nh,
4289 struct netlink_ext_ack *extack)
4294 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4295 else if (nh->has_encap)
4296 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4304 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4305 const struct nh_notifier_single_info *nh,
4306 struct netlink_ext_ack *extack)
4310 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4314 /* Device only nexthops with an IPIP device are programmed as
4315 * encapsulating adjacency entries.
4317 if (!nh->gw_family && !nh->is_reject &&
4318 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
4319 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4327 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4328 const struct nh_notifier_grp_info *nh_grp,
4329 struct netlink_ext_ack *extack)
4333 if (nh_grp->is_fdb) {
4334 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4338 for (i = 0; i < nh_grp->num_nh; i++) {
4339 const struct nh_notifier_single_info *nh;
4342 nh = &nh_grp->nh_entries[i].nh;
4343 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4352 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4353 unsigned long event,
4354 struct nh_notifier_info *info)
4356 if (event != NEXTHOP_EVENT_REPLACE)
4359 switch (info->type) {
4360 case NH_NOTIFIER_INFO_TYPE_SINGLE:
4361 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
4363 case NH_NOTIFIER_INFO_TYPE_GRP:
4364 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4368 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4373 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4374 const struct nh_notifier_info *info)
4376 const struct net_device *dev;
4378 switch (info->type) {
4379 case NH_NOTIFIER_INFO_TYPE_SINGLE:
4380 dev = info->nh->dev;
4381 return info->nh->gw_family || info->nh->is_reject ||
4382 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
4383 case NH_NOTIFIER_INFO_TYPE_GRP:
4384 /* Already validated earlier. */
4391 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
4392 struct mlxsw_sp_nexthop *nh)
4394 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index;
4396 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
4397 nh->should_offload = 1;
4398 /* While nexthops that discard packets do not forward packets
4399 * via an egress RIF, they still need to be programmed using a
4400 * valid RIF, so use the loopback RIF created during init.
4402 nh->rif = mlxsw_sp->router->rifs[lb_rif_index];
4405 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
4406 struct mlxsw_sp_nexthop *nh)
4409 nh->should_offload = 0;
4413 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
4414 struct mlxsw_sp_nexthop_group *nh_grp,
4415 struct mlxsw_sp_nexthop *nh,
4416 struct nh_notifier_single_info *nh_obj, int weight)
4418 struct net_device *dev = nh_obj->dev;
4421 nh->nhgi = nh_grp->nhgi;
4422 nh->nh_weight = weight;
4424 switch (nh_obj->gw_family) {
4426 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
4427 nh->neigh_tbl = &arp_tbl;
4430 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
4431 #if IS_ENABLED(CONFIG_IPV6)
4432 nh->neigh_tbl = &nd_tbl;
4437 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4438 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4439 nh->ifindex = dev->ifindex;
4441 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4445 if (nh_obj->is_reject)
4446 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
4451 list_del(&nh->router_list_node);
4452 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4456 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
4457 struct mlxsw_sp_nexthop *nh)
4459 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
4460 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
4461 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4462 list_del(&nh->router_list_node);
4463 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4467 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
4468 struct mlxsw_sp_nexthop_group *nh_grp,
4469 struct nh_notifier_info *info)
4471 struct mlxsw_sp_nexthop_group_info *nhgi;
4472 struct mlxsw_sp_nexthop *nh;
4476 switch (info->type) {
4477 case NH_NOTIFIER_INFO_TYPE_SINGLE:
4480 case NH_NOTIFIER_INFO_TYPE_GRP:
4481 nhs = info->nh_grp->num_nh;
4487 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4490 nh_grp->nhgi = nhgi;
4491 nhgi->nh_grp = nh_grp;
4492 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
4494 for (i = 0; i < nhgi->count; i++) {
4495 struct nh_notifier_single_info *nh_obj;
4498 nh = &nhgi->nexthops[i];
4499 switch (info->type) {
4500 case NH_NOTIFIER_INFO_TYPE_SINGLE:
4504 case NH_NOTIFIER_INFO_TYPE_GRP:
4505 nh_obj = &info->nh_grp->nh_entries[i].nh;
4506 weight = info->nh_grp->nh_entries[i].weight;
4510 goto err_nexthop_obj_init;
4512 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
4515 goto err_nexthop_obj_init;
4517 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4519 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
4520 goto err_group_refresh;
4527 err_nexthop_obj_init:
4528 for (i--; i >= 0; i--) {
4529 nh = &nhgi->nexthops[i];
4530 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
4537 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4538 struct mlxsw_sp_nexthop_group *nh_grp)
4540 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4543 for (i = nhgi->count - 1; i >= 0; i--) {
4544 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4546 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
4548 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4549 WARN_ON_ONCE(nhgi->adj_index_valid);
4553 static struct mlxsw_sp_nexthop_group *
4554 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
4555 struct nh_notifier_info *info)
4557 struct mlxsw_sp_nexthop_group *nh_grp;
4560 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4562 return ERR_PTR(-ENOMEM);
4563 INIT_LIST_HEAD(&nh_grp->vr_list);
4564 err = rhashtable_init(&nh_grp->vr_ht,
4565 &mlxsw_sp_nexthop_group_vr_ht_params);
4567 goto err_nexthop_group_vr_ht_init;
4568 INIT_LIST_HEAD(&nh_grp->fib_list);
4569 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
4570 nh_grp->obj.id = info->id;
4572 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
4574 goto err_nexthop_group_info_init;
4576 nh_grp->can_destroy = false;
4580 err_nexthop_group_info_init:
4581 rhashtable_destroy(&nh_grp->vr_ht);
4582 err_nexthop_group_vr_ht_init:
4584 return ERR_PTR(err);
4588 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
4589 struct mlxsw_sp_nexthop_group *nh_grp)
4591 if (!nh_grp->can_destroy)
4593 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
4594 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
4595 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
4596 rhashtable_destroy(&nh_grp->vr_ht);
4600 static struct mlxsw_sp_nexthop_group *
4601 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
4603 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
4605 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
4607 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
4609 mlxsw_sp_nexthop_group_ht_params);
4612 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
4613 struct mlxsw_sp_nexthop_group *nh_grp)
4615 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4619 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
4620 struct mlxsw_sp_nexthop_group *nh_grp,
4621 struct mlxsw_sp_nexthop_group *old_nh_grp,
4622 struct netlink_ext_ack *extack)
4624 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
4625 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
4628 old_nh_grp->nhgi = new_nhgi;
4629 new_nhgi->nh_grp = old_nh_grp;
4630 nh_grp->nhgi = old_nhgi;
4631 old_nhgi->nh_grp = nh_grp;
4633 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
4634 /* Both the old adjacency index and the new one are valid.
4635 * Routes are currently using the old one. Tell the device to
4636 * replace the old adjacency index with the new one.
4638 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp,
4639 old_nhgi->adj_index,
4640 old_nhgi->ecmp_size);
4642 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
4645 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
4646 /* The old adjacency index is valid, while the new one is not.
4647 * Iterate over all the routes using the group and change them
4648 * to trap packets to the CPU.
4650 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
4652 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
4655 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
4656 /* The old adjacency index is invalid, while the new one is.
4657 * Iterate over all the routes using the group and change them
4658 * to forward packets using the new valid index.
4660 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
4662 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
4667 /* Make sure the flags are set / cleared based on the new nexthop group
4670 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp);
4672 /* At this point 'nh_grp' is just a shell that is not used by anyone
4673 * and its nexthop group info is the old info that was just replaced
4674 * with the new one. Remove it.
4676 nh_grp->can_destroy = true;
4677 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
4682 old_nhgi->nh_grp = old_nh_grp;
4683 nh_grp->nhgi = new_nhgi;
4684 new_nhgi->nh_grp = nh_grp;
4685 old_nh_grp->nhgi = old_nhgi;
4689 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
4690 struct nh_notifier_info *info)
4692 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
4693 struct netlink_ext_ack *extack = info->extack;
4696 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
4698 return PTR_ERR(nh_grp);
4700 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
4702 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
4704 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
4705 old_nh_grp, extack);
4708 nh_grp->can_destroy = true;
4709 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
4715 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
4716 struct nh_notifier_info *info)
4718 struct mlxsw_sp_nexthop_group *nh_grp;
4720 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
4724 nh_grp->can_destroy = true;
4725 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4727 /* If the group still has routes using it, then defer the delete
4728 * operation until the last route using it is deleted.
4730 if (!list_empty(&nh_grp->fib_list))
4732 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
4735 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
4736 unsigned long event, void *ptr)
4738 struct nh_notifier_info *info = ptr;
4739 struct mlxsw_sp_router *router;
4742 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
4743 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info);
4747 mutex_lock(&router->lock);
4750 case NEXTHOP_EVENT_REPLACE:
4751 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info);
4753 case NEXTHOP_EVENT_DEL:
4754 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info);
4760 mutex_unlock(&router->lock);
4763 return notifier_from_errno(err);
4766 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4767 struct fib_info *fi)
4769 const struct fib_nh *nh = fib_info_nh(fi, 0);
4771 return nh->fib_nh_scope == RT_SCOPE_LINK ||
4772 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
4776 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
4777 struct mlxsw_sp_nexthop_group *nh_grp)
4779 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
4780 struct mlxsw_sp_nexthop_group_info *nhgi;
4781 struct mlxsw_sp_nexthop *nh;
4784 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4787 nh_grp->nhgi = nhgi;
4788 nhgi->nh_grp = nh_grp;
4789 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
4791 for (i = 0; i < nhgi->count; i++) {
4792 struct fib_nh *fib_nh;
4794 nh = &nhgi->nexthops[i];
4795 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
4796 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
4798 goto err_nexthop4_init;
4800 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4802 goto err_group_refresh;
4809 for (i--; i >= 0; i--) {
4810 nh = &nhgi->nexthops[i];
4811 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4818 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4819 struct mlxsw_sp_nexthop_group *nh_grp)
4821 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4824 for (i = nhgi->count - 1; i >= 0; i--) {
4825 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4827 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
4829 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4830 WARN_ON_ONCE(nhgi->adj_index_valid);
4834 static struct mlxsw_sp_nexthop_group *
4835 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
4837 struct mlxsw_sp_nexthop_group *nh_grp;
4840 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4842 return ERR_PTR(-ENOMEM);
4843 INIT_LIST_HEAD(&nh_grp->vr_list);
4844 err = rhashtable_init(&nh_grp->vr_ht,
4845 &mlxsw_sp_nexthop_group_vr_ht_params);
4847 goto err_nexthop_group_vr_ht_init;
4848 INIT_LIST_HEAD(&nh_grp->fib_list);
4849 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
4850 nh_grp->ipv4.fi = fi;
4853 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
4855 goto err_nexthop_group_info_init;
4857 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4859 goto err_nexthop_group_insert;
4861 nh_grp->can_destroy = true;
4865 err_nexthop_group_insert:
4866 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4867 err_nexthop_group_info_init:
4869 rhashtable_destroy(&nh_grp->vr_ht);
4870 err_nexthop_group_vr_ht_init:
4872 return ERR_PTR(err);
4876 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
4877 struct mlxsw_sp_nexthop_group *nh_grp)
4879 if (!nh_grp->can_destroy)
4881 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4882 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
4883 fib_info_put(nh_grp->ipv4.fi);
4884 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
4885 rhashtable_destroy(&nh_grp->vr_ht);
4889 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
4890 struct mlxsw_sp_fib_entry *fib_entry,
4891 struct fib_info *fi)
4893 struct mlxsw_sp_nexthop_group *nh_grp;
4896 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
4898 if (WARN_ON_ONCE(!nh_grp))
4903 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
4905 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
4907 return PTR_ERR(nh_grp);
4910 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
4911 fib_entry->nh_group = nh_grp;
4915 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
4916 struct mlxsw_sp_fib_entry *fib_entry)
4918 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4920 list_del(&fib_entry->nexthop_group_node);
4921 if (!list_empty(&nh_grp->fib_list))
4924 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
4925 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
4929 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
4933 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4935 struct mlxsw_sp_fib4_entry *fib4_entry;
4937 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
4939 return !fib4_entry->tos;
4943 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
4945 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
4947 switch (fib_entry->fib_node->fib->proto) {
4948 case MLXSW_SP_L3_PROTO_IPV4:
4949 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
4952 case MLXSW_SP_L3_PROTO_IPV6:
4956 switch (fib_entry->type) {
4957 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4958 return !!nh_group->nhgi->adj_index_valid;
4959 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4960 return !!nh_group->nhgi->nh_rif;
4961 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4962 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4963 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4970 static struct mlxsw_sp_nexthop *
4971 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
4972 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4976 for (i = 0; i < nh_grp->nhgi->count; i++) {
4977 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4978 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4980 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
4981 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
4982 &rt->fib6_nh->fib_nh_gw6))
4991 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
4992 struct fib_entry_notifier_info *fen_info)
4994 u32 *p_dst = (u32 *) &fen_info->dst;
4995 struct fib_rt_info fri;
4997 fri.fi = fen_info->fi;
4998 fri.tb_id = fen_info->tb_id;
4999 fri.dst = cpu_to_be32(*p_dst);
5000 fri.dst_len = fen_info->dst_len;
5001 fri.tos = fen_info->tos;
5002 fri.type = fen_info->type;
5003 fri.offload = false;
5005 fri.offload_failed = true;
5006 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5010 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5011 struct mlxsw_sp_fib_entry *fib_entry)
5013 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5014 int dst_len = fib_entry->fib_node->key.prefix_len;
5015 struct mlxsw_sp_fib4_entry *fib4_entry;
5016 struct fib_rt_info fri;
5017 bool should_offload;
5019 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5020 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5022 fri.fi = fib4_entry->fi;
5023 fri.tb_id = fib4_entry->tb_id;
5024 fri.dst = cpu_to_be32(*p_dst);
5025 fri.dst_len = dst_len;
5026 fri.tos = fib4_entry->tos;
5027 fri.type = fib4_entry->type;
5028 fri.offload = should_offload;
5029 fri.trap = !should_offload;
5030 fri.offload_failed = false;
5031 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5035 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5036 struct mlxsw_sp_fib_entry *fib_entry)
5038 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5039 int dst_len = fib_entry->fib_node->key.prefix_len;
5040 struct mlxsw_sp_fib4_entry *fib4_entry;
5041 struct fib_rt_info fri;
5043 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5045 fri.fi = fib4_entry->fi;
5046 fri.tb_id = fib4_entry->tb_id;
5047 fri.dst = cpu_to_be32(*p_dst);
5048 fri.dst_len = dst_len;
5049 fri.tos = fib4_entry->tos;
5050 fri.type = fib4_entry->type;
5051 fri.offload = false;
5053 fri.offload_failed = false;
5054 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5057 #if IS_ENABLED(CONFIG_IPV6)
5059 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5060 struct fib6_info **rt_arr,
5065 /* In IPv6 a multipath route is represented using multiple routes, so
5066 * we need to set the flags on all of them.
5068 for (i = 0; i < nrt6; i++)
5069 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i],
5070 false, false, true);
5074 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5075 struct fib6_info **rt_arr,
5081 #if IS_ENABLED(CONFIG_IPV6)
5083 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5084 struct mlxsw_sp_fib_entry *fib_entry)
5086 struct mlxsw_sp_fib6_entry *fib6_entry;
5087 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5088 bool should_offload;
5090 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5092 /* In IPv6 a multipath route is represented using multiple routes, so
5093 * we need to set the flags on all of them.
5095 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5097 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5098 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5099 should_offload, !should_offload, false);
5103 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5104 struct mlxsw_sp_fib_entry *fib_entry)
5109 #if IS_ENABLED(CONFIG_IPV6)
5111 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5112 struct mlxsw_sp_fib_entry *fib_entry)
5114 struct mlxsw_sp_fib6_entry *fib6_entry;
5115 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5117 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5119 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5120 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5121 false, false, false);
5125 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5126 struct mlxsw_sp_fib_entry *fib_entry)
5132 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5133 struct mlxsw_sp_fib_entry *fib_entry)
5135 switch (fib_entry->fib_node->fib->proto) {
5136 case MLXSW_SP_L3_PROTO_IPV4:
5137 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
5139 case MLXSW_SP_L3_PROTO_IPV6:
5140 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
5146 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5147 struct mlxsw_sp_fib_entry *fib_entry)
5149 switch (fib_entry->fib_node->fib->proto) {
5150 case MLXSW_SP_L3_PROTO_IPV4:
5151 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5153 case MLXSW_SP_L3_PROTO_IPV6:
5154 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5160 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
5161 struct mlxsw_sp_fib_entry *fib_entry,
5162 enum mlxsw_sp_fib_entry_op op)
5165 case MLXSW_SP_FIB_ENTRY_OP_WRITE:
5166 case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
5167 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
5169 case MLXSW_SP_FIB_ENTRY_OP_DELETE:
5170 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5177 struct mlxsw_sp_fib_entry_op_ctx_basic {
5178 char ralue_pl[MLXSW_REG_RALUE_LEN];
5182 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5183 enum mlxsw_sp_l3proto proto,
5184 enum mlxsw_sp_fib_entry_op op,
5185 u16 virtual_router, u8 prefix_len,
5186 unsigned char *addr,
5187 struct mlxsw_sp_fib_entry_priv *priv)
5189 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5190 enum mlxsw_reg_ralxx_protocol ralxx_proto;
5191 char *ralue_pl = op_ctx_basic->ralue_pl;
5192 enum mlxsw_reg_ralue_op ralue_op;
5194 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
5197 case MLXSW_SP_FIB_ENTRY_OP_WRITE:
5198 case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
5199 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
5201 case MLXSW_SP_FIB_ENTRY_OP_DELETE:
5202 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
5210 case MLXSW_SP_L3_PROTO_IPV4:
5211 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
5212 virtual_router, prefix_len, (u32 *) addr);
5214 case MLXSW_SP_L3_PROTO_IPV6:
5215 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
5216 virtual_router, prefix_len, addr);
5222 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5223 enum mlxsw_reg_ralue_trap_action trap_action,
5224 u16 trap_id, u32 adjacency_index, u16 ecmp_size)
5226 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5228 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action,
5229 trap_id, adjacency_index, ecmp_size);
5233 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5234 enum mlxsw_reg_ralue_trap_action trap_action,
5235 u16 trap_id, u16 local_erif)
5237 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5239 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action,
5240 trap_id, local_erif);
5244 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
5246 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5248 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl);
5252 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5255 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5257 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr);
5261 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
5262 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5263 bool *postponed_for_bulk)
5265 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
5267 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
5268 op_ctx_basic->ralue_pl);
5272 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv)
5277 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5278 struct mlxsw_sp_fib_entry *fib_entry,
5279 enum mlxsw_sp_fib_entry_op op)
5281 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
5283 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv);
5284 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id,
5285 fib_entry->fib_node->key.prefix_len,
5286 fib_entry->fib_node->key.addr,
5290 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
5291 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5292 const struct mlxsw_sp_router_ll_ops *ll_ops)
5294 bool postponed_for_bulk = false;
5297 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk);
5298 if (!postponed_for_bulk)
5299 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
5303 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp)
5305 enum mlxsw_reg_ratr_trap_action trap_action;
5306 char ratr_pl[MLXSW_REG_RATR_LEN];
5309 if (mlxsw_sp->router->adj_discard_index_valid)
5312 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
5313 &mlxsw_sp->router->adj_discard_index);
5317 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
5318 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
5319 MLXSW_REG_RATR_TYPE_ETHERNET,
5320 mlxsw_sp->router->adj_discard_index,
5321 mlxsw_sp->router->lb_rif_index);
5322 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
5323 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
5324 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
5326 goto err_ratr_write;
5328 mlxsw_sp->router->adj_discard_index_valid = true;
5333 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
5334 mlxsw_sp->router->adj_discard_index);
5338 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
5339 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5340 struct mlxsw_sp_fib_entry *fib_entry,
5341 enum mlxsw_sp_fib_entry_op op)
5343 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5344 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5345 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
5346 enum mlxsw_reg_ralue_trap_action trap_action;
5348 u32 adjacency_index = 0;
5352 /* In case the nexthop group adjacency index is valid, use it
5353 * with provided ECMP size. Otherwise, setup trap and pass
5354 * traffic to kernel.
5356 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5357 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5358 adjacency_index = nhgi->adj_index;
5359 ecmp_size = nhgi->ecmp_size;
5360 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
5361 err = mlxsw_sp_adj_discard_write(mlxsw_sp);
5364 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5365 adjacency_index = mlxsw_sp->router->adj_discard_index;
5368 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5369 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5372 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5373 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id,
5374 adjacency_index, ecmp_size);
5375 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5378 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
5379 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5380 struct mlxsw_sp_fib_entry *fib_entry,
5381 enum mlxsw_sp_fib_entry_op op)
5383 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5384 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
5385 enum mlxsw_reg_ralue_trap_action trap_action;
5389 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5390 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5391 rif_index = rif->rif_index;
5393 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5394 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5397 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5398 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index);
5399 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5402 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
5403 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5404 struct mlxsw_sp_fib_entry *fib_entry,
5405 enum mlxsw_sp_fib_entry_op op)
5407 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5409 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5410 ll_ops->fib_entry_act_ip2me_pack(op_ctx);
5411 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5414 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
5415 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5416 struct mlxsw_sp_fib_entry *fib_entry,
5417 enum mlxsw_sp_fib_entry_op op)
5419 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5420 enum mlxsw_reg_ralue_trap_action trap_action;
5422 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
5423 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5424 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0);
5425 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5429 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
5430 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5431 struct mlxsw_sp_fib_entry *fib_entry,
5432 enum mlxsw_sp_fib_entry_op op)
5434 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5435 enum mlxsw_reg_ralue_trap_action trap_action;
5438 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5439 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
5441 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5442 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0);
5443 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5447 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
5448 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5449 struct mlxsw_sp_fib_entry *fib_entry,
5450 enum mlxsw_sp_fib_entry_op op)
5452 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5453 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
5454 const struct mlxsw_sp_ipip_ops *ipip_ops;
5457 if (WARN_ON(!ipip_entry))
5460 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
5461 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
5462 fib_entry->decap.tunnel_index);
5466 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5467 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
5468 fib_entry->decap.tunnel_index);
5469 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5472 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
5473 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5474 struct mlxsw_sp_fib_entry *fib_entry,
5475 enum mlxsw_sp_fib_entry_op op)
5477 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5479 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
5480 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
5481 fib_entry->decap.tunnel_index);
5482 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
5485 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
5486 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5487 struct mlxsw_sp_fib_entry *fib_entry,
5488 enum mlxsw_sp_fib_entry_op op)
5490 switch (fib_entry->type) {
5491 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5492 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
5493 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5494 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
5495 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
5496 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
5497 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5498 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
5499 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
5500 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
5501 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5502 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
5503 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5504 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
5509 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
5510 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5511 struct mlxsw_sp_fib_entry *fib_entry,
5512 enum mlxsw_sp_fib_entry_op op)
5514 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
5519 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
5524 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
5525 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5526 struct mlxsw_sp_fib_entry *fib_entry,
5529 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
5530 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE :
5531 MLXSW_SP_FIB_ENTRY_OP_UPDATE);
5534 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
5535 struct mlxsw_sp_fib_entry *fib_entry)
5537 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5539 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5540 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false);
5543 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
5544 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5545 struct mlxsw_sp_fib_entry *fib_entry)
5547 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
5549 if (!ll_ops->fib_entry_is_committed(fib_entry->priv))
5551 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
5552 MLXSW_SP_FIB_ENTRY_OP_DELETE);
5556 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5557 const struct fib_entry_notifier_info *fen_info,
5558 struct mlxsw_sp_fib_entry *fib_entry)
5560 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
5561 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
5562 struct mlxsw_sp_router *router = mlxsw_sp->router;
5563 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
5564 int ifindex = nhgi->nexthops[0].ifindex;
5565 struct mlxsw_sp_ipip_entry *ipip_entry;
5567 switch (fen_info->type) {
5569 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
5570 MLXSW_SP_L3_PROTO_IPV4, dip);
5571 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
5572 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
5573 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
5577 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
5578 MLXSW_SP_L3_PROTO_IPV4,
5582 tunnel_index = router->nve_decap_config.tunnel_index;
5583 fib_entry->decap.tunnel_index = tunnel_index;
5584 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
5589 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5592 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5594 case RTN_UNREACHABLE:
5596 /* Packets hitting these routes need to be trapped, but
5597 * can do so with a lower priority than packets directed
5598 * at the host, so use action type local instead of trap.
5600 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5604 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5606 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5614 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
5615 struct mlxsw_sp_fib_entry *fib_entry)
5617 switch (fib_entry->type) {
5618 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5619 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
5626 static struct mlxsw_sp_fib4_entry *
5627 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
5628 struct mlxsw_sp_fib_node *fib_node,
5629 const struct fib_entry_notifier_info *fen_info)
5631 struct mlxsw_sp_fib4_entry *fib4_entry;
5632 struct mlxsw_sp_fib_entry *fib_entry;
5635 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
5637 return ERR_PTR(-ENOMEM);
5638 fib_entry = &fib4_entry->common;
5640 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
5641 if (IS_ERR(fib_entry->priv)) {
5642 err = PTR_ERR(fib_entry->priv);
5643 goto err_fib_entry_priv_create;
5646 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
5648 goto err_nexthop4_group_get;
5650 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
5653 goto err_nexthop_group_vr_link;
5655 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
5657 goto err_fib4_entry_type_set;
5659 fib4_entry->fi = fen_info->fi;
5660 fib_info_hold(fib4_entry->fi);
5661 fib4_entry->tb_id = fen_info->tb_id;
5662 fib4_entry->type = fen_info->type;
5663 fib4_entry->tos = fen_info->tos;
5665 fib_entry->fib_node = fib_node;
5669 err_fib4_entry_type_set:
5670 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
5671 err_nexthop_group_vr_link:
5672 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
5673 err_nexthop4_group_get:
5674 mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
5675 err_fib_entry_priv_create:
5677 return ERR_PTR(err);
5680 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5681 struct mlxsw_sp_fib4_entry *fib4_entry)
5683 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
5685 fib_info_put(fib4_entry->fi);
5686 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common);
5687 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group,
5689 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
5690 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv);
5694 static struct mlxsw_sp_fib4_entry *
5695 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5696 const struct fib_entry_notifier_info *fen_info)
5698 struct mlxsw_sp_fib4_entry *fib4_entry;
5699 struct mlxsw_sp_fib_node *fib_node;
5700 struct mlxsw_sp_fib *fib;
5701 struct mlxsw_sp_vr *vr;
5703 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
5706 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
5708 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
5709 sizeof(fen_info->dst),
5714 fib4_entry = container_of(fib_node->fib_entry,
5715 struct mlxsw_sp_fib4_entry, common);
5716 if (fib4_entry->tb_id == fen_info->tb_id &&
5717 fib4_entry->tos == fen_info->tos &&
5718 fib4_entry->type == fen_info->type &&
5719 fib4_entry->fi == fen_info->fi)
5725 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
5726 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
5727 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
5728 .key_len = sizeof(struct mlxsw_sp_fib_key),
5729 .automatic_shrinking = true,
5732 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
5733 struct mlxsw_sp_fib_node *fib_node)
5735 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
5736 mlxsw_sp_fib_ht_params);
5739 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
5740 struct mlxsw_sp_fib_node *fib_node)
5742 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
5743 mlxsw_sp_fib_ht_params);
5746 static struct mlxsw_sp_fib_node *
5747 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
5748 size_t addr_len, unsigned char prefix_len)
5750 struct mlxsw_sp_fib_key key;
5752 memset(&key, 0, sizeof(key));
5753 memcpy(key.addr, addr, addr_len);
5754 key.prefix_len = prefix_len;
5755 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
5758 static struct mlxsw_sp_fib_node *
5759 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
5760 size_t addr_len, unsigned char prefix_len)
5762 struct mlxsw_sp_fib_node *fib_node;
5764 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
5768 list_add(&fib_node->list, &fib->node_list);
5769 memcpy(fib_node->key.addr, addr, addr_len);
5770 fib_node->key.prefix_len = prefix_len;
5775 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
5777 list_del(&fib_node->list);
5781 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
5782 struct mlxsw_sp_fib_node *fib_node)
5784 struct mlxsw_sp_prefix_usage req_prefix_usage;
5785 struct mlxsw_sp_fib *fib = fib_node->fib;
5786 struct mlxsw_sp_lpm_tree *lpm_tree;
5789 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
5790 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5793 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5794 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
5795 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5797 if (IS_ERR(lpm_tree))
5798 return PTR_ERR(lpm_tree);
5800 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5802 goto err_lpm_tree_replace;
5805 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
5808 err_lpm_tree_replace:
5809 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5813 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
5814 struct mlxsw_sp_fib_node *fib_node)
5816 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
5817 struct mlxsw_sp_prefix_usage req_prefix_usage;
5818 struct mlxsw_sp_fib *fib = fib_node->fib;
5821 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
5823 /* Try to construct a new LPM tree from the current prefix usage
5824 * minus the unused one. If we fail, continue using the old one.
5826 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
5827 mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
5828 fib_node->key.prefix_len);
5829 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
5831 if (IS_ERR(lpm_tree))
5834 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
5836 goto err_lpm_tree_replace;
5840 err_lpm_tree_replace:
5841 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
5844 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
5845 struct mlxsw_sp_fib_node *fib_node,
5846 struct mlxsw_sp_fib *fib)
5850 err = mlxsw_sp_fib_node_insert(fib, fib_node);
5853 fib_node->fib = fib;
5855 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
5857 goto err_fib_lpm_tree_link;
5861 err_fib_lpm_tree_link:
5862 fib_node->fib = NULL;
5863 mlxsw_sp_fib_node_remove(fib, fib_node);
5867 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
5868 struct mlxsw_sp_fib_node *fib_node)
5870 struct mlxsw_sp_fib *fib = fib_node->fib;
5872 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
5873 fib_node->fib = NULL;
5874 mlxsw_sp_fib_node_remove(fib, fib_node);
5877 static struct mlxsw_sp_fib_node *
5878 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
5879 size_t addr_len, unsigned char prefix_len,
5880 enum mlxsw_sp_l3proto proto)
5882 struct mlxsw_sp_fib_node *fib_node;
5883 struct mlxsw_sp_fib *fib;
5884 struct mlxsw_sp_vr *vr;
5887 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
5889 return ERR_CAST(vr);
5890 fib = mlxsw_sp_vr_fib(vr, proto);
5892 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
5896 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
5899 goto err_fib_node_create;
5902 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
5904 goto err_fib_node_init;
5909 mlxsw_sp_fib_node_destroy(fib_node);
5910 err_fib_node_create:
5911 mlxsw_sp_vr_put(mlxsw_sp, vr);
5912 return ERR_PTR(err);
5915 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
5916 struct mlxsw_sp_fib_node *fib_node)
5918 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5920 if (fib_node->fib_entry)
5922 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
5923 mlxsw_sp_fib_node_destroy(fib_node);
5924 mlxsw_sp_vr_put(mlxsw_sp, vr);
5927 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5928 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5929 struct mlxsw_sp_fib_entry *fib_entry)
5931 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5932 bool is_new = !fib_node->fib_entry;
5935 fib_node->fib_entry = fib_entry;
5937 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new);
5939 goto err_fib_entry_update;
5943 err_fib_entry_update:
5944 fib_node->fib_entry = NULL;
5948 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5949 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5950 struct mlxsw_sp_fib_entry *fib_entry)
5952 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
5955 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
5956 fib_node->fib_entry = NULL;
5960 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5961 struct mlxsw_sp_fib_entry *fib_entry)
5963 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
5965 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
5966 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry);
5969 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
5971 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
5972 struct mlxsw_sp_fib4_entry *fib4_replaced;
5974 if (!fib_node->fib_entry)
5977 fib4_replaced = container_of(fib_node->fib_entry,
5978 struct mlxsw_sp_fib4_entry, common);
5979 if (fib4_entry->tb_id == RT_TABLE_MAIN &&
5980 fib4_replaced->tb_id == RT_TABLE_LOCAL)
5987 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
5988 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
5989 const struct fib_entry_notifier_info *fen_info)
5991 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
5992 struct mlxsw_sp_fib_entry *replaced;
5993 struct mlxsw_sp_fib_node *fib_node;
5996 if (mlxsw_sp->router->aborted)
5999 if (fen_info->fi->nh &&
6000 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id))
6003 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
6004 &fen_info->dst, sizeof(fen_info->dst),
6006 MLXSW_SP_L3_PROTO_IPV4);
6007 if (IS_ERR(fib_node)) {
6008 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
6009 return PTR_ERR(fib_node);
6012 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
6013 if (IS_ERR(fib4_entry)) {
6014 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
6015 err = PTR_ERR(fib4_entry);
6016 goto err_fib4_entry_create;
6019 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
6020 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6021 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6025 replaced = fib_node->fib_entry;
6026 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
6028 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
6029 goto err_fib_node_entry_link;
6032 /* Nothing to replace */
6036 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
6037 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
6039 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
6043 err_fib_node_entry_link:
6044 fib_node->fib_entry = replaced;
6045 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6046 err_fib4_entry_create:
6047 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6051 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
6052 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6053 struct fib_entry_notifier_info *fen_info)
6055 struct mlxsw_sp_fib4_entry *fib4_entry;
6056 struct mlxsw_sp_fib_node *fib_node;
6059 if (mlxsw_sp->router->aborted)
6062 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
6065 fib_node = fib4_entry->common.fib_node;
6067 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
6068 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6069 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6073 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
6075 /* Multicast routes aren't supported, so ignore them. Neighbour
6076 * Discovery packets are specifically trapped.
6078 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
6081 /* Cloned routes are irrelevant in the forwarding path. */
6082 if (rt->fib6_flags & RTF_CACHE)
6088 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
6090 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6092 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
6094 return ERR_PTR(-ENOMEM);
6096 /* In case of route replace, replaced route is deleted with
6097 * no notification. Take reference to prevent accessing freed
6100 mlxsw_sp_rt6->rt = rt;
6103 return mlxsw_sp_rt6;
6106 #if IS_ENABLED(CONFIG_IPV6)
6107 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6109 fib6_info_release(rt);
6112 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6117 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
6119 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
6121 if (!mlxsw_sp_rt6->rt->nh)
6122 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
6123 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
6124 kfree(mlxsw_sp_rt6);
6127 static struct fib6_info *
6128 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
6130 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
6134 static struct mlxsw_sp_rt6 *
6135 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
6136 const struct fib6_info *rt)
6138 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6140 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
6141 if (mlxsw_sp_rt6->rt == rt)
6142 return mlxsw_sp_rt6;
6148 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
6149 const struct fib6_info *rt,
6150 enum mlxsw_sp_ipip_type *ret)
6152 return rt->fib6_nh->fib_nh_dev &&
6153 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
6156 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
6157 struct mlxsw_sp_nexthop_group *nh_grp,
6158 struct mlxsw_sp_nexthop *nh,
6159 const struct fib6_info *rt)
6161 struct net_device *dev = rt->fib6_nh->fib_nh_dev;
6163 nh->nhgi = nh_grp->nhgi;
6164 nh->nh_weight = rt->fib6_nh->fib_nh_weight;
6165 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
6166 #if IS_ENABLED(CONFIG_IPV6)
6167 nh->neigh_tbl = &nd_tbl;
6169 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
6171 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
6175 nh->ifindex = dev->ifindex;
6177 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
6180 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
6181 struct mlxsw_sp_nexthop *nh)
6183 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
6184 list_del(&nh->router_list_node);
6185 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6188 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
6189 const struct fib6_info *rt)
6191 return rt->fib6_nh->fib_nh_gw_family ||
6192 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
6196 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
6197 struct mlxsw_sp_nexthop_group *nh_grp,
6198 struct mlxsw_sp_fib6_entry *fib6_entry)
6200 struct mlxsw_sp_nexthop_group_info *nhgi;
6201 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6202 struct mlxsw_sp_nexthop *nh;
6205 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
6209 nh_grp->nhgi = nhgi;
6210 nhgi->nh_grp = nh_grp;
6211 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
6212 struct mlxsw_sp_rt6, list);
6213 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
6214 nhgi->count = fib6_entry->nrt6;
6215 for (i = 0; i < nhgi->count; i++) {
6216 struct fib6_info *rt = mlxsw_sp_rt6->rt;
6218 nh = &nhgi->nexthops[i];
6219 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
6221 goto err_nexthop6_init;
6222 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
6224 nh_grp->nhgi = nhgi;
6225 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6227 goto err_group_refresh;
6234 for (i--; i >= 0; i--) {
6235 nh = &nhgi->nexthops[i];
6236 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6243 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
6244 struct mlxsw_sp_nexthop_group *nh_grp)
6246 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
6249 for (i = nhgi->count - 1; i >= 0; i--) {
6250 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
6252 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6254 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6255 WARN_ON_ONCE(nhgi->adj_index_valid);
6259 static struct mlxsw_sp_nexthop_group *
6260 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
6261 struct mlxsw_sp_fib6_entry *fib6_entry)
6263 struct mlxsw_sp_nexthop_group *nh_grp;
6266 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
6268 return ERR_PTR(-ENOMEM);
6269 INIT_LIST_HEAD(&nh_grp->vr_list);
6270 err = rhashtable_init(&nh_grp->vr_ht,
6271 &mlxsw_sp_nexthop_group_vr_ht_params);
6273 goto err_nexthop_group_vr_ht_init;
6274 INIT_LIST_HEAD(&nh_grp->fib_list);
6275 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
6277 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
6279 goto err_nexthop_group_info_init;
6281 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
6283 goto err_nexthop_group_insert;
6285 nh_grp->can_destroy = true;
6289 err_nexthop_group_insert:
6290 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6291 err_nexthop_group_info_init:
6292 rhashtable_destroy(&nh_grp->vr_ht);
6293 err_nexthop_group_vr_ht_init:
6295 return ERR_PTR(err);
6299 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
6300 struct mlxsw_sp_nexthop_group *nh_grp)
6302 if (!nh_grp->can_destroy)
6304 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
6305 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6306 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
6307 rhashtable_destroy(&nh_grp->vr_ht);
6311 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
6312 struct mlxsw_sp_fib6_entry *fib6_entry)
6314 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6315 struct mlxsw_sp_nexthop_group *nh_grp;
6318 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
6320 if (WARN_ON_ONCE(!nh_grp))
6325 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
6327 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
6329 return PTR_ERR(nh_grp);
6332 /* The route and the nexthop are described by the same struct, so we
6333 * need to the update the nexthop offload indication for the new route.
6335 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
6338 list_add_tail(&fib6_entry->common.nexthop_group_node,
6340 fib6_entry->common.nh_group = nh_grp;
6345 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
6346 struct mlxsw_sp_fib_entry *fib_entry)
6348 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
6350 list_del(&fib_entry->nexthop_group_node);
6351 if (!list_empty(&nh_grp->fib_list))
6354 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
6355 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
6359 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
6362 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
6363 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6364 struct mlxsw_sp_fib6_entry *fib6_entry)
6366 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
6367 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6370 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib);
6371 fib6_entry->common.nh_group = NULL;
6372 list_del(&fib6_entry->common.nexthop_group_node);
6374 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6376 goto err_nexthop6_group_get;
6378 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group,
6381 goto err_nexthop_group_vr_link;
6383 /* In case this entry is offloaded, then the adjacency index
6384 * currently associated with it in the device's table is that
6385 * of the old group. Start using the new one instead.
6387 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx,
6388 &fib6_entry->common, false);
6390 goto err_fib_entry_update;
6392 if (list_empty(&old_nh_grp->fib_list))
6393 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
6397 err_fib_entry_update:
6398 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6400 err_nexthop_group_vr_link:
6401 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6402 err_nexthop6_group_get:
6403 list_add_tail(&fib6_entry->common.nexthop_group_node,
6404 &old_nh_grp->fib_list);
6405 fib6_entry->common.nh_group = old_nh_grp;
6406 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib);
6411 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
6412 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6413 struct mlxsw_sp_fib6_entry *fib6_entry,
6414 struct fib6_info **rt_arr, unsigned int nrt6)
6416 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6419 for (i = 0; i < nrt6; i++) {
6420 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
6421 if (IS_ERR(mlxsw_sp_rt6)) {
6422 err = PTR_ERR(mlxsw_sp_rt6);
6423 goto err_rt6_create;
6426 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
6430 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
6432 goto err_nexthop6_group_update;
6436 err_nexthop6_group_update:
6439 for (i--; i >= 0; i--) {
6441 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
6442 struct mlxsw_sp_rt6, list);
6443 list_del(&mlxsw_sp_rt6->list);
6444 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6450 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
6451 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6452 struct mlxsw_sp_fib6_entry *fib6_entry,
6453 struct fib6_info **rt_arr, unsigned int nrt6)
6455 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6458 for (i = 0; i < nrt6; i++) {
6459 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
6461 if (WARN_ON_ONCE(!mlxsw_sp_rt6))
6465 list_del(&mlxsw_sp_rt6->list);
6466 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6469 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
6472 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
6473 struct mlxsw_sp_fib_entry *fib_entry,
6474 const struct fib6_info *rt)
6476 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
6477 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6478 else if (rt->fib6_type == RTN_BLACKHOLE)
6479 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
6480 else if (rt->fib6_flags & RTF_REJECT)
6481 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
6482 else if (fib_entry->nh_group->nhgi->gateway)
6483 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
6485 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
6489 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
6491 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
6493 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
6496 list_del(&mlxsw_sp_rt6->list);
6497 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6501 static struct mlxsw_sp_fib6_entry *
6502 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
6503 struct mlxsw_sp_fib_node *fib_node,
6504 struct fib6_info **rt_arr, unsigned int nrt6)
6506 struct mlxsw_sp_fib6_entry *fib6_entry;
6507 struct mlxsw_sp_fib_entry *fib_entry;
6508 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6511 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
6513 return ERR_PTR(-ENOMEM);
6514 fib_entry = &fib6_entry->common;
6516 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
6517 if (IS_ERR(fib_entry->priv)) {
6518 err = PTR_ERR(fib_entry->priv);
6519 goto err_fib_entry_priv_create;
6522 INIT_LIST_HEAD(&fib6_entry->rt6_list);
6524 for (i = 0; i < nrt6; i++) {
6525 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
6526 if (IS_ERR(mlxsw_sp_rt6)) {
6527 err = PTR_ERR(mlxsw_sp_rt6);
6528 goto err_rt6_create;
6530 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
6534 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6536 goto err_nexthop6_group_get;
6538 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
6541 goto err_nexthop_group_vr_link;
6543 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
6545 fib_entry->fib_node = fib_node;
6549 err_nexthop_group_vr_link:
6550 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
6551 err_nexthop6_group_get:
6554 for (i--; i >= 0; i--) {
6556 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
6557 struct mlxsw_sp_rt6, list);
6558 list_del(&mlxsw_sp_rt6->list);
6559 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6561 mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
6562 err_fib_entry_priv_create:
6564 return ERR_PTR(err);
6567 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6568 struct mlxsw_sp_fib6_entry *fib6_entry)
6570 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6572 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6574 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6575 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
6576 WARN_ON(fib6_entry->nrt6);
6577 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv);
6581 static struct mlxsw_sp_fib6_entry *
6582 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6583 const struct fib6_info *rt)
6585 struct mlxsw_sp_fib6_entry *fib6_entry;
6586 struct mlxsw_sp_fib_node *fib_node;
6587 struct mlxsw_sp_fib *fib;
6588 struct fib6_info *cmp_rt;
6589 struct mlxsw_sp_vr *vr;
6591 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
6594 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
6596 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
6597 sizeof(rt->fib6_dst.addr),
6602 fib6_entry = container_of(fib_node->fib_entry,
6603 struct mlxsw_sp_fib6_entry, common);
6604 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6605 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
6606 rt->fib6_metric == cmp_rt->fib6_metric &&
6607 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
6613 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
6615 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6616 struct mlxsw_sp_fib6_entry *fib6_replaced;
6617 struct fib6_info *rt, *rt_replaced;
6619 if (!fib_node->fib_entry)
6622 fib6_replaced = container_of(fib_node->fib_entry,
6623 struct mlxsw_sp_fib6_entry,
6625 rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6626 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
6627 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
6628 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
6634 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
6635 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6636 struct fib6_info **rt_arr, unsigned int nrt6)
6638 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
6639 struct mlxsw_sp_fib_entry *replaced;
6640 struct mlxsw_sp_fib_node *fib_node;
6641 struct fib6_info *rt = rt_arr[0];
6644 if (mlxsw_sp->router->aborted)
6647 if (rt->fib6_src.plen)
6650 if (mlxsw_sp_fib6_rt_should_ignore(rt))
6653 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id))
6656 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
6658 sizeof(rt->fib6_dst.addr),
6660 MLXSW_SP_L3_PROTO_IPV6);
6661 if (IS_ERR(fib_node))
6662 return PTR_ERR(fib_node);
6664 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
6666 if (IS_ERR(fib6_entry)) {
6667 err = PTR_ERR(fib6_entry);
6668 goto err_fib6_entry_create;
6671 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
6672 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6673 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6677 replaced = fib_node->fib_entry;
6678 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
6680 goto err_fib_node_entry_link;
6682 /* Nothing to replace */
6686 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
6687 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
6689 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
6693 err_fib_node_entry_link:
6694 fib_node->fib_entry = replaced;
6695 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6696 err_fib6_entry_create:
6697 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6701 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
6702 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6703 struct fib6_info **rt_arr, unsigned int nrt6)
6705 struct mlxsw_sp_fib6_entry *fib6_entry;
6706 struct mlxsw_sp_fib_node *fib_node;
6707 struct fib6_info *rt = rt_arr[0];
6710 if (mlxsw_sp->router->aborted)
6713 if (rt->fib6_src.plen)
6716 if (mlxsw_sp_fib6_rt_should_ignore(rt))
6719 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
6721 sizeof(rt->fib6_dst.addr),
6723 MLXSW_SP_L3_PROTO_IPV6);
6724 if (IS_ERR(fib_node))
6725 return PTR_ERR(fib_node);
6727 if (WARN_ON_ONCE(!fib_node->fib_entry)) {
6728 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6732 fib6_entry = container_of(fib_node->fib_entry,
6733 struct mlxsw_sp_fib6_entry, common);
6734 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
6736 goto err_fib6_entry_nexthop_add;
6740 err_fib6_entry_nexthop_add:
6741 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6745 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
6746 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
6747 struct fib6_info **rt_arr, unsigned int nrt6)
6749 struct mlxsw_sp_fib6_entry *fib6_entry;
6750 struct mlxsw_sp_fib_node *fib_node;
6751 struct fib6_info *rt = rt_arr[0];
6754 if (mlxsw_sp->router->aborted)
6757 if (mlxsw_sp_fib6_rt_should_ignore(rt))
6760 /* Multipath routes are first added to the FIB trie and only then
6761 * notified. If we vetoed the addition, we will get a delete
6762 * notification for a route we do not have. Therefore, do not warn if
6763 * route was not found.
6765 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
6769 /* If not all the nexthops are deleted, then only reduce the nexthop
6772 if (nrt6 != fib6_entry->nrt6) {
6773 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
6777 fib_node = fib6_entry->common.fib_node;
6779 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
6780 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6781 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6785 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
6786 enum mlxsw_sp_l3proto proto,
6789 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
6790 enum mlxsw_reg_ralxx_protocol ralxx_proto =
6791 (enum mlxsw_reg_ralxx_protocol) proto;
6792 struct mlxsw_sp_fib_entry_priv *priv;
6793 char xralta_pl[MLXSW_REG_XRALTA_LEN];
6794 char xralst_pl[MLXSW_REG_XRALST_LEN];
6797 mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id);
6798 err = ll_ops->ralta_write(mlxsw_sp, xralta_pl);
6802 mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id);
6803 err = ll_ops->ralst_write(mlxsw_sp, xralst_pl);
6807 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
6808 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
6809 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
6810 char xraltb_pl[MLXSW_REG_XRALTB_LEN];
6812 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
6813 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id);
6814 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
6818 priv = mlxsw_sp_fib_entry_priv_create(ll_ops);
6820 return PTR_ERR(priv);
6822 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE,
6823 vr->id, 0, NULL, priv);
6824 ll_ops->fib_entry_act_ip2me_pack(op_ctx);
6825 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL);
6826 mlxsw_sp_fib_entry_priv_put(priv);
6834 static struct mlxsw_sp_mr_table *
6835 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
6837 if (family == RTNL_FAMILY_IPMR)
6838 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
6840 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
6843 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
6844 struct mfc_entry_notifier_info *men_info,
6847 struct mlxsw_sp_mr_table *mrt;
6848 struct mlxsw_sp_vr *vr;
6850 if (mlxsw_sp->router->aborted)
6853 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
6857 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6858 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
6861 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
6862 struct mfc_entry_notifier_info *men_info)
6864 struct mlxsw_sp_mr_table *mrt;
6865 struct mlxsw_sp_vr *vr;
6867 if (mlxsw_sp->router->aborted)
6870 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
6874 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
6875 mlxsw_sp_mr_route_del(mrt, men_info->mfc);
6876 mlxsw_sp_vr_put(mlxsw_sp, vr);
6880 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
6881 struct vif_entry_notifier_info *ven_info)
6883 struct mlxsw_sp_mr_table *mrt;
6884 struct mlxsw_sp_rif *rif;
6885 struct mlxsw_sp_vr *vr;
6887 if (mlxsw_sp->router->aborted)
6890 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
6894 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6895 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
6896 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
6897 ven_info->vif_index,
6898 ven_info->vif_flags, rif);
6902 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
6903 struct vif_entry_notifier_info *ven_info)
6905 struct mlxsw_sp_mr_table *mrt;
6906 struct mlxsw_sp_vr *vr;
6908 if (mlxsw_sp->router->aborted)
6911 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
6915 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
6916 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
6917 mlxsw_sp_vr_put(mlxsw_sp, vr);
6920 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
6922 enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4;
6925 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6926 MLXSW_SP_LPM_TREE_MIN);
6930 /* The multicast router code does not need an abort trap as by default,
6931 * packets that don't match any routes are trapped to the CPU.
6934 proto = MLXSW_SP_L3_PROTO_IPV6;
6935 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
6936 MLXSW_SP_LPM_TREE_MIN + 1);
6939 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
6940 struct mlxsw_sp_fib_node *fib_node)
6942 struct mlxsw_sp_fib4_entry *fib4_entry;
6944 fib4_entry = container_of(fib_node->fib_entry,
6945 struct mlxsw_sp_fib4_entry, common);
6946 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6947 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6948 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6951 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
6952 struct mlxsw_sp_fib_node *fib_node)
6954 struct mlxsw_sp_fib6_entry *fib6_entry;
6956 fib6_entry = container_of(fib_node->fib_entry,
6957 struct mlxsw_sp_fib6_entry, common);
6958 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
6959 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
6960 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6963 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
6964 struct mlxsw_sp_fib_node *fib_node)
6966 switch (fib_node->fib->proto) {
6967 case MLXSW_SP_L3_PROTO_IPV4:
6968 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
6970 case MLXSW_SP_L3_PROTO_IPV6:
6971 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
6976 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
6977 struct mlxsw_sp_vr *vr,
6978 enum mlxsw_sp_l3proto proto)
6980 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
6981 struct mlxsw_sp_fib_node *fib_node, *tmp;
6983 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
6984 bool do_break = &tmp->list == &fib->node_list;
6986 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
6992 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
6996 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
6997 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
6999 if (!mlxsw_sp_vr_is_used(vr))
7002 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
7003 mlxsw_sp_mr_table_flush(vr->mr_table[j]);
7004 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
7006 /* If virtual router was only used for IPv4, then it's no
7009 if (!mlxsw_sp_vr_is_used(vr))
7011 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
7014 /* After flushing all the routes, it is not possible anyone is still
7015 * using the adjacency index that is discarding packets, so free it in
7016 * case it was allocated.
7018 if (!mlxsw_sp->router->adj_discard_index_valid)
7020 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
7021 mlxsw_sp->router->adj_discard_index);
7022 mlxsw_sp->router->adj_discard_index_valid = false;
7025 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
7029 if (mlxsw_sp->router->aborted)
7031 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
7032 mlxsw_sp_router_fib_flush(mlxsw_sp);
7033 mlxsw_sp->router->aborted = true;
7034 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
7036 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
7039 struct mlxsw_sp_fib6_event {
7040 struct fib6_info **rt_arr;
7044 struct mlxsw_sp_fib_event {
7045 struct list_head list; /* node in fib queue */
7047 struct mlxsw_sp_fib6_event fib6_event;
7048 struct fib_entry_notifier_info fen_info;
7049 struct fib_rule_notifier_info fr_info;
7050 struct fib_nh_notifier_info fnh_info;
7051 struct mfc_entry_notifier_info men_info;
7052 struct vif_entry_notifier_info ven_info;
7054 struct mlxsw_sp *mlxsw_sp;
7055 unsigned long event;
7060 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
7061 struct fib6_entry_notifier_info *fen6_info)
7063 struct fib6_info *rt = fen6_info->rt;
7064 struct fib6_info **rt_arr;
7065 struct fib6_info *iter;
7069 nrt6 = fen6_info->nsiblings + 1;
7071 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
7075 fib6_event->rt_arr = rt_arr;
7076 fib6_event->nrt6 = nrt6;
7081 if (!fen6_info->nsiblings)
7084 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
7085 if (i == fen6_info->nsiblings)
7088 rt_arr[i + 1] = iter;
7089 fib6_info_hold(iter);
7092 WARN_ON_ONCE(i != fen6_info->nsiblings);
7098 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
7102 for (i = 0; i < fib6_event->nrt6; i++)
7103 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
7104 kfree(fib6_event->rt_arr);
7107 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
7108 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
7109 struct mlxsw_sp_fib_event *fib_event)
7113 mlxsw_sp_span_respin(mlxsw_sp);
7115 switch (fib_event->event) {
7116 case FIB_EVENT_ENTRY_REPLACE:
7117 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
7119 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
7120 mlxsw_sp_router_fib_abort(mlxsw_sp);
7121 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
7122 &fib_event->fen_info);
7124 fib_info_put(fib_event->fen_info.fi);
7126 case FIB_EVENT_ENTRY_DEL:
7127 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
7129 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
7130 fib_info_put(fib_event->fen_info.fi);
7132 case FIB_EVENT_NH_ADD:
7133 case FIB_EVENT_NH_DEL:
7134 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
7135 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
7140 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
7141 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
7142 struct mlxsw_sp_fib_event *fib_event)
7144 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event;
7147 mlxsw_sp_span_respin(mlxsw_sp);
7149 switch (fib_event->event) {
7150 case FIB_EVENT_ENTRY_REPLACE:
7151 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
7152 fib_event->fib6_event.nrt6);
7154 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
7155 mlxsw_sp_router_fib_abort(mlxsw_sp);
7156 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7160 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
7162 case FIB_EVENT_ENTRY_APPEND:
7163 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
7164 fib_event->fib6_event.nrt6);
7166 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
7167 mlxsw_sp_router_fib_abort(mlxsw_sp);
7168 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7172 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
7174 case FIB_EVENT_ENTRY_DEL:
7175 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
7176 fib_event->fib6_event.nrt6);
7178 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
7179 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
7184 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
7185 struct mlxsw_sp_fib_event *fib_event)
7191 mutex_lock(&mlxsw_sp->router->lock);
7192 switch (fib_event->event) {
7193 case FIB_EVENT_ENTRY_REPLACE:
7194 case FIB_EVENT_ENTRY_ADD:
7195 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
7197 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
7199 mlxsw_sp_router_fib_abort(mlxsw_sp);
7200 mr_cache_put(fib_event->men_info.mfc);
7202 case FIB_EVENT_ENTRY_DEL:
7203 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
7204 mr_cache_put(fib_event->men_info.mfc);
7206 case FIB_EVENT_VIF_ADD:
7207 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
7208 &fib_event->ven_info);
7210 mlxsw_sp_router_fib_abort(mlxsw_sp);
7211 dev_put(fib_event->ven_info.dev);
7213 case FIB_EVENT_VIF_DEL:
7214 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
7215 dev_put(fib_event->ven_info.dev);
7218 mutex_unlock(&mlxsw_sp->router->lock);
7222 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
7224 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
7225 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx;
7226 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
7227 struct mlxsw_sp_fib_event *next_fib_event;
7228 struct mlxsw_sp_fib_event *fib_event;
7229 int last_family = AF_UNSPEC;
7230 LIST_HEAD(fib_event_queue);
7232 spin_lock_bh(&router->fib_event_queue_lock);
7233 list_splice_init(&router->fib_event_queue, &fib_event_queue);
7234 spin_unlock_bh(&router->fib_event_queue_lock);
7236 /* Router lock is held here to make sure per-instance
7237 * operation context is not used in between FIB4/6 events
7240 mutex_lock(&router->lock);
7241 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
7242 list_for_each_entry_safe(fib_event, next_fib_event,
7243 &fib_event_queue, list) {
7244 /* Check if the next entry in the queue exists and it is
7245 * of the same type (family and event) as the currect one.
7246 * In that case it is permitted to do the bulking
7247 * of multiple FIB entries to a single register write.
7249 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) &&
7250 fib_event->family == next_fib_event->family &&
7251 fib_event->event == next_fib_event->event;
7252 op_ctx->event = fib_event->event;
7254 /* In case family of this and the previous entry are different, context
7255 * reinitialization is going to be needed now, indicate that.
7256 * Note that since last_family is initialized to AF_UNSPEC, this is always
7257 * going to happen for the first entry processed in the work.
7259 if (fib_event->family != last_family)
7260 op_ctx->initialized = false;
7262 switch (fib_event->family) {
7264 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx,
7268 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx,
7271 case RTNL_FAMILY_IP6MR:
7272 case RTNL_FAMILY_IPMR:
7273 /* Unlock here as inside FIBMR the lock is taken again
7274 * under RTNL. The per-instance operation context
7275 * is not used by FIBMR.
7277 mutex_unlock(&router->lock);
7278 mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
7280 mutex_lock(&router->lock);
7285 last_family = fib_event->family;
7289 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
7290 mutex_unlock(&router->lock);
7293 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
7294 struct fib_notifier_info *info)
7296 struct fib_entry_notifier_info *fen_info;
7297 struct fib_nh_notifier_info *fnh_info;
7299 switch (fib_event->event) {
7300 case FIB_EVENT_ENTRY_REPLACE:
7301 case FIB_EVENT_ENTRY_DEL:
7302 fen_info = container_of(info, struct fib_entry_notifier_info,
7304 fib_event->fen_info = *fen_info;
7305 /* Take reference on fib_info to prevent it from being
7306 * freed while event is queued. Release it afterwards.
7308 fib_info_hold(fib_event->fen_info.fi);
7310 case FIB_EVENT_NH_ADD:
7311 case FIB_EVENT_NH_DEL:
7312 fnh_info = container_of(info, struct fib_nh_notifier_info,
7314 fib_event->fnh_info = *fnh_info;
7315 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
7320 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
7321 struct fib_notifier_info *info)
7323 struct fib6_entry_notifier_info *fen6_info;
7326 switch (fib_event->event) {
7327 case FIB_EVENT_ENTRY_REPLACE:
7328 case FIB_EVENT_ENTRY_APPEND:
7329 case FIB_EVENT_ENTRY_DEL:
7330 fen6_info = container_of(info, struct fib6_entry_notifier_info,
7332 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
7343 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
7344 struct fib_notifier_info *info)
7346 switch (fib_event->event) {
7347 case FIB_EVENT_ENTRY_REPLACE:
7348 case FIB_EVENT_ENTRY_ADD:
7349 case FIB_EVENT_ENTRY_DEL:
7350 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
7351 mr_cache_hold(fib_event->men_info.mfc);
7353 case FIB_EVENT_VIF_ADD:
7354 case FIB_EVENT_VIF_DEL:
7355 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
7356 dev_hold(fib_event->ven_info.dev);
7361 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
7362 struct fib_notifier_info *info,
7363 struct mlxsw_sp *mlxsw_sp)
7365 struct netlink_ext_ack *extack = info->extack;
7366 struct fib_rule_notifier_info *fr_info;
7367 struct fib_rule *rule;
7370 /* nothing to do at the moment */
7371 if (event == FIB_EVENT_RULE_DEL)
7374 if (mlxsw_sp->router->aborted)
7377 fr_info = container_of(info, struct fib_rule_notifier_info, info);
7378 rule = fr_info->rule;
7380 /* Rule only affects locally generated traffic */
7381 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
7384 switch (info->family) {
7386 if (!fib4_rule_default(rule) && !rule->l3mdev)
7390 if (!fib6_rule_default(rule) && !rule->l3mdev)
7393 case RTNL_FAMILY_IPMR:
7394 if (!ipmr_rule_default(rule) && !rule->l3mdev)
7397 case RTNL_FAMILY_IP6MR:
7398 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
7404 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
7409 /* Called with rcu_read_lock() */
7410 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
7411 unsigned long event, void *ptr)
7413 struct mlxsw_sp_fib_event *fib_event;
7414 struct fib_notifier_info *info = ptr;
7415 struct mlxsw_sp_router *router;
7418 if ((info->family != AF_INET && info->family != AF_INET6 &&
7419 info->family != RTNL_FAMILY_IPMR &&
7420 info->family != RTNL_FAMILY_IP6MR))
7423 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7426 case FIB_EVENT_RULE_ADD:
7427 case FIB_EVENT_RULE_DEL:
7428 err = mlxsw_sp_router_fib_rule_event(event, info,
7430 return notifier_from_errno(err);
7431 case FIB_EVENT_ENTRY_ADD:
7432 case FIB_EVENT_ENTRY_REPLACE:
7433 case FIB_EVENT_ENTRY_APPEND:
7434 if (router->aborted) {
7435 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
7436 return notifier_from_errno(-EINVAL);
7438 if (info->family == AF_INET) {
7439 struct fib_entry_notifier_info *fen_info = ptr;
7441 if (fen_info->fi->fib_nh_is_v6) {
7442 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
7443 return notifier_from_errno(-EINVAL);
7449 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
7453 fib_event->mlxsw_sp = router->mlxsw_sp;
7454 fib_event->event = event;
7455 fib_event->family = info->family;
7457 switch (info->family) {
7459 mlxsw_sp_router_fib4_event(fib_event, info);
7462 err = mlxsw_sp_router_fib6_event(fib_event, info);
7466 case RTNL_FAMILY_IP6MR:
7467 case RTNL_FAMILY_IPMR:
7468 mlxsw_sp_router_fibmr_event(fib_event, info);
7472 /* Enqueue the event and trigger the work */
7473 spin_lock_bh(&router->fib_event_queue_lock);
7474 list_add_tail(&fib_event->list, &router->fib_event_queue);
7475 spin_unlock_bh(&router->fib_event_queue_lock);
7476 mlxsw_core_schedule_work(&router->fib_event_work);
7485 static struct mlxsw_sp_rif *
7486 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
7487 const struct net_device *dev)
7491 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
7492 if (mlxsw_sp->router->rifs[i] &&
7493 mlxsw_sp->router->rifs[i]->dev == dev)
7494 return mlxsw_sp->router->rifs[i];
7499 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
7500 const struct net_device *dev)
7502 struct mlxsw_sp_rif *rif;
7504 mutex_lock(&mlxsw_sp->router->lock);
7505 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7506 mutex_unlock(&mlxsw_sp->router->lock);
7511 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
7513 struct mlxsw_sp_rif *rif;
7516 mutex_lock(&mlxsw_sp->router->lock);
7517 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7521 /* We only return the VID for VLAN RIFs. Otherwise we return an
7522 * invalid value (0).
7524 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
7527 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7530 mutex_unlock(&mlxsw_sp->router->lock);
7534 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
7536 char ritr_pl[MLXSW_REG_RITR_LEN];
7539 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
7540 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7544 mlxsw_reg_ritr_enable_set(ritr_pl, false);
7545 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7548 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
7549 struct mlxsw_sp_rif *rif)
7551 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
7552 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
7553 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
7557 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
7558 unsigned long event)
7560 struct inet6_dev *inet6_dev;
7561 bool addr_list_empty = true;
7562 struct in_device *idev;
7569 idev = __in_dev_get_rcu(dev);
7570 if (idev && idev->ifa_list)
7571 addr_list_empty = false;
7573 inet6_dev = __in6_dev_get(dev);
7574 if (addr_list_empty && inet6_dev &&
7575 !list_empty(&inet6_dev->addr_list))
7576 addr_list_empty = false;
7579 /* macvlans do not have a RIF, but rather piggy back on the
7580 * RIF of their lower device.
7582 if (netif_is_macvlan(dev) && addr_list_empty)
7585 if (rif && addr_list_empty &&
7586 !netif_is_l3_slave(rif->dev))
7588 /* It is possible we already removed the RIF ourselves
7589 * if it was assigned to a netdev that is now a bridge
7598 static enum mlxsw_sp_rif_type
7599 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
7600 const struct net_device *dev)
7602 enum mlxsw_sp_fid_type type;
7604 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
7605 return MLXSW_SP_RIF_TYPE_IPIP_LB;
7607 /* Otherwise RIF type is derived from the type of the underlying FID. */
7608 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
7609 type = MLXSW_SP_FID_TYPE_8021Q;
7610 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
7611 type = MLXSW_SP_FID_TYPE_8021Q;
7612 else if (netif_is_bridge_master(dev))
7613 type = MLXSW_SP_FID_TYPE_8021D;
7615 type = MLXSW_SP_FID_TYPE_RFID;
7617 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
7620 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
7624 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
7625 if (!mlxsw_sp->router->rifs[i]) {
7634 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
7636 struct net_device *l3_dev)
7638 struct mlxsw_sp_rif *rif;
7640 rif = kzalloc(rif_size, GFP_KERNEL);
7644 INIT_LIST_HEAD(&rif->nexthop_list);
7645 INIT_LIST_HEAD(&rif->neigh_list);
7647 ether_addr_copy(rif->addr, l3_dev->dev_addr);
7648 rif->mtu = l3_dev->mtu;
7652 rif->rif_index = rif_index;
7657 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
7660 return mlxsw_sp->router->rifs[rif_index];
7663 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
7665 return rif->rif_index;
7668 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7670 return lb_rif->common.rif_index;
7673 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7675 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
7676 struct mlxsw_sp_vr *ul_vr;
7678 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
7679 if (WARN_ON(IS_ERR(ul_vr)))
7685 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7687 return lb_rif->ul_rif_id;
7690 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
7692 return rif->dev->ifindex;
7695 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
7700 static struct mlxsw_sp_rif *
7701 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
7702 const struct mlxsw_sp_rif_params *params,
7703 struct netlink_ext_ack *extack)
7705 u32 tb_id = l3mdev_fib_table(params->dev);
7706 const struct mlxsw_sp_rif_ops *ops;
7707 struct mlxsw_sp_fid *fid = NULL;
7708 enum mlxsw_sp_rif_type type;
7709 struct mlxsw_sp_rif *rif;
7710 struct mlxsw_sp_vr *vr;
7714 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
7715 ops = mlxsw_sp->rif_ops_arr[type];
7717 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
7719 return ERR_CAST(vr);
7722 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
7724 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
7725 goto err_rif_index_alloc;
7728 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
7734 mlxsw_sp->router->rifs[rif_index] = rif;
7735 rif->mlxsw_sp = mlxsw_sp;
7739 fid = ops->fid_get(rif, extack);
7748 ops->setup(rif, params);
7750 err = ops->configure(rif);
7754 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
7755 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
7757 goto err_mr_rif_add;
7760 mlxsw_sp_rif_counters_alloc(rif);
7765 for (i--; i >= 0; i--)
7766 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
7767 ops->deconfigure(rif);
7770 mlxsw_sp_fid_put(fid);
7772 mlxsw_sp->router->rifs[rif_index] = NULL;
7776 err_rif_index_alloc:
7778 mlxsw_sp_vr_put(mlxsw_sp, vr);
7779 return ERR_PTR(err);
7782 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
7784 const struct mlxsw_sp_rif_ops *ops = rif->ops;
7785 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7786 struct mlxsw_sp_fid *fid = rif->fid;
7787 struct mlxsw_sp_vr *vr;
7790 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
7791 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7793 mlxsw_sp_rif_counters_free(rif);
7794 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7795 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
7796 ops->deconfigure(rif);
7798 /* Loopback RIFs are not associated with a FID. */
7799 mlxsw_sp_fid_put(fid);
7800 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
7804 mlxsw_sp_vr_put(mlxsw_sp, vr);
7807 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
7808 struct net_device *dev)
7810 struct mlxsw_sp_rif *rif;
7812 mutex_lock(&mlxsw_sp->router->lock);
7813 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7816 mlxsw_sp_rif_destroy(rif);
7818 mutex_unlock(&mlxsw_sp->router->lock);
7822 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
7823 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7825 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7827 params->vid = mlxsw_sp_port_vlan->vid;
7828 params->lag = mlxsw_sp_port->lagged;
7830 params->lag_id = mlxsw_sp_port->lag_id;
7832 params->system_port = mlxsw_sp_port->local_port;
7835 static struct mlxsw_sp_rif_subport *
7836 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
7838 return container_of(rif, struct mlxsw_sp_rif_subport, common);
7841 static struct mlxsw_sp_rif *
7842 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
7843 const struct mlxsw_sp_rif_params *params,
7844 struct netlink_ext_ack *extack)
7846 struct mlxsw_sp_rif_subport *rif_subport;
7847 struct mlxsw_sp_rif *rif;
7849 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
7851 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
7853 rif_subport = mlxsw_sp_rif_subport_rif(rif);
7854 refcount_inc(&rif_subport->ref_count);
7858 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
7860 struct mlxsw_sp_rif_subport *rif_subport;
7862 rif_subport = mlxsw_sp_rif_subport_rif(rif);
7863 if (!refcount_dec_and_test(&rif_subport->ref_count))
7866 mlxsw_sp_rif_destroy(rif);
7870 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7871 struct net_device *l3_dev,
7872 struct netlink_ext_ack *extack)
7874 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7875 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7876 struct mlxsw_sp_rif_params params = {
7879 u16 vid = mlxsw_sp_port_vlan->vid;
7880 struct mlxsw_sp_rif *rif;
7881 struct mlxsw_sp_fid *fid;
7884 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan);
7885 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack);
7887 return PTR_ERR(rif);
7889 /* FID was already created, just take a reference */
7890 fid = rif->ops->fid_get(rif, extack);
7891 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
7893 goto err_fid_port_vid_map;
7895 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
7897 goto err_port_vid_learning_set;
7899 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
7900 BR_STATE_FORWARDING);
7902 goto err_port_vid_stp_set;
7904 mlxsw_sp_port_vlan->fid = fid;
7908 err_port_vid_stp_set:
7909 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7910 err_port_vid_learning_set:
7911 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7912 err_fid_port_vid_map:
7913 mlxsw_sp_fid_put(fid);
7914 mlxsw_sp_rif_subport_put(rif);
7919 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7921 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7922 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
7923 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
7924 u16 vid = mlxsw_sp_port_vlan->vid;
7926 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
7929 mlxsw_sp_port_vlan->fid = NULL;
7930 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
7931 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
7932 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
7933 mlxsw_sp_fid_put(fid);
7934 mlxsw_sp_rif_subport_put(rif);
7938 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7939 struct net_device *l3_dev,
7940 struct netlink_ext_ack *extack)
7942 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
7943 struct mlxsw_sp_rif *rif;
7946 mutex_lock(&mlxsw_sp->router->lock);
7947 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7951 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
7954 mutex_unlock(&mlxsw_sp->router->lock);
7959 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
7961 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
7963 mutex_lock(&mlxsw_sp->router->lock);
7964 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7965 mutex_unlock(&mlxsw_sp->router->lock);
7968 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
7969 struct net_device *port_dev,
7970 unsigned long event, u16 vid,
7971 struct netlink_ext_ack *extack)
7973 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
7974 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
7976 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7977 if (WARN_ON(!mlxsw_sp_port_vlan))
7982 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7985 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
7992 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
7993 unsigned long event,
7994 struct netlink_ext_ack *extack)
7996 if (netif_is_bridge_port(port_dev) ||
7997 netif_is_lag_port(port_dev) ||
7998 netif_is_ovs_port(port_dev))
8001 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
8002 MLXSW_SP_DEFAULT_VID, extack);
8005 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
8006 struct net_device *lag_dev,
8007 unsigned long event, u16 vid,
8008 struct netlink_ext_ack *extack)
8010 struct net_device *port_dev;
8011 struct list_head *iter;
8014 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
8015 if (mlxsw_sp_port_dev_check(port_dev)) {
8016 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
8028 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
8029 unsigned long event,
8030 struct netlink_ext_ack *extack)
8032 if (netif_is_bridge_port(lag_dev))
8035 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
8036 MLXSW_SP_DEFAULT_VID, extack);
8039 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
8040 struct net_device *l3_dev,
8041 unsigned long event,
8042 struct netlink_ext_ack *extack)
8044 struct mlxsw_sp_rif_params params = {
8047 struct mlxsw_sp_rif *rif;
8051 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) {
8054 br_vlan_get_proto(l3_dev, &proto);
8055 if (proto == ETH_P_8021AD) {
8056 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
8060 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack);
8062 return PTR_ERR(rif);
8065 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
8066 mlxsw_sp_rif_destroy(rif);
8073 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
8074 struct net_device *vlan_dev,
8075 unsigned long event,
8076 struct netlink_ext_ack *extack)
8078 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
8079 u16 vid = vlan_dev_vlan_id(vlan_dev);
8081 if (netif_is_bridge_port(vlan_dev))
8084 if (mlxsw_sp_port_dev_check(real_dev))
8085 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
8086 event, vid, extack);
8087 else if (netif_is_lag_master(real_dev))
8088 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
8090 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
8091 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
8097 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
8099 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
8100 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
8102 return ether_addr_equal_masked(mac, vrrp4, mask);
8105 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
8107 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
8108 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
8110 return ether_addr_equal_masked(mac, vrrp6, mask);
8113 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8114 const u8 *mac, bool adding)
8116 char ritr_pl[MLXSW_REG_RITR_LEN];
8117 u8 vrrp_id = adding ? mac[5] : 0;
8120 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
8121 !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
8124 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
8125 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8129 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
8130 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
8132 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
8134 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8137 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
8138 const struct net_device *macvlan_dev,
8139 struct netlink_ext_ack *extack)
8141 struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
8142 struct mlxsw_sp_rif *rif;
8145 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
8147 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
8151 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8152 mlxsw_sp_fid_index(rif->fid), true);
8156 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
8157 macvlan_dev->dev_addr, true);
8159 goto err_rif_vrrp_add;
8161 /* Make sure the bridge driver does not have this MAC pointing at
8164 if (rif->ops->fdb_del)
8165 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
8170 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8171 mlxsw_sp_fid_index(rif->fid), false);
8175 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
8176 const struct net_device *macvlan_dev)
8178 struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
8179 struct mlxsw_sp_rif *rif;
8181 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
8182 /* If we do not have a RIF, then we already took care of
8183 * removing the macvlan's MAC during RIF deletion.
8187 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
8189 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8190 mlxsw_sp_fid_index(rif->fid), false);
8193 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
8194 const struct net_device *macvlan_dev)
8196 mutex_lock(&mlxsw_sp->router->lock);
8197 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
8198 mutex_unlock(&mlxsw_sp->router->lock);
8201 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
8202 struct net_device *macvlan_dev,
8203 unsigned long event,
8204 struct netlink_ext_ack *extack)
8208 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
8210 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
8217 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
8218 struct net_device *dev,
8219 const unsigned char *dev_addr,
8220 struct netlink_ext_ack *extack)
8222 struct mlxsw_sp_rif *rif;
8225 /* A RIF is not created for macvlan netdevs. Their MAC is used to
8228 if (netif_is_macvlan(dev) || netif_is_l3_master(dev))
8231 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
8232 rif = mlxsw_sp->router->rifs[i];
8233 if (rif && rif->ops &&
8234 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB)
8236 if (rif && rif->dev && rif->dev != dev &&
8237 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
8238 mlxsw_sp->mac_mask)) {
8239 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
8247 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
8248 struct net_device *dev,
8249 unsigned long event,
8250 struct netlink_ext_ack *extack)
8252 if (mlxsw_sp_port_dev_check(dev))
8253 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
8254 else if (netif_is_lag_master(dev))
8255 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
8256 else if (netif_is_bridge_master(dev))
8257 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
8259 else if (is_vlan_dev(dev))
8260 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
8262 else if (netif_is_macvlan(dev))
8263 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
8269 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
8270 unsigned long event, void *ptr)
8272 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
8273 struct net_device *dev = ifa->ifa_dev->dev;
8274 struct mlxsw_sp_router *router;
8275 struct mlxsw_sp_rif *rif;
8278 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
8279 if (event == NETDEV_UP)
8282 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
8283 mutex_lock(&router->lock);
8284 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
8285 if (!mlxsw_sp_rif_should_config(rif, dev, event))
8288 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
8290 mutex_unlock(&router->lock);
8291 return notifier_from_errno(err);
8294 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
8295 unsigned long event, void *ptr)
8297 struct in_validator_info *ivi = (struct in_validator_info *) ptr;
8298 struct net_device *dev = ivi->ivi_dev->dev;
8299 struct mlxsw_sp *mlxsw_sp;
8300 struct mlxsw_sp_rif *rif;
8303 mlxsw_sp = mlxsw_sp_lower_get(dev);
8307 mutex_lock(&mlxsw_sp->router->lock);
8308 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8309 if (!mlxsw_sp_rif_should_config(rif, dev, event))
8312 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
8317 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
8319 mutex_unlock(&mlxsw_sp->router->lock);
8320 return notifier_from_errno(err);
8323 struct mlxsw_sp_inet6addr_event_work {
8324 struct work_struct work;
8325 struct mlxsw_sp *mlxsw_sp;
8326 struct net_device *dev;
8327 unsigned long event;
8330 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
8332 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
8333 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
8334 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
8335 struct net_device *dev = inet6addr_work->dev;
8336 unsigned long event = inet6addr_work->event;
8337 struct mlxsw_sp_rif *rif;
8340 mutex_lock(&mlxsw_sp->router->lock);
8342 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8343 if (!mlxsw_sp_rif_should_config(rif, dev, event))
8346 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
8348 mutex_unlock(&mlxsw_sp->router->lock);
8351 kfree(inet6addr_work);
8354 /* Called with rcu_read_lock() */
8355 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
8356 unsigned long event, void *ptr)
8358 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
8359 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
8360 struct net_device *dev = if6->idev->dev;
8361 struct mlxsw_sp_router *router;
8363 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
8364 if (event == NETDEV_UP)
8367 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
8368 if (!inet6addr_work)
8371 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
8372 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
8373 inet6addr_work->mlxsw_sp = router->mlxsw_sp;
8374 inet6addr_work->dev = dev;
8375 inet6addr_work->event = event;
8377 mlxsw_core_schedule_work(&inet6addr_work->work);
8382 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
8383 unsigned long event, void *ptr)
8385 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
8386 struct net_device *dev = i6vi->i6vi_dev->dev;
8387 struct mlxsw_sp *mlxsw_sp;
8388 struct mlxsw_sp_rif *rif;
8391 mlxsw_sp = mlxsw_sp_lower_get(dev);
8395 mutex_lock(&mlxsw_sp->router->lock);
8396 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8397 if (!mlxsw_sp_rif_should_config(rif, dev, event))
8400 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
8405 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
8407 mutex_unlock(&mlxsw_sp->router->lock);
8408 return notifier_from_errno(err);
8411 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8412 const char *mac, int mtu)
8414 char ritr_pl[MLXSW_REG_RITR_LEN];
8417 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
8418 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8422 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
8423 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
8424 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
8425 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8429 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
8430 struct mlxsw_sp_rif *rif)
8432 struct net_device *dev = rif->dev;
8436 fid_index = mlxsw_sp_fid_index(rif->fid);
8438 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
8442 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
8447 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
8449 goto err_rif_fdb_op;
8451 if (rif->mtu != dev->mtu) {
8452 struct mlxsw_sp_vr *vr;
8455 /* The RIF is relevant only to its mr_table instance, as unlike
8456 * unicast routing, in multicast routing a RIF cannot be shared
8457 * between several multicast routing tables.
8459 vr = &mlxsw_sp->router->vrs[rif->vr_id];
8460 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
8461 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
8465 ether_addr_copy(rif->addr, dev->dev_addr);
8466 rif->mtu = dev->mtu;
8468 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
8473 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
8475 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
8479 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
8480 struct netdev_notifier_pre_changeaddr_info *info)
8482 struct netlink_ext_ack *extack;
8484 extack = netdev_notifier_info_to_extack(&info->info);
8485 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
8486 info->dev_addr, extack);
8489 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
8490 unsigned long event, void *ptr)
8492 struct mlxsw_sp *mlxsw_sp;
8493 struct mlxsw_sp_rif *rif;
8496 mlxsw_sp = mlxsw_sp_lower_get(dev);
8500 mutex_lock(&mlxsw_sp->router->lock);
8501 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8506 case NETDEV_CHANGEMTU:
8507 case NETDEV_CHANGEADDR:
8508 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
8510 case NETDEV_PRE_CHANGEADDR:
8511 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
8516 mutex_unlock(&mlxsw_sp->router->lock);
8520 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
8521 struct net_device *l3_dev,
8522 struct netlink_ext_ack *extack)
8524 struct mlxsw_sp_rif *rif;
8526 /* If netdev is already associated with a RIF, then we need to
8527 * destroy it and create a new one with the new virtual router ID.
8529 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
8531 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
8534 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
8537 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
8538 struct net_device *l3_dev)
8540 struct mlxsw_sp_rif *rif;
8542 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
8545 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
8548 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
8549 struct netdev_notifier_changeupper_info *info)
8551 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
8554 /* We do not create a RIF for a macvlan, but only use it to
8555 * direct more MAC addresses to the router.
8557 if (!mlxsw_sp || netif_is_macvlan(l3_dev))
8560 mutex_lock(&mlxsw_sp->router->lock);
8562 case NETDEV_PRECHANGEUPPER:
8564 case NETDEV_CHANGEUPPER:
8565 if (info->linking) {
8566 struct netlink_ext_ack *extack;
8568 extack = netdev_notifier_info_to_extack(&info->info);
8569 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
8571 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
8575 mutex_unlock(&mlxsw_sp->router->lock);
8580 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
8581 struct netdev_nested_priv *priv)
8583 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
8585 if (!netif_is_macvlan(dev))
8588 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
8589 mlxsw_sp_fid_index(rif->fid), false);
8592 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
8594 struct netdev_nested_priv priv = {
8595 .data = (void *)rif,
8598 if (!netif_is_macvlan_port(rif->dev))
8601 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
8602 return netdev_walk_all_upper_dev_rcu(rif->dev,
8603 __mlxsw_sp_rif_macvlan_flush, &priv);
8606 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
8607 const struct mlxsw_sp_rif_params *params)
8609 struct mlxsw_sp_rif_subport *rif_subport;
8611 rif_subport = mlxsw_sp_rif_subport_rif(rif);
8612 refcount_set(&rif_subport->ref_count, 1);
8613 rif_subport->vid = params->vid;
8614 rif_subport->lag = params->lag;
8616 rif_subport->lag_id = params->lag_id;
8618 rif_subport->system_port = params->system_port;
8621 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
8623 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8624 struct mlxsw_sp_rif_subport *rif_subport;
8625 char ritr_pl[MLXSW_REG_RITR_LEN];
8627 rif_subport = mlxsw_sp_rif_subport_rif(rif);
8628 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
8629 rif->rif_index, rif->vr_id, rif->dev->mtu);
8630 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
8631 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
8632 rif_subport->lag ? rif_subport->lag_id :
8633 rif_subport->system_port,
8636 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8639 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
8643 err = mlxsw_sp_rif_subport_op(rif, true);
8647 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
8648 mlxsw_sp_fid_index(rif->fid), true);
8650 goto err_rif_fdb_op;
8652 mlxsw_sp_fid_rif_set(rif->fid, rif);
8656 mlxsw_sp_rif_subport_op(rif, false);
8660 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
8662 struct mlxsw_sp_fid *fid = rif->fid;
8664 mlxsw_sp_fid_rif_set(fid, NULL);
8665 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
8666 mlxsw_sp_fid_index(fid), false);
8667 mlxsw_sp_rif_macvlan_flush(rif);
8668 mlxsw_sp_rif_subport_op(rif, false);
8671 static struct mlxsw_sp_fid *
8672 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
8673 struct netlink_ext_ack *extack)
8675 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
8678 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
8679 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
8680 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
8681 .setup = mlxsw_sp_rif_subport_setup,
8682 .configure = mlxsw_sp_rif_subport_configure,
8683 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
8684 .fid_get = mlxsw_sp_rif_subport_fid_get,
8687 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
8688 enum mlxsw_reg_ritr_if_type type,
8689 u16 vid_fid, bool enable)
8691 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8692 char ritr_pl[MLXSW_REG_RITR_LEN];
8694 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
8696 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
8697 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
8699 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8702 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
8704 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
8707 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
8709 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8710 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
8713 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
8718 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
8719 mlxsw_sp_router_port(mlxsw_sp), true);
8721 goto err_fid_mc_flood_set;
8723 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
8724 mlxsw_sp_router_port(mlxsw_sp), true);
8726 goto err_fid_bc_flood_set;
8728 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
8729 mlxsw_sp_fid_index(rif->fid), true);
8731 goto err_rif_fdb_op;
8733 mlxsw_sp_fid_rif_set(rif->fid, rif);
8737 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
8738 mlxsw_sp_router_port(mlxsw_sp), false);
8739 err_fid_bc_flood_set:
8740 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
8741 mlxsw_sp_router_port(mlxsw_sp), false);
8742 err_fid_mc_flood_set:
8743 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
8747 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
8749 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
8750 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8751 struct mlxsw_sp_fid *fid = rif->fid;
8753 mlxsw_sp_fid_rif_set(fid, NULL);
8754 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
8755 mlxsw_sp_fid_index(fid), false);
8756 mlxsw_sp_rif_macvlan_flush(rif);
8757 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
8758 mlxsw_sp_router_port(mlxsw_sp), false);
8759 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
8760 mlxsw_sp_router_port(mlxsw_sp), false);
8761 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
8764 static struct mlxsw_sp_fid *
8765 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
8766 struct netlink_ext_ack *extack)
8768 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
8771 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
8773 struct switchdev_notifier_fdb_info info;
8774 struct net_device *dev;
8776 dev = br_fdb_find_port(rif->dev, mac, 0);
8782 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
8786 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
8787 .type = MLXSW_SP_RIF_TYPE_FID,
8788 .rif_size = sizeof(struct mlxsw_sp_rif),
8789 .configure = mlxsw_sp_rif_fid_configure,
8790 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
8791 .fid_get = mlxsw_sp_rif_fid_fid_get,
8792 .fdb_del = mlxsw_sp_rif_fid_fdb_del,
8795 static struct mlxsw_sp_fid *
8796 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
8797 struct netlink_ext_ack *extack)
8799 struct net_device *br_dev;
8803 if (is_vlan_dev(rif->dev)) {
8804 vid = vlan_dev_vlan_id(rif->dev);
8805 br_dev = vlan_dev_real_dev(rif->dev);
8806 if (WARN_ON(!netif_is_bridge_master(br_dev)))
8807 return ERR_PTR(-EINVAL);
8809 err = br_vlan_get_pvid(rif->dev, &vid);
8810 if (err < 0 || !vid) {
8811 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
8812 return ERR_PTR(-EINVAL);
8816 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
8819 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
8821 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
8822 struct switchdev_notifier_fdb_info info;
8823 struct net_device *br_dev;
8824 struct net_device *dev;
8826 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
8827 dev = br_fdb_find_port(br_dev, mac, vid);
8833 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
8837 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
8838 .type = MLXSW_SP_RIF_TYPE_VLAN,
8839 .rif_size = sizeof(struct mlxsw_sp_rif),
8840 .configure = mlxsw_sp_rif_fid_configure,
8841 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
8842 .fid_get = mlxsw_sp_rif_vlan_fid_get,
8843 .fdb_del = mlxsw_sp_rif_vlan_fdb_del,
8846 static struct mlxsw_sp_rif_ipip_lb *
8847 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
8849 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
8853 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
8854 const struct mlxsw_sp_rif_params *params)
8856 struct mlxsw_sp_rif_params_ipip_lb *params_lb;
8857 struct mlxsw_sp_rif_ipip_lb *rif_lb;
8859 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
8861 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
8862 rif_lb->lb_config = params_lb->lb_config;
8866 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
8868 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8869 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
8870 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8871 struct mlxsw_sp_vr *ul_vr;
8874 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
8876 return PTR_ERR(ul_vr);
8878 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
8880 goto err_loopback_op;
8882 lb_rif->ul_vr_id = ul_vr->id;
8883 lb_rif->ul_rif_id = 0;
8888 mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8892 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
8894 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
8895 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8896 struct mlxsw_sp_vr *ul_vr;
8898 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
8899 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
8902 mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
8905 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
8906 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
8907 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
8908 .setup = mlxsw_sp_rif_ipip_lb_setup,
8909 .configure = mlxsw_sp1_rif_ipip_lb_configure,
8910 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure,
8913 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
8914 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
8915 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops,
8916 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
8917 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops,
8921 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
8923 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8924 char ritr_pl[MLXSW_REG_RITR_LEN];
8926 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
8927 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
8928 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
8929 MLXSW_REG_RITR_LOOPBACK_GENERIC);
8931 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8934 static struct mlxsw_sp_rif *
8935 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
8936 struct netlink_ext_ack *extack)
8938 struct mlxsw_sp_rif *ul_rif;
8942 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
8944 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8945 return ERR_PTR(err);
8948 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
8950 return ERR_PTR(-ENOMEM);
8952 mlxsw_sp->router->rifs[rif_index] = ul_rif;
8953 ul_rif->mlxsw_sp = mlxsw_sp;
8954 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
8961 mlxsw_sp->router->rifs[rif_index] = NULL;
8963 return ERR_PTR(err);
8966 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
8968 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
8970 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
8971 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
8975 static struct mlxsw_sp_rif *
8976 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
8977 struct netlink_ext_ack *extack)
8979 struct mlxsw_sp_vr *vr;
8982 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
8984 return ERR_CAST(vr);
8986 if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
8989 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
8990 if (IS_ERR(vr->ul_rif)) {
8991 err = PTR_ERR(vr->ul_rif);
8992 goto err_ul_rif_create;
8996 refcount_set(&vr->ul_rif_refcnt, 1);
9001 mlxsw_sp_vr_put(mlxsw_sp, vr);
9002 return ERR_PTR(err);
9005 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
9007 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
9008 struct mlxsw_sp_vr *vr;
9010 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
9012 if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
9016 mlxsw_sp_ul_rif_destroy(ul_rif);
9017 mlxsw_sp_vr_put(mlxsw_sp, vr);
9020 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
9023 struct mlxsw_sp_rif *ul_rif;
9026 mutex_lock(&mlxsw_sp->router->lock);
9027 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
9028 if (IS_ERR(ul_rif)) {
9029 err = PTR_ERR(ul_rif);
9032 *ul_rif_index = ul_rif->rif_index;
9034 mutex_unlock(&mlxsw_sp->router->lock);
9038 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
9040 struct mlxsw_sp_rif *ul_rif;
9042 mutex_lock(&mlxsw_sp->router->lock);
9043 ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
9044 if (WARN_ON(!ul_rif))
9047 mlxsw_sp_ul_rif_put(ul_rif);
9049 mutex_unlock(&mlxsw_sp->router->lock);
9053 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
9055 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9056 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
9057 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9058 struct mlxsw_sp_rif *ul_rif;
9061 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
9063 return PTR_ERR(ul_rif);
9065 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
9067 goto err_loopback_op;
9069 lb_rif->ul_vr_id = 0;
9070 lb_rif->ul_rif_id = ul_rif->rif_index;
9075 mlxsw_sp_ul_rif_put(ul_rif);
9079 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
9081 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9082 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9083 struct mlxsw_sp_rif *ul_rif;
9085 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
9086 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
9087 mlxsw_sp_ul_rif_put(ul_rif);
9090 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
9091 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
9092 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
9093 .setup = mlxsw_sp_rif_ipip_lb_setup,
9094 .configure = mlxsw_sp2_rif_ipip_lb_configure,
9095 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure,
9098 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
9099 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
9100 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops,
9101 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
9102 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops,
9105 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
9107 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
9109 mlxsw_sp->router->rifs = kcalloc(max_rifs,
9110 sizeof(struct mlxsw_sp_rif *),
9112 if (!mlxsw_sp->router->rifs)
9118 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
9122 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
9123 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
9125 kfree(mlxsw_sp->router->rifs);
9129 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
9131 char tigcr_pl[MLXSW_REG_TIGCR_LEN];
9133 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
9134 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
9137 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
9141 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
9142 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
9144 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
9147 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
9151 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
9154 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
9156 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
9159 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
9161 struct mlxsw_sp_router *router;
9163 /* Flush pending FIB notifications and then flush the device's
9164 * table before requesting another dump. The FIB notification
9165 * block is unregistered, so no need to take RTNL.
9167 mlxsw_core_flush_owq();
9168 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
9169 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
9172 #ifdef CONFIG_IP_ROUTE_MULTIPATH
9173 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
9175 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
9178 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
9180 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
9183 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
9185 struct net *net = mlxsw_sp_net(mlxsw_sp);
9186 bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy;
9188 mlxsw_sp_mp_hash_header_set(recr2_pl,
9189 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
9190 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
9191 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
9192 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
9195 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
9196 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
9197 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
9198 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
9201 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl)
9203 bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp));
9205 mlxsw_sp_mp_hash_header_set(recr2_pl,
9206 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
9207 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
9208 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
9209 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
9210 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
9212 mlxsw_sp_mp_hash_field_set(recr2_pl,
9213 MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
9215 mlxsw_sp_mp_hash_header_set(recr2_pl,
9216 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
9217 mlxsw_sp_mp_hash_field_set(recr2_pl,
9218 MLXSW_REG_RECR2_TCP_UDP_SPORT);
9219 mlxsw_sp_mp_hash_field_set(recr2_pl,
9220 MLXSW_REG_RECR2_TCP_UDP_DPORT);
9224 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
9226 char recr2_pl[MLXSW_REG_RECR2_LEN];
9229 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
9230 mlxsw_reg_recr2_pack(recr2_pl, seed);
9231 mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl);
9232 mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl);
9234 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
9237 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
9243 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
9245 char rdpm_pl[MLXSW_REG_RDPM_LEN];
9248 MLXSW_REG_ZERO(rdpm, rdpm_pl);
9250 /* HW is determining switch priority based on DSCP-bits, but the
9251 * kernel is still doing that based on the ToS. Since there's a
9252 * mismatch in bits we need to make sure to translate the right
9253 * value ToS would observe, skipping the 2 least-significant ECN bits.
9255 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
9256 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
9258 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
9261 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
9263 struct net *net = mlxsw_sp_net(mlxsw_sp);
9264 bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
9265 char rgcr_pl[MLXSW_REG_RGCR_LEN];
9268 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
9270 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
9272 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
9273 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
9274 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
9275 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
9278 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
9280 char rgcr_pl[MLXSW_REG_RGCR_LEN];
9282 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
9283 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
9286 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
9287 .init = mlxsw_sp_router_ll_basic_init,
9288 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
9289 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
9290 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
9291 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic),
9292 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack,
9293 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack,
9294 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack,
9295 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack,
9296 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack,
9297 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit,
9298 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed,
9301 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router)
9303 size_t max_size = 0;
9306 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
9307 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size;
9309 if (size > max_size)
9312 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size,
9314 if (!router->ll_op_ctx)
9316 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list);
9320 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router)
9322 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
9323 kfree(router->ll_op_ctx);
9326 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp)
9331 /* Create a generic loopback RIF associated with the main table
9332 * (default VRF). Any table can be used, but the main table exists
9333 * anyway, so we do not waste resources.
9335 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN,
9340 mlxsw_sp->router->lb_rif_index = lb_rif_index;
9345 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
9347 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index);
9350 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
9351 struct netlink_ext_ack *extack)
9353 struct mlxsw_sp_router *router;
9356 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
9359 mutex_init(&router->lock);
9360 mlxsw_sp->router = router;
9361 router->mlxsw_sp = mlxsw_sp;
9363 err = mlxsw_sp_router_xm_init(mlxsw_sp);
9367 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ?
9368 &mlxsw_sp_router_ll_xm_ops :
9369 &mlxsw_sp_router_ll_basic_ops;
9370 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
9372 err = mlxsw_sp_router_ll_op_ctx_init(router);
9374 goto err_ll_op_ctx_init;
9376 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
9377 err = __mlxsw_sp_router_init(mlxsw_sp);
9379 goto err_router_init;
9381 err = mlxsw_sp_rifs_init(mlxsw_sp);
9385 err = mlxsw_sp_ipips_init(mlxsw_sp);
9387 goto err_ipips_init;
9389 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
9390 &mlxsw_sp_nexthop_ht_params);
9392 goto err_nexthop_ht_init;
9394 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
9395 &mlxsw_sp_nexthop_group_ht_params);
9397 goto err_nexthop_group_ht_init;
9399 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
9400 err = mlxsw_sp_lpm_init(mlxsw_sp);
9404 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
9408 err = mlxsw_sp_vrs_init(mlxsw_sp);
9412 err = mlxsw_sp_lb_rif_init(mlxsw_sp);
9414 goto err_lb_rif_init;
9416 err = mlxsw_sp_neigh_init(mlxsw_sp);
9418 goto err_neigh_init;
9420 err = mlxsw_sp_mp_hash_init(mlxsw_sp);
9422 goto err_mp_hash_init;
9424 err = mlxsw_sp_dscp_init(mlxsw_sp);
9428 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
9429 INIT_LIST_HEAD(&router->fib_event_queue);
9430 spin_lock_init(&router->fib_event_queue_lock);
9432 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
9433 err = register_inetaddr_notifier(&router->inetaddr_nb);
9435 goto err_register_inetaddr_notifier;
9437 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
9438 err = register_inet6addr_notifier(&router->inet6addr_nb);
9440 goto err_register_inet6addr_notifier;
9442 mlxsw_sp->router->netevent_nb.notifier_call =
9443 mlxsw_sp_router_netevent_event;
9444 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
9446 goto err_register_netevent_notifier;
9448 mlxsw_sp->router->nexthop_nb.notifier_call =
9449 mlxsw_sp_nexthop_obj_event;
9450 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
9451 &mlxsw_sp->router->nexthop_nb,
9454 goto err_register_nexthop_notifier;
9456 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
9457 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
9458 &mlxsw_sp->router->fib_nb,
9459 mlxsw_sp_router_fib_dump_flush, extack);
9461 goto err_register_fib_notifier;
9465 err_register_fib_notifier:
9466 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
9467 &mlxsw_sp->router->nexthop_nb);
9468 err_register_nexthop_notifier:
9469 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
9470 err_register_netevent_notifier:
9471 unregister_inet6addr_notifier(&router->inet6addr_nb);
9472 err_register_inet6addr_notifier:
9473 unregister_inetaddr_notifier(&router->inetaddr_nb);
9474 err_register_inetaddr_notifier:
9475 mlxsw_core_flush_owq();
9476 WARN_ON(!list_empty(&router->fib_event_queue));
9479 mlxsw_sp_neigh_fini(mlxsw_sp);
9481 mlxsw_sp_lb_rif_fini(mlxsw_sp);
9483 mlxsw_sp_vrs_fini(mlxsw_sp);
9485 mlxsw_sp_mr_fini(mlxsw_sp);
9487 mlxsw_sp_lpm_fini(mlxsw_sp);
9489 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
9490 err_nexthop_group_ht_init:
9491 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
9492 err_nexthop_ht_init:
9493 mlxsw_sp_ipips_fini(mlxsw_sp);
9495 mlxsw_sp_rifs_fini(mlxsw_sp);
9497 __mlxsw_sp_router_fini(mlxsw_sp);
9499 mlxsw_sp_router_ll_op_ctx_fini(router);
9501 mlxsw_sp_router_xm_fini(mlxsw_sp);
9503 mutex_destroy(&mlxsw_sp->router->lock);
9504 kfree(mlxsw_sp->router);
9508 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
9510 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
9511 &mlxsw_sp->router->fib_nb);
9512 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
9513 &mlxsw_sp->router->nexthop_nb);
9514 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
9515 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
9516 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
9517 mlxsw_core_flush_owq();
9518 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
9519 mlxsw_sp_neigh_fini(mlxsw_sp);
9520 mlxsw_sp_lb_rif_fini(mlxsw_sp);
9521 mlxsw_sp_vrs_fini(mlxsw_sp);
9522 mlxsw_sp_mr_fini(mlxsw_sp);
9523 mlxsw_sp_lpm_fini(mlxsw_sp);
9524 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
9525 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
9526 mlxsw_sp_ipips_fini(mlxsw_sp);
9527 mlxsw_sp_rifs_fini(mlxsw_sp);
9528 __mlxsw_sp_router_fini(mlxsw_sp);
9529 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router);
9530 mlxsw_sp_router_xm_fini(mlxsw_sp);
9531 mutex_destroy(&mlxsw_sp->router->lock);
9532 kfree(mlxsw_sp->router);