1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
4 #include <linux/etherdevice.h>
5 #include <linux/inetdevice.h>
6 #include <net/netevent.h>
8 #include <net/dst_metadata.h>
13 #include "../nfp_net_repr.h"
14 #include "../nfp_net.h"
16 #define NFP_FL_MAX_ROUTES 32
18 #define NFP_TUN_PRE_TUN_RULE_LIMIT 32
19 #define NFP_TUN_PRE_TUN_RULE_DEL BIT(0)
20 #define NFP_TUN_PRE_TUN_IDX_BIT BIT(3)
21 #define NFP_TUN_PRE_TUN_IPV6_BIT BIT(7)
24 * struct nfp_tun_pre_tun_rule - rule matched before decap
25 * @flags: options for the rule offset
26 * @port_idx: index of destination MAC address for the rule
27 * @vlan_tci: VLAN info associated with MAC
28 * @host_ctx_id: stats context of rule to update
30 struct nfp_tun_pre_tun_rule {
38 * struct nfp_tun_active_tuns - periodic message of active tunnels
39 * @seq: sequence number of the message
40 * @count: number of tunnels report in message
41 * @flags: options part of the request
42 * @tun_info.ipv4: dest IPv4 address of active route
43 * @tun_info.egress_port: port the encapsulated packet egressed
44 * @tun_info.extra: reserved for future use
45 * @tun_info: tunnels that have sent traffic in reported period
47 struct nfp_tun_active_tuns {
51 struct route_ip_info {
59 * struct nfp_tun_active_tuns_v6 - periodic message of active IPv6 tunnels
60 * @seq: sequence number of the message
61 * @count: number of tunnels report in message
62 * @flags: options part of the request
63 * @tun_info.ipv6: dest IPv6 address of active route
64 * @tun_info.egress_port: port the encapsulated packet egressed
65 * @tun_info.extra: reserved for future use
66 * @tun_info: tunnels that have sent traffic in reported period
68 struct nfp_tun_active_tuns_v6 {
72 struct route_ip_info_v6 {
80 * struct nfp_tun_req_route_ipv4 - NFP requests a route/neighbour lookup
81 * @ingress_port: ingress port of packet that signalled request
82 * @ipv4_addr: destination ipv4 address for route
83 * @reserved: reserved for future use
85 struct nfp_tun_req_route_ipv4 {
92 * struct nfp_tun_req_route_ipv6 - NFP requests an IPv6 route/neighbour lookup
93 * @ingress_port: ingress port of packet that signalled request
94 * @ipv6_addr: destination ipv6 address for route
96 struct nfp_tun_req_route_ipv6 {
98 struct in6_addr ipv6_addr;
102 * struct nfp_offloaded_route - routes that are offloaded to the NFP
103 * @list: list pointer
104 * @ip_add: destination of route - can be IPv4 or IPv6
106 struct nfp_offloaded_route {
107 struct list_head list;
111 #define NFP_FL_IPV4_ADDRS_MAX 32
114 * struct nfp_tun_ipv4_addr - set the IP address list on the NFP
115 * @count: number of IPs populated in the array
116 * @ipv4_addr: array of IPV4_ADDRS_MAX 32 bit IPv4 addresses
118 struct nfp_tun_ipv4_addr {
120 __be32 ipv4_addr[NFP_FL_IPV4_ADDRS_MAX];
124 * struct nfp_ipv4_addr_entry - cached IPv4 addresses
125 * @ipv4_addr: IP address
126 * @ref_count: number of rules currently using this IP
127 * @list: list pointer
129 struct nfp_ipv4_addr_entry {
132 struct list_head list;
135 #define NFP_FL_IPV6_ADDRS_MAX 4
138 * struct nfp_tun_ipv6_addr - set the IP address list on the NFP
139 * @count: number of IPs populated in the array
140 * @ipv6_addr: array of IPV6_ADDRS_MAX 128 bit IPv6 addresses
142 struct nfp_tun_ipv6_addr {
144 struct in6_addr ipv6_addr[NFP_FL_IPV6_ADDRS_MAX];
147 #define NFP_TUN_MAC_OFFLOAD_DEL_FLAG 0x2
150 * struct nfp_tun_mac_addr_offload - configure MAC address of tunnel EP on NFP
151 * @flags: MAC address offload options
152 * @count: number of MAC addresses in the message (should be 1)
153 * @index: index of MAC address in the lookup table
154 * @addr: interface MAC address
156 struct nfp_tun_mac_addr_offload {
163 enum nfp_flower_mac_offload_cmd {
164 NFP_TUNNEL_MAC_OFFLOAD_ADD = 0,
165 NFP_TUNNEL_MAC_OFFLOAD_DEL = 1,
166 NFP_TUNNEL_MAC_OFFLOAD_MOD = 2,
169 #define NFP_MAX_MAC_INDEX 0xff
172 * struct nfp_tun_offloaded_mac - hashtable entry for an offloaded MAC
173 * @ht_node: Hashtable entry
174 * @addr: Offloaded MAC address
175 * @index: Offloaded index for given MAC address
176 * @ref_count: Number of devs using this MAC address
177 * @repr_list: List of reprs sharing this MAC address
178 * @bridge_count: Number of bridge/internal devs with MAC
180 struct nfp_tun_offloaded_mac {
181 struct rhash_head ht_node;
185 struct list_head repr_list;
189 static const struct rhashtable_params offloaded_macs_params = {
190 .key_offset = offsetof(struct nfp_tun_offloaded_mac, addr),
191 .head_offset = offsetof(struct nfp_tun_offloaded_mac, ht_node),
193 .automatic_shrinking = true,
196 void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb)
198 struct nfp_tun_active_tuns *payload;
199 struct net_device *netdev;
200 int count, i, pay_len;
205 payload = nfp_flower_cmsg_get_data(skb);
206 count = be32_to_cpu(payload->count);
207 if (count > NFP_FL_MAX_ROUTES) {
208 nfp_flower_cmsg_warn(app, "Tunnel keep-alive request exceeds max routes.\n");
212 pay_len = nfp_flower_cmsg_get_data_len(skb);
213 if (pay_len != struct_size(payload, tun_info, count)) {
214 nfp_flower_cmsg_warn(app, "Corruption in tunnel keep-alive message.\n");
219 for (i = 0; i < count; i++) {
220 ipv4_addr = payload->tun_info[i].ipv4;
221 port = be32_to_cpu(payload->tun_info[i].egress_port);
222 netdev = nfp_app_dev_get(app, port, NULL);
226 n = neigh_lookup(&arp_tbl, &ipv4_addr, netdev);
230 /* Update the used timestamp of neighbour */
231 neigh_event_send(n, NULL);
237 void nfp_tunnel_keep_alive_v6(struct nfp_app *app, struct sk_buff *skb)
239 #if IS_ENABLED(CONFIG_IPV6)
240 struct nfp_tun_active_tuns_v6 *payload;
241 struct net_device *netdev;
242 int count, i, pay_len;
247 payload = nfp_flower_cmsg_get_data(skb);
248 count = be32_to_cpu(payload->count);
249 if (count > NFP_FL_IPV6_ADDRS_MAX) {
250 nfp_flower_cmsg_warn(app, "IPv6 tunnel keep-alive request exceeds max routes.\n");
254 pay_len = nfp_flower_cmsg_get_data_len(skb);
255 if (pay_len != struct_size(payload, tun_info, count)) {
256 nfp_flower_cmsg_warn(app, "Corruption in tunnel keep-alive message.\n");
261 for (i = 0; i < count; i++) {
262 ipv6_add = &payload->tun_info[i].ipv6;
263 port = be32_to_cpu(payload->tun_info[i].egress_port);
264 netdev = nfp_app_dev_get(app, port, NULL);
268 n = neigh_lookup(&nd_tbl, ipv6_add, netdev);
272 /* Update the used timestamp of neighbour */
273 neigh_event_send(n, NULL);
281 nfp_flower_xmit_tun_conf(struct nfp_app *app, u8 mtype, u16 plen, void *pdata,
284 struct nfp_flower_priv *priv = app->priv;
288 if (!(priv->flower_ext_feats & NFP_FL_FEATS_DECAP_V2) &&
289 (mtype == NFP_FLOWER_CMSG_TYPE_TUN_NEIGH ||
290 mtype == NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6))
291 plen -= sizeof(struct nfp_tun_neigh_ext);
293 skb = nfp_flower_cmsg_alloc(app, plen, mtype, flag);
297 msg = nfp_flower_cmsg_get_data(skb);
298 memcpy(msg, pdata, nfp_flower_cmsg_get_data_len(skb));
300 nfp_ctrl_tx(app->ctrl, skb);
305 nfp_tun_mutual_link(struct nfp_predt_entry *predt,
306 struct nfp_neigh_entry *neigh)
308 struct nfp_fl_payload *flow_pay = predt->flow_pay;
309 struct nfp_tun_neigh_ext *ext;
310 struct nfp_tun_neigh *common;
312 if (flow_pay->pre_tun_rule.is_ipv6 != neigh->is_ipv6)
315 /* In the case of bonding it is possible that there might already
316 * be a flow linked (as the MAC address gets shared). If a flow
317 * is already linked just return.
322 common = neigh->is_ipv6 ?
323 &((struct nfp_tun_neigh_v6 *)neigh->payload)->common :
324 &((struct nfp_tun_neigh_v4 *)neigh->payload)->common;
325 ext = neigh->is_ipv6 ?
326 &((struct nfp_tun_neigh_v6 *)neigh->payload)->ext :
327 &((struct nfp_tun_neigh_v4 *)neigh->payload)->ext;
329 if (memcmp(flow_pay->pre_tun_rule.loc_mac,
330 common->src_addr, ETH_ALEN) ||
331 memcmp(flow_pay->pre_tun_rule.rem_mac,
332 common->dst_addr, ETH_ALEN))
335 list_add(&neigh->list_head, &predt->nn_list);
337 ext->host_ctx = flow_pay->meta.host_ctx_id;
338 ext->vlan_tci = flow_pay->pre_tun_rule.vlan_tci;
339 ext->vlan_tpid = flow_pay->pre_tun_rule.vlan_tpid;
343 nfp_tun_link_predt_entries(struct nfp_app *app,
344 struct nfp_neigh_entry *nn_entry)
346 struct nfp_flower_priv *priv = app->priv;
347 struct nfp_predt_entry *predt, *tmp;
349 list_for_each_entry_safe(predt, tmp, &priv->predt_list, list_head) {
350 nfp_tun_mutual_link(predt, nn_entry);
354 void nfp_tun_link_and_update_nn_entries(struct nfp_app *app,
355 struct nfp_predt_entry *predt)
357 struct nfp_flower_priv *priv = app->priv;
358 struct nfp_neigh_entry *nn_entry;
359 struct rhashtable_iter iter;
363 rhashtable_walk_enter(&priv->neigh_table, &iter);
364 rhashtable_walk_start(&iter);
365 while ((nn_entry = rhashtable_walk_next(&iter)) != NULL) {
366 if (IS_ERR(nn_entry))
368 nfp_tun_mutual_link(predt, nn_entry);
369 neigh_size = nn_entry->is_ipv6 ?
370 sizeof(struct nfp_tun_neigh_v6) :
371 sizeof(struct nfp_tun_neigh_v4);
372 type = nn_entry->is_ipv6 ? NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6 :
373 NFP_FLOWER_CMSG_TYPE_TUN_NEIGH;
374 nfp_flower_xmit_tun_conf(app, type, neigh_size,
378 rhashtable_walk_stop(&iter);
379 rhashtable_walk_exit(&iter);
382 static void nfp_tun_cleanup_nn_entries(struct nfp_app *app)
384 struct nfp_flower_priv *priv = app->priv;
385 struct nfp_neigh_entry *neigh;
386 struct nfp_tun_neigh_ext *ext;
387 struct rhashtable_iter iter;
391 rhashtable_walk_enter(&priv->neigh_table, &iter);
392 rhashtable_walk_start(&iter);
393 while ((neigh = rhashtable_walk_next(&iter)) != NULL) {
396 ext = neigh->is_ipv6 ?
397 &((struct nfp_tun_neigh_v6 *)neigh->payload)->ext :
398 &((struct nfp_tun_neigh_v4 *)neigh->payload)->ext;
399 ext->host_ctx = cpu_to_be32(U32_MAX);
400 ext->vlan_tpid = cpu_to_be16(U16_MAX);
401 ext->vlan_tci = cpu_to_be16(U16_MAX);
403 neigh_size = neigh->is_ipv6 ?
404 sizeof(struct nfp_tun_neigh_v6) :
405 sizeof(struct nfp_tun_neigh_v4);
406 type = neigh->is_ipv6 ? NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6 :
407 NFP_FLOWER_CMSG_TYPE_TUN_NEIGH;
408 nfp_flower_xmit_tun_conf(app, type, neigh_size, neigh->payload,
411 rhashtable_remove_fast(&priv->neigh_table, &neigh->ht_node,
414 list_del(&neigh->list_head);
417 rhashtable_walk_stop(&iter);
418 rhashtable_walk_exit(&iter);
421 void nfp_tun_unlink_and_update_nn_entries(struct nfp_app *app,
422 struct nfp_predt_entry *predt)
424 struct nfp_neigh_entry *neigh, *tmp;
425 struct nfp_tun_neigh_ext *ext;
429 list_for_each_entry_safe(neigh, tmp, &predt->nn_list, list_head) {
430 ext = neigh->is_ipv6 ?
431 &((struct nfp_tun_neigh_v6 *)neigh->payload)->ext :
432 &((struct nfp_tun_neigh_v4 *)neigh->payload)->ext;
434 ext->host_ctx = cpu_to_be32(U32_MAX);
435 ext->vlan_tpid = cpu_to_be16(U16_MAX);
436 ext->vlan_tci = cpu_to_be16(U16_MAX);
437 list_del(&neigh->list_head);
438 neigh_size = neigh->is_ipv6 ?
439 sizeof(struct nfp_tun_neigh_v6) :
440 sizeof(struct nfp_tun_neigh_v4);
441 type = neigh->is_ipv6 ? NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6 :
442 NFP_FLOWER_CMSG_TYPE_TUN_NEIGH;
443 nfp_flower_xmit_tun_conf(app, type, neigh_size, neigh->payload,
449 nfp_tun_write_neigh(struct net_device *netdev, struct nfp_app *app,
450 void *flow, struct neighbour *neigh, bool is_ipv6)
452 bool neigh_invalid = !(neigh->nud_state & NUD_VALID) || neigh->dead;
453 size_t neigh_size = is_ipv6 ? sizeof(struct nfp_tun_neigh_v6) :
454 sizeof(struct nfp_tun_neigh_v4);
455 unsigned long cookie = (unsigned long)neigh;
456 struct nfp_flower_priv *priv = app->priv;
457 struct nfp_neigh_entry *nn_entry;
461 port_id = nfp_flower_get_port_id_from_netdev(app, netdev);
465 spin_lock_bh(&priv->predt_lock);
466 nn_entry = rhashtable_lookup_fast(&priv->neigh_table, &cookie,
468 if (!nn_entry && !neigh_invalid) {
469 struct nfp_tun_neigh_ext *ext;
470 struct nfp_tun_neigh *common;
472 nn_entry = kzalloc(sizeof(*nn_entry) + neigh_size,
477 nn_entry->payload = (char *)&nn_entry[1];
478 nn_entry->neigh_cookie = cookie;
479 nn_entry->is_ipv6 = is_ipv6;
480 nn_entry->flow = NULL;
482 struct flowi6 *flowi6 = (struct flowi6 *)flow;
483 struct nfp_tun_neigh_v6 *payload;
485 payload = (struct nfp_tun_neigh_v6 *)nn_entry->payload;
486 payload->src_ipv6 = flowi6->saddr;
487 payload->dst_ipv6 = flowi6->daddr;
488 common = &payload->common;
490 mtype = NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6;
492 struct flowi4 *flowi4 = (struct flowi4 *)flow;
493 struct nfp_tun_neigh_v4 *payload;
495 payload = (struct nfp_tun_neigh_v4 *)nn_entry->payload;
496 payload->src_ipv4 = flowi4->saddr;
497 payload->dst_ipv4 = flowi4->daddr;
498 common = &payload->common;
500 mtype = NFP_FLOWER_CMSG_TYPE_TUN_NEIGH;
502 ext->host_ctx = cpu_to_be32(U32_MAX);
503 ext->vlan_tpid = cpu_to_be16(U16_MAX);
504 ext->vlan_tci = cpu_to_be16(U16_MAX);
505 ether_addr_copy(common->src_addr, netdev->dev_addr);
506 neigh_ha_snapshot(common->dst_addr, neigh, netdev);
507 common->port_id = cpu_to_be32(port_id);
509 if (rhashtable_insert_fast(&priv->neigh_table,
514 nfp_tun_link_predt_entries(app, nn_entry);
515 nfp_flower_xmit_tun_conf(app, mtype, neigh_size,
518 } else if (nn_entry && neigh_invalid) {
520 struct flowi6 *flowi6 = (struct flowi6 *)flow;
521 struct nfp_tun_neigh_v6 *payload;
523 payload = (struct nfp_tun_neigh_v6 *)nn_entry->payload;
524 memset(payload, 0, sizeof(struct nfp_tun_neigh_v6));
525 payload->dst_ipv6 = flowi6->daddr;
526 mtype = NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6;
528 struct flowi4 *flowi4 = (struct flowi4 *)flow;
529 struct nfp_tun_neigh_v4 *payload;
531 payload = (struct nfp_tun_neigh_v4 *)nn_entry->payload;
532 memset(payload, 0, sizeof(struct nfp_tun_neigh_v4));
533 payload->dst_ipv4 = flowi4->daddr;
534 mtype = NFP_FLOWER_CMSG_TYPE_TUN_NEIGH;
536 /* Trigger ARP to verify invalid neighbour state. */
537 neigh_event_send(neigh, NULL);
538 rhashtable_remove_fast(&priv->neigh_table,
542 nfp_flower_xmit_tun_conf(app, mtype, neigh_size,
547 list_del(&nn_entry->list_head);
551 spin_unlock_bh(&priv->predt_lock);
556 spin_unlock_bh(&priv->predt_lock);
557 nfp_flower_cmsg_warn(app, "Neighbour configuration failed.\n");
561 nfp_tun_neigh_event_handler(struct notifier_block *nb, unsigned long event,
564 struct nfp_flower_priv *app_priv;
565 struct netevent_redirect *redir;
572 case NETEVENT_REDIRECT:
573 redir = (struct netevent_redirect *)ptr;
576 case NETEVENT_NEIGH_UPDATE:
577 n = (struct neighbour *)ptr;
583 neigh_invalid = !(n->nud_state & NUD_VALID) || n->dead;
585 app_priv = container_of(nb, struct nfp_flower_priv, tun.neigh_nb);
588 if (!nfp_netdev_is_nfp_repr(n->dev) &&
589 !nfp_flower_internal_port_can_offload(app, n->dev))
592 #if IS_ENABLED(CONFIG_INET)
593 if (n->tbl->family == AF_INET6) {
594 #if IS_ENABLED(CONFIG_IPV6)
595 struct flowi6 flow6 = {};
597 flow6.daddr = *(struct in6_addr *)n->primary_key;
598 if (!neigh_invalid) {
599 struct dst_entry *dst;
600 /* Use ipv6_dst_lookup_flow to populate flow6->saddr
601 * and other fields. This information is only needed
602 * for new entries, lookup can be skipped when an entry
603 * gets invalidated - as only the daddr is needed for
606 dst = ip6_dst_lookup_flow(dev_net(n->dev), NULL,
613 nfp_tun_write_neigh(n->dev, app, &flow6, n, true);
616 #endif /* CONFIG_IPV6 */
618 struct flowi4 flow4 = {};
620 flow4.daddr = *(__be32 *)n->primary_key;
621 if (!neigh_invalid) {
623 /* Use ip_route_output_key to populate flow4->saddr and
624 * other fields. This information is only needed for
625 * new entries, lookup can be skipped when an entry
626 * gets invalidated - as only the daddr is needed for
629 rt = ip_route_output_key(dev_net(n->dev), &flow4);
630 err = PTR_ERR_OR_ZERO(rt);
636 nfp_tun_write_neigh(n->dev, app, &flow4, n, false);
640 #endif /* CONFIG_INET */
645 void nfp_tunnel_request_route_v4(struct nfp_app *app, struct sk_buff *skb)
647 struct nfp_tun_req_route_ipv4 *payload;
648 struct net_device *netdev;
649 struct flowi4 flow = {};
654 payload = nfp_flower_cmsg_get_data(skb);
657 netdev = nfp_app_dev_get(app, be32_to_cpu(payload->ingress_port), NULL);
659 goto fail_rcu_unlock;
661 flow.daddr = payload->ipv4_addr;
662 flow.flowi4_proto = IPPROTO_UDP;
664 #if IS_ENABLED(CONFIG_INET)
665 /* Do a route lookup on same namespace as ingress port. */
666 rt = ip_route_output_key(dev_net(netdev), &flow);
667 err = PTR_ERR_OR_ZERO(rt);
669 goto fail_rcu_unlock;
671 goto fail_rcu_unlock;
674 /* Get the neighbour entry for the lookup */
675 n = dst_neigh_lookup(&rt->dst, &flow.daddr);
678 goto fail_rcu_unlock;
679 nfp_tun_write_neigh(n->dev, app, &flow, n, false);
686 nfp_flower_cmsg_warn(app, "Requested route not found.\n");
689 void nfp_tunnel_request_route_v6(struct nfp_app *app, struct sk_buff *skb)
691 struct nfp_tun_req_route_ipv6 *payload;
692 struct net_device *netdev;
693 struct flowi6 flow = {};
694 struct dst_entry *dst;
697 payload = nfp_flower_cmsg_get_data(skb);
700 netdev = nfp_app_dev_get(app, be32_to_cpu(payload->ingress_port), NULL);
702 goto fail_rcu_unlock;
704 flow.daddr = payload->ipv6_addr;
705 flow.flowi6_proto = IPPROTO_UDP;
707 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
708 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(netdev), NULL, &flow,
711 goto fail_rcu_unlock;
713 goto fail_rcu_unlock;
716 n = dst_neigh_lookup(dst, &flow.daddr);
719 goto fail_rcu_unlock;
721 nfp_tun_write_neigh(n->dev, app, &flow, n, true);
728 nfp_flower_cmsg_warn(app, "Requested IPv6 route not found.\n");
731 static void nfp_tun_write_ipv4_list(struct nfp_app *app)
733 struct nfp_flower_priv *priv = app->priv;
734 struct nfp_ipv4_addr_entry *entry;
735 struct nfp_tun_ipv4_addr payload;
736 struct list_head *ptr, *storage;
739 memset(&payload, 0, sizeof(struct nfp_tun_ipv4_addr));
740 mutex_lock(&priv->tun.ipv4_off_lock);
742 list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
743 if (count >= NFP_FL_IPV4_ADDRS_MAX) {
744 mutex_unlock(&priv->tun.ipv4_off_lock);
745 nfp_flower_cmsg_warn(app, "IPv4 offload exceeds limit.\n");
748 entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
749 payload.ipv4_addr[count++] = entry->ipv4_addr;
751 payload.count = cpu_to_be32(count);
752 mutex_unlock(&priv->tun.ipv4_off_lock);
754 nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_IPS,
755 sizeof(struct nfp_tun_ipv4_addr),
756 &payload, GFP_KERNEL);
759 void nfp_tunnel_add_ipv4_off(struct nfp_app *app, __be32 ipv4)
761 struct nfp_flower_priv *priv = app->priv;
762 struct nfp_ipv4_addr_entry *entry;
763 struct list_head *ptr, *storage;
765 mutex_lock(&priv->tun.ipv4_off_lock);
766 list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
767 entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
768 if (entry->ipv4_addr == ipv4) {
770 mutex_unlock(&priv->tun.ipv4_off_lock);
775 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
777 mutex_unlock(&priv->tun.ipv4_off_lock);
778 nfp_flower_cmsg_warn(app, "Mem error when offloading IP address.\n");
781 entry->ipv4_addr = ipv4;
782 entry->ref_count = 1;
783 list_add_tail(&entry->list, &priv->tun.ipv4_off_list);
784 mutex_unlock(&priv->tun.ipv4_off_lock);
786 nfp_tun_write_ipv4_list(app);
789 void nfp_tunnel_del_ipv4_off(struct nfp_app *app, __be32 ipv4)
791 struct nfp_flower_priv *priv = app->priv;
792 struct nfp_ipv4_addr_entry *entry;
793 struct list_head *ptr, *storage;
795 mutex_lock(&priv->tun.ipv4_off_lock);
796 list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
797 entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
798 if (entry->ipv4_addr == ipv4) {
800 if (!entry->ref_count) {
801 list_del(&entry->list);
807 mutex_unlock(&priv->tun.ipv4_off_lock);
809 nfp_tun_write_ipv4_list(app);
812 static void nfp_tun_write_ipv6_list(struct nfp_app *app)
814 struct nfp_flower_priv *priv = app->priv;
815 struct nfp_ipv6_addr_entry *entry;
816 struct nfp_tun_ipv6_addr payload;
819 memset(&payload, 0, sizeof(struct nfp_tun_ipv6_addr));
820 mutex_lock(&priv->tun.ipv6_off_lock);
821 list_for_each_entry(entry, &priv->tun.ipv6_off_list, list) {
822 if (count >= NFP_FL_IPV6_ADDRS_MAX) {
823 nfp_flower_cmsg_warn(app, "Too many IPv6 tunnel endpoint addresses, some cannot be offloaded.\n");
826 payload.ipv6_addr[count++] = entry->ipv6_addr;
828 mutex_unlock(&priv->tun.ipv6_off_lock);
829 payload.count = cpu_to_be32(count);
831 nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_IPS_V6,
832 sizeof(struct nfp_tun_ipv6_addr),
833 &payload, GFP_KERNEL);
836 struct nfp_ipv6_addr_entry *
837 nfp_tunnel_add_ipv6_off(struct nfp_app *app, struct in6_addr *ipv6)
839 struct nfp_flower_priv *priv = app->priv;
840 struct nfp_ipv6_addr_entry *entry;
842 mutex_lock(&priv->tun.ipv6_off_lock);
843 list_for_each_entry(entry, &priv->tun.ipv6_off_list, list)
844 if (!memcmp(&entry->ipv6_addr, ipv6, sizeof(*ipv6))) {
846 mutex_unlock(&priv->tun.ipv6_off_lock);
850 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
852 mutex_unlock(&priv->tun.ipv6_off_lock);
853 nfp_flower_cmsg_warn(app, "Mem error when offloading IP address.\n");
856 entry->ipv6_addr = *ipv6;
857 entry->ref_count = 1;
858 list_add_tail(&entry->list, &priv->tun.ipv6_off_list);
859 mutex_unlock(&priv->tun.ipv6_off_lock);
861 nfp_tun_write_ipv6_list(app);
867 nfp_tunnel_put_ipv6_off(struct nfp_app *app, struct nfp_ipv6_addr_entry *entry)
869 struct nfp_flower_priv *priv = app->priv;
872 mutex_lock(&priv->tun.ipv6_off_lock);
873 if (!--entry->ref_count) {
874 list_del(&entry->list);
878 mutex_unlock(&priv->tun.ipv6_off_lock);
881 nfp_tun_write_ipv6_list(app);
885 __nfp_tunnel_offload_mac(struct nfp_app *app, const u8 *mac, u16 idx, bool del)
887 struct nfp_tun_mac_addr_offload payload;
889 memset(&payload, 0, sizeof(payload));
892 payload.flags = cpu_to_be16(NFP_TUN_MAC_OFFLOAD_DEL_FLAG);
894 /* FW supports multiple MACs per cmsg but restrict to single. */
895 payload.count = cpu_to_be16(1);
896 payload.index = cpu_to_be16(idx);
897 ether_addr_copy(payload.addr, mac);
899 return nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_MAC,
900 sizeof(struct nfp_tun_mac_addr_offload),
901 &payload, GFP_KERNEL);
904 static bool nfp_tunnel_port_is_phy_repr(int port)
906 if (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port) ==
907 NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT)
913 static u16 nfp_tunnel_get_mac_idx_from_phy_port_id(int port)
915 return port << 8 | NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT;
918 static u16 nfp_tunnel_get_global_mac_idx_from_ida(int id)
920 return id << 8 | NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT;
923 static int nfp_tunnel_get_ida_from_global_mac_idx(u16 nfp_mac_idx)
925 return nfp_mac_idx >> 8;
928 static bool nfp_tunnel_is_mac_idx_global(u16 nfp_mac_idx)
930 return (nfp_mac_idx & 0xff) == NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT;
933 static struct nfp_tun_offloaded_mac *
934 nfp_tunnel_lookup_offloaded_macs(struct nfp_app *app, const u8 *mac)
936 struct nfp_flower_priv *priv = app->priv;
938 return rhashtable_lookup_fast(&priv->tun.offloaded_macs, mac,
939 offloaded_macs_params);
943 nfp_tunnel_offloaded_macs_inc_ref_and_link(struct nfp_tun_offloaded_mac *entry,
944 struct net_device *netdev, bool mod)
946 if (nfp_netdev_is_nfp_repr(netdev)) {
947 struct nfp_flower_repr_priv *repr_priv;
948 struct nfp_repr *repr;
950 repr = netdev_priv(netdev);
951 repr_priv = repr->app_priv;
953 /* If modifing MAC, remove repr from old list first. */
955 list_del(&repr_priv->mac_list);
957 list_add_tail(&repr_priv->mac_list, &entry->repr_list);
958 } else if (nfp_flower_is_supported_bridge(netdev)) {
959 entry->bridge_count++;
966 nfp_tunnel_add_shared_mac(struct nfp_app *app, struct net_device *netdev,
969 struct nfp_flower_priv *priv = app->priv;
970 struct nfp_tun_offloaded_mac *entry;
971 int ida_idx = -1, err;
974 entry = nfp_tunnel_lookup_offloaded_macs(app, netdev->dev_addr);
975 if (entry && nfp_tunnel_is_mac_idx_global(entry->index)) {
976 if (entry->bridge_count ||
977 !nfp_flower_is_supported_bridge(netdev)) {
978 nfp_tunnel_offloaded_macs_inc_ref_and_link(entry,
983 /* MAC is global but matches need to go to pre_tun table. */
984 nfp_mac_idx = entry->index | NFP_TUN_PRE_TUN_IDX_BIT;
988 /* Assign a global index if non-repr or MAC is now shared. */
989 if (entry || !port) {
990 ida_idx = ida_alloc_max(&priv->tun.mac_off_ids,
991 NFP_MAX_MAC_INDEX, GFP_KERNEL);
996 nfp_tunnel_get_global_mac_idx_from_ida(ida_idx);
998 if (nfp_flower_is_supported_bridge(netdev))
999 nfp_mac_idx |= NFP_TUN_PRE_TUN_IDX_BIT;
1003 nfp_tunnel_get_mac_idx_from_phy_port_id(port);
1008 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1014 ether_addr_copy(entry->addr, netdev->dev_addr);
1015 INIT_LIST_HEAD(&entry->repr_list);
1017 if (rhashtable_insert_fast(&priv->tun.offloaded_macs,
1019 offloaded_macs_params)) {
1021 goto err_free_entry;
1025 err = __nfp_tunnel_offload_mac(app, netdev->dev_addr,
1026 nfp_mac_idx, false);
1028 /* If not shared then free. */
1029 if (!entry->ref_count)
1030 goto err_remove_hash;
1034 entry->index = nfp_mac_idx;
1035 nfp_tunnel_offloaded_macs_inc_ref_and_link(entry, netdev, mod);
1040 rhashtable_remove_fast(&priv->tun.offloaded_macs, &entry->ht_node,
1041 offloaded_macs_params);
1046 ida_free(&priv->tun.mac_off_ids, ida_idx);
1052 nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
1053 const u8 *mac, bool mod)
1055 struct nfp_flower_priv *priv = app->priv;
1056 struct nfp_flower_repr_priv *repr_priv;
1057 struct nfp_tun_offloaded_mac *entry;
1058 struct nfp_repr *repr;
1062 entry = nfp_tunnel_lookup_offloaded_macs(app, mac);
1067 /* If del is part of a mod then mac_list is still in use elsewheree. */
1068 if (nfp_netdev_is_nfp_repr(netdev) && !mod) {
1069 repr = netdev_priv(netdev);
1070 repr_priv = repr->app_priv;
1071 list_del(&repr_priv->mac_list);
1074 if (nfp_flower_is_supported_bridge(netdev)) {
1075 entry->bridge_count--;
1077 if (!entry->bridge_count && entry->ref_count) {
1078 nfp_mac_idx = entry->index & ~NFP_TUN_PRE_TUN_IDX_BIT;
1079 if (__nfp_tunnel_offload_mac(app, mac, nfp_mac_idx,
1081 nfp_flower_cmsg_warn(app, "MAC offload index revert failed on %s.\n",
1082 netdev_name(netdev));
1086 entry->index = nfp_mac_idx;
1091 /* If MAC is now used by 1 repr set the offloaded MAC index to port. */
1092 if (entry->ref_count == 1 && list_is_singular(&entry->repr_list)) {
1095 repr_priv = list_first_entry(&entry->repr_list,
1096 struct nfp_flower_repr_priv,
1098 repr = repr_priv->nfp_repr;
1099 port = nfp_repr_get_port_id(repr->netdev);
1100 nfp_mac_idx = nfp_tunnel_get_mac_idx_from_phy_port_id(port);
1101 err = __nfp_tunnel_offload_mac(app, mac, nfp_mac_idx, false);
1103 nfp_flower_cmsg_warn(app, "MAC offload index revert failed on %s.\n",
1104 netdev_name(netdev));
1108 ida_idx = nfp_tunnel_get_ida_from_global_mac_idx(entry->index);
1109 ida_free(&priv->tun.mac_off_ids, ida_idx);
1110 entry->index = nfp_mac_idx;
1114 if (entry->ref_count)
1117 WARN_ON_ONCE(rhashtable_remove_fast(&priv->tun.offloaded_macs,
1119 offloaded_macs_params));
1121 if (nfp_flower_is_supported_bridge(netdev))
1122 nfp_mac_idx = entry->index & ~NFP_TUN_PRE_TUN_IDX_BIT;
1124 nfp_mac_idx = entry->index;
1126 /* If MAC has global ID then extract and free the ida entry. */
1127 if (nfp_tunnel_is_mac_idx_global(nfp_mac_idx)) {
1128 ida_idx = nfp_tunnel_get_ida_from_global_mac_idx(entry->index);
1129 ida_free(&priv->tun.mac_off_ids, ida_idx);
1134 return __nfp_tunnel_offload_mac(app, mac, 0, true);
1138 nfp_tunnel_offload_mac(struct nfp_app *app, struct net_device *netdev,
1139 enum nfp_flower_mac_offload_cmd cmd)
1141 struct nfp_flower_non_repr_priv *nr_priv = NULL;
1142 bool non_repr = false, *mac_offloaded;
1146 if (nfp_netdev_is_nfp_repr(netdev)) {
1147 struct nfp_flower_repr_priv *repr_priv;
1148 struct nfp_repr *repr;
1150 repr = netdev_priv(netdev);
1151 if (repr->app != app)
1154 repr_priv = repr->app_priv;
1155 if (repr_priv->on_bridge)
1158 mac_offloaded = &repr_priv->mac_offloaded;
1159 off_mac = &repr_priv->offloaded_mac_addr[0];
1160 port = nfp_repr_get_port_id(netdev);
1161 if (!nfp_tunnel_port_is_phy_repr(port))
1163 } else if (nfp_fl_is_netdev_to_offload(netdev)) {
1164 nr_priv = nfp_flower_non_repr_priv_get(app, netdev);
1168 mac_offloaded = &nr_priv->mac_offloaded;
1169 off_mac = &nr_priv->offloaded_mac_addr[0];
1175 if (!is_valid_ether_addr(netdev->dev_addr)) {
1177 goto err_put_non_repr_priv;
1180 if (cmd == NFP_TUNNEL_MAC_OFFLOAD_MOD && !*mac_offloaded)
1181 cmd = NFP_TUNNEL_MAC_OFFLOAD_ADD;
1184 case NFP_TUNNEL_MAC_OFFLOAD_ADD:
1185 err = nfp_tunnel_add_shared_mac(app, netdev, port, false);
1187 goto err_put_non_repr_priv;
1190 __nfp_flower_non_repr_priv_get(nr_priv);
1192 *mac_offloaded = true;
1193 ether_addr_copy(off_mac, netdev->dev_addr);
1195 case NFP_TUNNEL_MAC_OFFLOAD_DEL:
1196 /* Only attempt delete if add was successful. */
1197 if (!*mac_offloaded)
1201 __nfp_flower_non_repr_priv_put(nr_priv);
1203 *mac_offloaded = false;
1205 err = nfp_tunnel_del_shared_mac(app, netdev, netdev->dev_addr,
1208 goto err_put_non_repr_priv;
1211 case NFP_TUNNEL_MAC_OFFLOAD_MOD:
1212 /* Ignore if changing to the same address. */
1213 if (ether_addr_equal(netdev->dev_addr, off_mac))
1216 err = nfp_tunnel_add_shared_mac(app, netdev, port, true);
1218 goto err_put_non_repr_priv;
1220 /* Delete the previous MAC address. */
1221 err = nfp_tunnel_del_shared_mac(app, netdev, off_mac, true);
1223 nfp_flower_cmsg_warn(app, "Failed to remove offload of replaced MAC addr on %s.\n",
1224 netdev_name(netdev));
1226 ether_addr_copy(off_mac, netdev->dev_addr);
1230 goto err_put_non_repr_priv;
1234 __nfp_flower_non_repr_priv_put(nr_priv);
1238 err_put_non_repr_priv:
1240 __nfp_flower_non_repr_priv_put(nr_priv);
1245 int nfp_tunnel_mac_event_handler(struct nfp_app *app,
1246 struct net_device *netdev,
1247 unsigned long event, void *ptr)
1251 if (event == NETDEV_DOWN) {
1252 err = nfp_tunnel_offload_mac(app, netdev,
1253 NFP_TUNNEL_MAC_OFFLOAD_DEL);
1255 nfp_flower_cmsg_warn(app, "Failed to delete offload MAC on %s.\n",
1256 netdev_name(netdev));
1257 } else if (event == NETDEV_UP) {
1258 err = nfp_tunnel_offload_mac(app, netdev,
1259 NFP_TUNNEL_MAC_OFFLOAD_ADD);
1261 nfp_flower_cmsg_warn(app, "Failed to offload MAC on %s.\n",
1262 netdev_name(netdev));
1263 } else if (event == NETDEV_CHANGEADDR) {
1264 /* Only offload addr change if netdev is already up. */
1265 if (!(netdev->flags & IFF_UP))
1268 err = nfp_tunnel_offload_mac(app, netdev,
1269 NFP_TUNNEL_MAC_OFFLOAD_MOD);
1271 nfp_flower_cmsg_warn(app, "Failed to offload MAC change on %s.\n",
1272 netdev_name(netdev));
1273 } else if (event == NETDEV_CHANGEUPPER) {
1274 /* If a repr is attached to a bridge then tunnel packets
1275 * entering the physical port are directed through the bridge
1276 * datapath and cannot be directly detunneled. Therefore,
1277 * associated offloaded MACs and indexes should not be used
1278 * by fw for detunneling.
1280 struct netdev_notifier_changeupper_info *info = ptr;
1281 struct net_device *upper = info->upper_dev;
1282 struct nfp_flower_repr_priv *repr_priv;
1283 struct nfp_repr *repr;
1285 if (!nfp_netdev_is_nfp_repr(netdev) ||
1286 !nfp_flower_is_supported_bridge(upper))
1289 repr = netdev_priv(netdev);
1290 if (repr->app != app)
1293 repr_priv = repr->app_priv;
1295 if (info->linking) {
1296 if (nfp_tunnel_offload_mac(app, netdev,
1297 NFP_TUNNEL_MAC_OFFLOAD_DEL))
1298 nfp_flower_cmsg_warn(app, "Failed to delete offloaded MAC on %s.\n",
1299 netdev_name(netdev));
1300 repr_priv->on_bridge = true;
1302 repr_priv->on_bridge = false;
1304 if (!(netdev->flags & IFF_UP))
1307 if (nfp_tunnel_offload_mac(app, netdev,
1308 NFP_TUNNEL_MAC_OFFLOAD_ADD))
1309 nfp_flower_cmsg_warn(app, "Failed to offload MAC on %s.\n",
1310 netdev_name(netdev));
1316 int nfp_flower_xmit_pre_tun_flow(struct nfp_app *app,
1317 struct nfp_fl_payload *flow)
1319 struct nfp_flower_priv *app_priv = app->priv;
1320 struct nfp_tun_offloaded_mac *mac_entry;
1321 struct nfp_flower_meta_tci *key_meta;
1322 struct nfp_tun_pre_tun_rule payload;
1323 struct net_device *internal_dev;
1326 if (app_priv->pre_tun_rule_cnt == NFP_TUN_PRE_TUN_RULE_LIMIT)
1329 memset(&payload, 0, sizeof(struct nfp_tun_pre_tun_rule));
1331 internal_dev = flow->pre_tun_rule.dev;
1332 payload.vlan_tci = flow->pre_tun_rule.vlan_tci;
1333 payload.host_ctx_id = flow->meta.host_ctx_id;
1335 /* Lookup MAC index for the pre-tunnel rule egress device.
1336 * Note that because the device is always an internal port, it will
1337 * have a constant global index so does not need to be tracked.
1339 mac_entry = nfp_tunnel_lookup_offloaded_macs(app,
1340 internal_dev->dev_addr);
1344 /* Set/clear IPV6 bit. cpu_to_be16() swap will lead to MSB being
1345 * set/clear for port_idx.
1347 key_meta = (struct nfp_flower_meta_tci *)flow->unmasked_data;
1348 if (key_meta->nfp_flow_key_layer & NFP_FLOWER_LAYER_IPV6)
1349 mac_entry->index |= NFP_TUN_PRE_TUN_IPV6_BIT;
1351 mac_entry->index &= ~NFP_TUN_PRE_TUN_IPV6_BIT;
1353 payload.port_idx = cpu_to_be16(mac_entry->index);
1355 /* Copy mac id and vlan to flow - dev may not exist at delete time. */
1356 flow->pre_tun_rule.vlan_tci = payload.vlan_tci;
1357 flow->pre_tun_rule.port_idx = payload.port_idx;
1359 err = nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_PRE_TUN_RULE,
1360 sizeof(struct nfp_tun_pre_tun_rule),
1361 (unsigned char *)&payload, GFP_KERNEL);
1365 app_priv->pre_tun_rule_cnt++;
1370 int nfp_flower_xmit_pre_tun_del_flow(struct nfp_app *app,
1371 struct nfp_fl_payload *flow)
1373 struct nfp_flower_priv *app_priv = app->priv;
1374 struct nfp_tun_pre_tun_rule payload;
1378 memset(&payload, 0, sizeof(struct nfp_tun_pre_tun_rule));
1380 tmp_flags |= NFP_TUN_PRE_TUN_RULE_DEL;
1381 payload.flags = cpu_to_be32(tmp_flags);
1382 payload.vlan_tci = flow->pre_tun_rule.vlan_tci;
1383 payload.port_idx = flow->pre_tun_rule.port_idx;
1385 err = nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_PRE_TUN_RULE,
1386 sizeof(struct nfp_tun_pre_tun_rule),
1387 (unsigned char *)&payload, GFP_KERNEL);
1391 app_priv->pre_tun_rule_cnt--;
1396 int nfp_tunnel_config_start(struct nfp_app *app)
1398 struct nfp_flower_priv *priv = app->priv;
1401 /* Initialise rhash for MAC offload tracking. */
1402 err = rhashtable_init(&priv->tun.offloaded_macs,
1403 &offloaded_macs_params);
1407 ida_init(&priv->tun.mac_off_ids);
1409 /* Initialise priv data for IPv4/v6 offloading. */
1410 mutex_init(&priv->tun.ipv4_off_lock);
1411 INIT_LIST_HEAD(&priv->tun.ipv4_off_list);
1412 mutex_init(&priv->tun.ipv6_off_lock);
1413 INIT_LIST_HEAD(&priv->tun.ipv6_off_list);
1415 /* Initialise priv data for neighbour offloading. */
1416 priv->tun.neigh_nb.notifier_call = nfp_tun_neigh_event_handler;
1418 err = register_netevent_notifier(&priv->tun.neigh_nb);
1420 rhashtable_free_and_destroy(&priv->tun.offloaded_macs,
1421 nfp_check_rhashtable_empty, NULL);
1428 void nfp_tunnel_config_stop(struct nfp_app *app)
1430 struct nfp_flower_priv *priv = app->priv;
1431 struct nfp_ipv4_addr_entry *ip_entry;
1432 struct list_head *ptr, *storage;
1434 unregister_netevent_notifier(&priv->tun.neigh_nb);
1436 ida_destroy(&priv->tun.mac_off_ids);
1438 /* Free any memory that may be occupied by ipv4 list. */
1439 list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
1440 ip_entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
1441 list_del(&ip_entry->list);
1445 mutex_destroy(&priv->tun.ipv6_off_lock);
1447 /* Destroy rhash. Entries should be cleaned on netdev notifier unreg. */
1448 rhashtable_free_and_destroy(&priv->tun.offloaded_macs,
1449 nfp_check_rhashtable_empty, NULL);
1451 nfp_tun_cleanup_nn_entries(app);