1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2023, Advanced Micro Devices, Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include "tc_conntrack.h"
15 static int efx_tc_flow_block(enum tc_setup_type type, void *type_data,
18 static const struct rhashtable_params efx_tc_ct_zone_ht_params = {
19 .key_len = offsetof(struct efx_tc_ct_zone, linkage),
21 .head_offset = offsetof(struct efx_tc_ct_zone, linkage),
24 static const struct rhashtable_params efx_tc_ct_ht_params = {
25 .key_len = offsetof(struct efx_tc_ct_entry, linkage),
27 .head_offset = offsetof(struct efx_tc_ct_entry, linkage),
30 static void efx_tc_ct_zone_free(void *ptr, void *arg)
32 struct efx_tc_ct_zone *zone = ptr;
33 struct efx_nic *efx = zone->efx;
35 netif_err(efx, drv, efx->net_dev,
36 "tc ct_zone %u still present at teardown, removing\n",
39 nf_flow_table_offload_del_cb(zone->nf_ft, efx_tc_flow_block, zone);
43 static void efx_tc_ct_free(void *ptr, void *arg)
45 struct efx_tc_ct_entry *conn = ptr;
46 struct efx_nic *efx = arg;
48 netif_err(efx, drv, efx->net_dev,
49 "tc ct_entry %lx still present at teardown\n",
52 /* We can release the counter, but we can't remove the CT itself
53 * from hardware because the table meta is already gone.
55 efx_tc_flower_release_counter(efx, conn->cnt);
59 int efx_tc_init_conntrack(struct efx_nic *efx)
63 rc = rhashtable_init(&efx->tc->ct_zone_ht, &efx_tc_ct_zone_ht_params);
66 rc = rhashtable_init(&efx->tc->ct_ht, &efx_tc_ct_ht_params);
71 rhashtable_destroy(&efx->tc->ct_zone_ht);
76 /* Only call this in init failure teardown.
77 * Normal exit should fini instead as there may be entries in the table.
79 void efx_tc_destroy_conntrack(struct efx_nic *efx)
81 rhashtable_destroy(&efx->tc->ct_ht);
82 rhashtable_destroy(&efx->tc->ct_zone_ht);
85 void efx_tc_fini_conntrack(struct efx_nic *efx)
87 rhashtable_free_and_destroy(&efx->tc->ct_zone_ht, efx_tc_ct_zone_free, NULL);
88 rhashtable_free_and_destroy(&efx->tc->ct_ht, efx_tc_ct_free, efx);
91 #define EFX_NF_TCP_FLAG(flg) cpu_to_be16(be32_to_cpu(TCP_FLAG_##flg) >> 16)
93 static int efx_tc_ct_parse_match(struct efx_nic *efx, struct flow_rule *fr,
94 struct efx_tc_ct_entry *conn)
96 struct flow_dissector *dissector = fr->match.dissector;
97 unsigned char ipv = 0;
100 if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_CONTROL)) {
101 struct flow_match_control fm;
103 flow_rule_match_control(fr, &fm);
104 if (IS_ALL_ONES(fm.mask->addr_type))
105 switch (fm.key->addr_type) {
106 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
109 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
118 netif_dbg(efx, drv, efx->net_dev,
119 "Conntrack missing ipv specification\n");
123 if (dissector->used_keys &
124 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
125 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
126 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
127 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
128 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
129 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
130 BIT_ULL(FLOW_DISSECTOR_KEY_META))) {
131 netif_dbg(efx, drv, efx->net_dev,
132 "Unsupported conntrack keys %#llx\n",
133 dissector->used_keys);
137 if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_BASIC)) {
138 struct flow_match_basic fm;
140 flow_rule_match_basic(fr, &fm);
141 if (!IS_ALL_ONES(fm.mask->n_proto)) {
142 netif_dbg(efx, drv, efx->net_dev,
143 "Conntrack eth_proto is not exact-match; mask %04x\n",
144 ntohs(fm.mask->n_proto));
147 conn->eth_proto = fm.key->n_proto;
148 if (conn->eth_proto != (ipv == 4 ? htons(ETH_P_IP)
149 : htons(ETH_P_IPV6))) {
150 netif_dbg(efx, drv, efx->net_dev,
151 "Conntrack eth_proto is not IPv%u, is %04x\n",
152 ipv, ntohs(conn->eth_proto));
155 if (!IS_ALL_ONES(fm.mask->ip_proto)) {
156 netif_dbg(efx, drv, efx->net_dev,
157 "Conntrack ip_proto is not exact-match; mask %02x\n",
161 conn->ip_proto = fm.key->ip_proto;
162 switch (conn->ip_proto) {
169 netif_dbg(efx, drv, efx->net_dev,
170 "Conntrack ip_proto not TCP or UDP, is %02x\n",
175 netif_dbg(efx, drv, efx->net_dev,
176 "Conntrack missing eth_proto, ip_proto\n");
180 if (ipv == 4 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
181 struct flow_match_ipv4_addrs fm;
183 flow_rule_match_ipv4_addrs(fr, &fm);
184 if (!IS_ALL_ONES(fm.mask->src)) {
185 netif_dbg(efx, drv, efx->net_dev,
186 "Conntrack ipv4.src is not exact-match; mask %08x\n",
187 ntohl(fm.mask->src));
190 conn->src_ip = fm.key->src;
191 if (!IS_ALL_ONES(fm.mask->dst)) {
192 netif_dbg(efx, drv, efx->net_dev,
193 "Conntrack ipv4.dst is not exact-match; mask %08x\n",
194 ntohl(fm.mask->dst));
197 conn->dst_ip = fm.key->dst;
198 } else if (ipv == 6 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
199 struct flow_match_ipv6_addrs fm;
201 flow_rule_match_ipv6_addrs(fr, &fm);
202 if (!efx_ipv6_addr_all_ones(&fm.mask->src)) {
203 netif_dbg(efx, drv, efx->net_dev,
204 "Conntrack ipv6.src is not exact-match; mask %pI6\n",
208 conn->src_ip6 = fm.key->src;
209 if (!efx_ipv6_addr_all_ones(&fm.mask->dst)) {
210 netif_dbg(efx, drv, efx->net_dev,
211 "Conntrack ipv6.dst is not exact-match; mask %pI6\n",
215 conn->dst_ip6 = fm.key->dst;
217 netif_dbg(efx, drv, efx->net_dev,
218 "Conntrack missing IPv%u addrs\n", ipv);
222 if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_PORTS)) {
223 struct flow_match_ports fm;
225 flow_rule_match_ports(fr, &fm);
226 if (!IS_ALL_ONES(fm.mask->src)) {
227 netif_dbg(efx, drv, efx->net_dev,
228 "Conntrack ports.src is not exact-match; mask %04x\n",
229 ntohs(fm.mask->src));
232 conn->l4_sport = fm.key->src;
233 if (!IS_ALL_ONES(fm.mask->dst)) {
234 netif_dbg(efx, drv, efx->net_dev,
235 "Conntrack ports.dst is not exact-match; mask %04x\n",
236 ntohs(fm.mask->dst));
239 conn->l4_dport = fm.key->dst;
241 netif_dbg(efx, drv, efx->net_dev, "Conntrack missing L4 ports\n");
245 if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_TCP)) {
246 __be16 tcp_interesting_flags;
247 struct flow_match_tcp fm;
250 netif_dbg(efx, drv, efx->net_dev,
251 "Conntrack matching on TCP keys but ipproto is not tcp\n");
254 flow_rule_match_tcp(fr, &fm);
255 tcp_interesting_flags = EFX_NF_TCP_FLAG(SYN) |
256 EFX_NF_TCP_FLAG(RST) |
257 EFX_NF_TCP_FLAG(FIN);
258 /* If any of the tcp_interesting_flags is set, we always
259 * inhibit CT lookup in LHS (so SW can update CT table).
261 if (fm.key->flags & tcp_interesting_flags) {
262 netif_dbg(efx, drv, efx->net_dev,
263 "Unsupported conntrack tcp.flags %04x/%04x\n",
264 ntohs(fm.key->flags), ntohs(fm.mask->flags));
267 /* Other TCP flags cannot be filtered at CT */
268 if (fm.mask->flags & ~tcp_interesting_flags) {
269 netif_dbg(efx, drv, efx->net_dev,
270 "Unsupported conntrack tcp.flags %04x/%04x\n",
271 ntohs(fm.key->flags), ntohs(fm.mask->flags));
279 static int efx_tc_ct_replace(struct efx_tc_ct_zone *ct_zone,
280 struct flow_cls_offload *tc)
282 struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
283 struct efx_tc_ct_entry *conn, *old;
284 struct efx_nic *efx = ct_zone->efx;
285 const struct flow_action_entry *fa;
286 struct efx_tc_counter *cnt;
289 if (WARN_ON(!efx->tc))
291 if (WARN_ON(!efx->tc->up))
294 conn = kzalloc(sizeof(*conn), GFP_USER);
297 conn->cookie = tc->cookie;
298 old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_ht,
300 efx_tc_ct_ht_params);
302 netif_dbg(efx, drv, efx->net_dev,
303 "Already offloaded conntrack (cookie %lx)\n", tc->cookie);
309 conn->zone = ct_zone;
310 rc = efx_tc_ct_parse_match(efx, fr, conn);
315 flow_action_for_each(i, fa, &fr->action) {
317 case FLOW_ACTION_CT_METADATA:
318 conn->mark = fa->ct_metadata.mark;
319 if (memchr_inv(fa->ct_metadata.labels, 0, sizeof(fa->ct_metadata.labels))) {
320 netif_dbg(efx, drv, efx->net_dev,
321 "Setting CT label not supported\n");
327 netif_dbg(efx, drv, efx->net_dev,
328 "Unhandled action %u for conntrack\n", fa->id);
334 /* fill in defaults for unmangled values */
335 conn->nat_ip = conn->dnat ? conn->dst_ip : conn->src_ip;
336 conn->l4_natport = conn->dnat ? conn->l4_dport : conn->l4_sport;
338 cnt = efx_tc_flower_allocate_counter(efx, EFX_TC_COUNTER_TYPE_CT);
345 rc = efx_mae_insert_ct(efx, conn);
347 netif_dbg(efx, drv, efx->net_dev,
348 "Failed to insert conntrack, %d\n", rc);
351 mutex_lock(&ct_zone->mutex);
352 list_add_tail(&conn->list, &ct_zone->cts);
353 mutex_unlock(&ct_zone->mutex);
357 efx_tc_flower_release_counter(efx, conn->cnt);
359 rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage,
360 efx_tc_ct_ht_params);
365 /* Caller must follow with efx_tc_ct_remove_finish() after RCU grace period! */
366 static void efx_tc_ct_remove(struct efx_nic *efx, struct efx_tc_ct_entry *conn)
370 /* Remove it from HW */
371 rc = efx_mae_remove_ct(efx, conn);
372 /* Delete it from SW */
373 rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage,
374 efx_tc_ct_ht_params);
376 netif_err(efx, drv, efx->net_dev,
377 "Failed to remove conntrack %lx from hw, rc %d\n",
380 netif_dbg(efx, drv, efx->net_dev, "Removed conntrack %lx\n",
385 static void efx_tc_ct_remove_finish(struct efx_nic *efx, struct efx_tc_ct_entry *conn)
387 /* Remove related CT counter. This is delayed after the conn object we
388 * are working with has been successfully removed. This protects the
389 * counter from being used-after-free inside efx_tc_ct_stats.
391 efx_tc_flower_release_counter(efx, conn->cnt);
395 static int efx_tc_ct_destroy(struct efx_tc_ct_zone *ct_zone,
396 struct flow_cls_offload *tc)
398 struct efx_nic *efx = ct_zone->efx;
399 struct efx_tc_ct_entry *conn;
401 conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie,
402 efx_tc_ct_ht_params);
404 netif_warn(efx, drv, efx->net_dev,
405 "Conntrack %lx not found to remove\n", tc->cookie);
409 mutex_lock(&ct_zone->mutex);
410 list_del(&conn->list);
411 efx_tc_ct_remove(efx, conn);
412 mutex_unlock(&ct_zone->mutex);
414 efx_tc_ct_remove_finish(efx, conn);
418 static int efx_tc_ct_stats(struct efx_tc_ct_zone *ct_zone,
419 struct flow_cls_offload *tc)
421 struct efx_nic *efx = ct_zone->efx;
422 struct efx_tc_ct_entry *conn;
423 struct efx_tc_counter *cnt;
426 conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie,
427 efx_tc_ct_ht_params);
429 netif_warn(efx, drv, efx->net_dev,
430 "Conntrack %lx not found for stats\n", tc->cookie);
436 spin_lock_bh(&cnt->lock);
437 /* Report only last use */
438 flow_stats_update(&tc->stats, 0, 0, 0, cnt->touched,
439 FLOW_ACTION_HW_STATS_DELAYED);
440 spin_unlock_bh(&cnt->lock);
446 static int efx_tc_flow_block(enum tc_setup_type type, void *type_data,
449 struct flow_cls_offload *tcb = type_data;
450 struct efx_tc_ct_zone *ct_zone = cb_priv;
452 if (type != TC_SETUP_CLSFLOWER)
455 switch (tcb->command) {
456 case FLOW_CLS_REPLACE:
457 return efx_tc_ct_replace(ct_zone, tcb);
458 case FLOW_CLS_DESTROY:
459 return efx_tc_ct_destroy(ct_zone, tcb);
461 return efx_tc_ct_stats(ct_zone, tcb);
469 struct efx_tc_ct_zone *efx_tc_ct_register_zone(struct efx_nic *efx, u16 zone,
470 struct nf_flowtable *ct_ft)
472 struct efx_tc_ct_zone *ct_zone, *old;
475 ct_zone = kzalloc(sizeof(*ct_zone), GFP_USER);
477 return ERR_PTR(-ENOMEM);
478 ct_zone->zone = zone;
479 old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_zone_ht,
481 efx_tc_ct_zone_ht_params);
483 /* don't need our new entry */
485 if (!refcount_inc_not_zero(&old->ref))
486 return ERR_PTR(-EAGAIN);
487 /* existing entry found */
488 WARN_ON_ONCE(old->nf_ft != ct_ft);
489 netif_dbg(efx, drv, efx->net_dev,
490 "Found existing ct_zone for %u\n", zone);
493 ct_zone->nf_ft = ct_ft;
495 INIT_LIST_HEAD(&ct_zone->cts);
496 mutex_init(&ct_zone->mutex);
497 rc = nf_flow_table_offload_add_cb(ct_ft, efx_tc_flow_block, ct_zone);
498 netif_dbg(efx, drv, efx->net_dev, "Adding new ct_zone for %u, rc %d\n",
502 refcount_set(&ct_zone->ref, 1);
505 rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage,
506 efx_tc_ct_zone_ht_params);
511 void efx_tc_ct_unregister_zone(struct efx_nic *efx,
512 struct efx_tc_ct_zone *ct_zone)
514 struct efx_tc_ct_entry *conn, *next;
516 if (!refcount_dec_and_test(&ct_zone->ref))
517 return; /* still in use */
518 nf_flow_table_offload_del_cb(ct_zone->nf_ft, efx_tc_flow_block, ct_zone);
519 rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage,
520 efx_tc_ct_zone_ht_params);
521 mutex_lock(&ct_zone->mutex);
522 list_for_each_entry(conn, &ct_zone->cts, list)
523 efx_tc_ct_remove(efx, conn);
525 /* need to use _safe because efx_tc_ct_remove_finish() frees conn */
526 list_for_each_entry_safe(conn, next, &ct_zone->cts, list)
527 efx_tc_ct_remove_finish(efx, conn);
528 mutex_unlock(&ct_zone->mutex);
529 mutex_destroy(&ct_zone->mutex);
530 netif_dbg(efx, drv, efx->net_dev, "Removed ct_zone for %u\n",