drivers/net/ethernet/netronome/nfp/flower/conntrack.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2021 Corigine, Inc. */
   3
   4 #include <net/tc_act/tc_csum.h>
   5 #include <net/tc_act/tc_ct.h>
   6
   7 #include "conntrack.h"
   8 #include "../nfp_port.h"
   9
  10 const struct rhashtable_params nfp_tc_ct_merge_params = {
  11         .head_offset            = offsetof(struct nfp_fl_ct_tc_merge,
  12                                            hash_node),
  13         .key_len                = sizeof(unsigned long) * 2,
  14         .key_offset             = offsetof(struct nfp_fl_ct_tc_merge, cookie),
  15         .automatic_shrinking    = true,
  16 };
  17
  18 const struct rhashtable_params nfp_nft_ct_merge_params = {
  19         .head_offset            = offsetof(struct nfp_fl_nft_tc_merge,
  20                                            hash_node),
  21         .key_len                = sizeof(unsigned long) * 3,
  22         .key_offset             = offsetof(struct nfp_fl_nft_tc_merge, cookie),
  23         .automatic_shrinking    = true,
  24 };
  25
  26 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
  27                                               enum flow_action_id act_id);
  28
  29 /**
  30  * get_hashentry() - Wrapper around hashtable lookup.
  31  * @ht:         hashtable where entry could be found
  32  * @key:        key to lookup
  33  * @params:     hashtable params
  34  * @size:       size of entry to allocate if not in table
  35  *
  36  * Returns an entry from a hashtable. If entry does not exist
  37  * yet allocate the memory for it and return the new entry.
  38  */
  39 static void *get_hashentry(struct rhashtable *ht, void *key,
  40                            const struct rhashtable_params params, size_t size)
  41 {
  42         void *result;
  43
  44         result = rhashtable_lookup_fast(ht, key, params);
  45
  46         if (result)
  47                 return result;
  48
  49         result = kzalloc(size, GFP_KERNEL);
  50         if (!result)
  51                 return ERR_PTR(-ENOMEM);
  52
  53         return result;
  54 }
  55
  56 bool is_pre_ct_flow(struct flow_cls_offload *flow)
  57 {
  58         struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
  59         struct flow_dissector *dissector = rule->match.dissector;
  60         struct flow_action_entry *act;
  61         struct flow_match_ct ct;
  62         int i;
  63
  64         if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
  65                 flow_rule_match_ct(rule, &ct);
  66                 if (ct.key->ct_state)
  67                         return false;
  68         }
  69
  70         if (flow->common.chain_index)
  71                 return false;
  72
  73         flow_action_for_each(i, act, &flow->rule->action) {
  74                 if (act->id == FLOW_ACTION_CT) {
  75                         /* The pre_ct rule only have the ct or ct nat action, cannot
  76                          * contains other ct action e.g ct commit and so on.
  77                          */
  78                         if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT))
  79                                 return true;
  80                         else
  81                                 return false;
  82                 }
  83         }
  84
  85         return false;
  86 }
  87
  88 bool is_post_ct_flow(struct flow_cls_offload *flow)
  89 {
  90         struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
  91         struct flow_dissector *dissector = rule->match.dissector;
  92         struct flow_action_entry *act;
  93         bool exist_ct_clear = false;
  94         struct flow_match_ct ct;
  95         int i;
  96
  97         if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
  98                 flow_rule_match_ct(rule, &ct);
  99                 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
 100                         return true;
 101         } else {
 102                 /* post ct entry cannot contains any ct action except ct_clear. */
 103                 flow_action_for_each(i, act, &flow->rule->action) {
 104                         if (act->id == FLOW_ACTION_CT) {
 105                                 /* ignore ct clear action. */
 106                                 if (act->ct.action == TCA_CT_ACT_CLEAR) {
 107                                         exist_ct_clear = true;
 108                                         continue;
 109                                 }
 110
 111                                 return false;
 112                         }
 113                 }
 114                 /* when do nat with ct, the post ct entry ignore the ct status,
 115                  * will match the nat field(sip/dip) instead. In this situation,
 116                  * the flow chain index is not zero and contains ct clear action.
 117                  */
 118                 if (flow->common.chain_index && exist_ct_clear)
 119                         return true;
 120         }
 121
 122         return false;
 123 }
 124
 125 /**
 126  * get_mangled_key() - Mangle the key if mangle act exists
 127  * @rule:       rule that carries the actions
 128  * @buf:        pointer to key to be mangled
 129  * @offset:     used to adjust mangled offset in L2/L3/L4 header
 130  * @key_sz:     key size
 131  * @htype:      mangling type
 132  *
 133  * Returns buf where the mangled key stores.
 134  */
 135 static void *get_mangled_key(struct flow_rule *rule, void *buf,
 136                              u32 offset, size_t key_sz,
 137                              enum flow_action_mangle_base htype)
 138 {
 139         struct flow_action_entry *act;
 140         u32 *val = (u32 *)buf;
 141         u32 off, msk, key;
 142         int i;
 143
 144         flow_action_for_each(i, act, &rule->action) {
 145                 if (act->id == FLOW_ACTION_MANGLE &&
 146                     act->mangle.htype == htype) {
 147                         off = act->mangle.offset - offset;
 148                         msk = act->mangle.mask;
 149                         key = act->mangle.val;
 150
 151                         /* Mangling is supposed to be u32 aligned */
 152                         if (off % 4 || off >= key_sz)
 153                                 continue;
 154
 155                         val[off >> 2] &= msk;
 156                         val[off >> 2] |= key;
 157                 }
 158         }
 159
 160         return buf;
 161 }
 162
 163 /* Only tos and ttl are involved in flow_match_ip structure, which
 164  * doesn't conform to the layout of ip/ipv6 header definition. So
 165  * they need particular process here: fill them into the ip/ipv6
 166  * header, so that mangling actions can work directly.
 167  */
 168 #define NFP_IPV4_TOS_MASK       GENMASK(23, 16)
 169 #define NFP_IPV4_TTL_MASK       GENMASK(31, 24)
 170 #define NFP_IPV6_TCLASS_MASK    GENMASK(27, 20)
 171 #define NFP_IPV6_HLIMIT_MASK    GENMASK(7, 0)
 172 static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf,
 173                                  bool is_v6)
 174 {
 175         struct flow_match_ip match;
 176         /* IPv4's ttl field is in third dword. */
 177         __be32 ip_hdr[3];
 178         u32 tmp, hdr_len;
 179
 180         flow_rule_match_ip(rule, &match);
 181
 182         if (is_v6) {
 183                 tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos);
 184                 ip_hdr[0] = cpu_to_be32(tmp);
 185                 tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl);
 186                 ip_hdr[1] = cpu_to_be32(tmp);
 187                 hdr_len = 2 * sizeof(__be32);
 188         } else {
 189                 tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos);
 190                 ip_hdr[0] = cpu_to_be32(tmp);
 191                 tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl);
 192                 ip_hdr[2] = cpu_to_be32(tmp);
 193                 hdr_len = 3 * sizeof(__be32);
 194         }
 195
 196         get_mangled_key(rule, ip_hdr, 0, hdr_len,
 197                         is_v6 ? FLOW_ACT_MANGLE_HDR_TYPE_IP6 :
 198                                 FLOW_ACT_MANGLE_HDR_TYPE_IP4);
 199
 200         match.key = buf;
 201
 202         if (is_v6) {
 203                 tmp = be32_to_cpu(ip_hdr[0]);
 204                 match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp);
 205                 tmp = be32_to_cpu(ip_hdr[1]);
 206                 match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp);
 207         } else {
 208                 tmp = be32_to_cpu(ip_hdr[0]);
 209                 match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp);
 210                 tmp = be32_to_cpu(ip_hdr[2]);
 211                 match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp);
 212         }
 213
 214         return buf;
 215 }
 216
 217 /* Note entry1 and entry2 are not swappable. only skip ip and
 218  * tport merge check for pre_ct and post_ct when pre_ct do nat.
 219  */
 220 static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1,
 221                                            struct nfp_fl_ct_flow_entry *entry2)
 222 {
 223         /* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */
 224         if ((entry1->flags & NFP_FL_ACTION_DO_NAT) &&
 225             entry2->type == CT_TYPE_POST_CT)
 226                 return false;
 227
 228         return true;
 229 }
 230
 231 /* Note entry1 and entry2 are not swappable, entry1 should be
 232  * the former flow whose mangle action need be taken into account
 233  * if existed, and entry2 should be the latter flow whose action
 234  * we don't care.
 235  */
 236 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
 237                               struct nfp_fl_ct_flow_entry *entry2)
 238 {
 239         unsigned long long ovlp_keys;
 240         bool out, is_v6 = false;
 241         u8 ip_proto = 0;
 242         ovlp_keys = entry1->rule->match.dissector->used_keys &
 243                         entry2->rule->match.dissector->used_keys;
 244         /* Temporary buffer for mangling keys, 64 is enough to cover max
 245          * struct size of key in various fields that may be mangled.
 246          * Supported fields to mangle:
 247          * mac_src/mac_dst(struct flow_match_eth_addrs, 12B)
 248          * nw_tos/nw_ttl(struct flow_match_ip, 2B)
 249          * nw_src/nw_dst(struct flow_match_ipv4/6_addrs, 32B)
 250          * tp_src/tp_dst(struct flow_match_ports, 4B)
 251          */
 252         char buf[64];
 253
 254         if (entry1->netdev && entry2->netdev &&
 255             entry1->netdev != entry2->netdev)
 256                 return -EINVAL;
 257
 258         /* Check the overlapped fields one by one, the unmasked part
 259          * should not conflict with each other.
 260          */
 261         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) {
 262                 struct flow_match_control match1, match2;
 263
 264                 flow_rule_match_control(entry1->rule, &match1);
 265                 flow_rule_match_control(entry2->rule, &match2);
 266                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 267                 if (out)
 268                         goto check_failed;
 269         }
 270
 271         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) {
 272                 struct flow_match_basic match1, match2;
 273
 274                 flow_rule_match_basic(entry1->rule, &match1);
 275                 flow_rule_match_basic(entry2->rule, &match2);
 276
 277                 /* n_proto field is a must in ct-related flows,
 278                  * it should be either ipv4 or ipv6.
 279                  */
 280                 is_v6 = match1.key->n_proto == htons(ETH_P_IPV6);
 281                 /* ip_proto field is a must when port field is cared */
 282                 ip_proto = match1.key->ip_proto;
 283
 284                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 285                 if (out)
 286                         goto check_failed;
 287         }
 288
 289         /* if pre ct entry do nat, the nat ip exists in nft entry,
 290          * will be do merge check when do nft and post ct merge,
 291          * so skip this ip merge check here.
 292          */
 293         if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) &&
 294             nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 295                 struct flow_match_ipv4_addrs match1, match2;
 296
 297                 flow_rule_match_ipv4_addrs(entry1->rule, &match1);
 298                 flow_rule_match_ipv4_addrs(entry2->rule, &match2);
 299
 300                 memcpy(buf, match1.key, sizeof(*match1.key));
 301                 match1.key = get_mangled_key(entry1->rule, buf,
 302                                              offsetof(struct iphdr, saddr),
 303                                              sizeof(*match1.key),
 304                                              FLOW_ACT_MANGLE_HDR_TYPE_IP4);
 305
 306                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 307                 if (out)
 308                         goto check_failed;
 309         }
 310
 311         /* if pre ct entry do nat, the nat ip exists in nft entry,
 312          * will be do merge check when do nft and post ct merge,
 313          * so skip this ip merge check here.
 314          */
 315         if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) &&
 316             nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 317                 struct flow_match_ipv6_addrs match1, match2;
 318
 319                 flow_rule_match_ipv6_addrs(entry1->rule, &match1);
 320                 flow_rule_match_ipv6_addrs(entry2->rule, &match2);
 321
 322                 memcpy(buf, match1.key, sizeof(*match1.key));
 323                 match1.key = get_mangled_key(entry1->rule, buf,
 324                                              offsetof(struct ipv6hdr, saddr),
 325                                              sizeof(*match1.key),
 326                                              FLOW_ACT_MANGLE_HDR_TYPE_IP6);
 327
 328                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 329                 if (out)
 330                         goto check_failed;
 331         }
 332
 333         /* if pre ct entry do nat, the nat tport exists in nft entry,
 334          * will be do merge check when do nft and post ct merge,
 335          * so skip this tport merge check here.
 336          */
 337         if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_PORTS)) &&
 338             nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 339                 enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC;
 340                 struct flow_match_ports match1, match2;
 341
 342                 flow_rule_match_ports(entry1->rule, &match1);
 343                 flow_rule_match_ports(entry2->rule, &match2);
 344
 345                 if (ip_proto == IPPROTO_UDP)
 346                         htype = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
 347                 else if (ip_proto == IPPROTO_TCP)
 348                         htype = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
 349
 350                 memcpy(buf, match1.key, sizeof(*match1.key));
 351                 match1.key = get_mangled_key(entry1->rule, buf, 0,
 352                                              sizeof(*match1.key), htype);
 353
 354                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 355                 if (out)
 356                         goto check_failed;
 357         }
 358
 359         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 360                 struct flow_match_eth_addrs match1, match2;
 361
 362                 flow_rule_match_eth_addrs(entry1->rule, &match1);
 363                 flow_rule_match_eth_addrs(entry2->rule, &match2);
 364
 365                 memcpy(buf, match1.key, sizeof(*match1.key));
 366                 match1.key = get_mangled_key(entry1->rule, buf, 0,
 367                                              sizeof(*match1.key),
 368                                              FLOW_ACT_MANGLE_HDR_TYPE_ETH);
 369
 370                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 371                 if (out)
 372                         goto check_failed;
 373         }
 374
 375         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN)) {
 376                 struct flow_match_vlan match1, match2;
 377
 378                 flow_rule_match_vlan(entry1->rule, &match1);
 379                 flow_rule_match_vlan(entry2->rule, &match2);
 380                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 381                 if (out)
 382                         goto check_failed;
 383         }
 384
 385         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_MPLS)) {
 386                 struct flow_match_mpls match1, match2;
 387
 388                 flow_rule_match_mpls(entry1->rule, &match1);
 389                 flow_rule_match_mpls(entry2->rule, &match2);
 390                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 391                 if (out)
 392                         goto check_failed;
 393         }
 394
 395         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_TCP)) {
 396                 struct flow_match_tcp match1, match2;
 397
 398                 flow_rule_match_tcp(entry1->rule, &match1);
 399                 flow_rule_match_tcp(entry2->rule, &match2);
 400                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 401                 if (out)
 402                         goto check_failed;
 403         }
 404
 405         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IP)) {
 406                 struct flow_match_ip match1, match2;
 407
 408                 flow_rule_match_ip(entry1->rule, &match1);
 409                 flow_rule_match_ip(entry2->rule, &match2);
 410
 411                 match1.key = get_mangled_tos_ttl(entry1->rule, buf, is_v6);
 412                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 413                 if (out)
 414                         goto check_failed;
 415         }
 416
 417         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 418                 struct flow_match_enc_keyid match1, match2;
 419
 420                 flow_rule_match_enc_keyid(entry1->rule, &match1);
 421                 flow_rule_match_enc_keyid(entry2->rule, &match2);
 422                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 423                 if (out)
 424                         goto check_failed;
 425         }
 426
 427         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 428                 struct flow_match_ipv4_addrs match1, match2;
 429
 430                 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
 431                 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
 432                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 433                 if (out)
 434                         goto check_failed;
 435         }
 436
 437         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
 438                 struct flow_match_ipv6_addrs match1, match2;
 439
 440                 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
 441                 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
 442                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 443                 if (out)
 444                         goto check_failed;
 445         }
 446
 447         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 448                 struct flow_match_control match1, match2;
 449
 450                 flow_rule_match_enc_control(entry1->rule, &match1);
 451                 flow_rule_match_enc_control(entry2->rule, &match2);
 452                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 453                 if (out)
 454                         goto check_failed;
 455         }
 456
 457         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) {
 458                 struct flow_match_ip match1, match2;
 459
 460                 flow_rule_match_enc_ip(entry1->rule, &match1);
 461                 flow_rule_match_enc_ip(entry2->rule, &match2);
 462                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 463                 if (out)
 464                         goto check_failed;
 465         }
 466
 467         if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
 468                 struct flow_match_enc_opts match1, match2;
 469
 470                 flow_rule_match_enc_opts(entry1->rule, &match1);
 471                 flow_rule_match_enc_opts(entry2->rule, &match2);
 472                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 473                 if (out)
 474                         goto check_failed;
 475         }
 476
 477         return 0;
 478
 479 check_failed:
 480         return -EINVAL;
 481 }
 482
 483 static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in,
 484                                    struct flow_rule *rule)
 485 {
 486         struct flow_match_vlan match;
 487
 488         if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)))
 489                 return -EOPNOTSUPP;
 490
 491         /* post_ct does not match VLAN KEY, can be merged. */
 492         if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)))
 493                 return 0;
 494
 495         switch (a_in->id) {
 496         /* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */
 497         case FLOW_ACTION_VLAN_POP:
 498                 return -EOPNOTSUPP;
 499
 500         case FLOW_ACTION_VLAN_PUSH:
 501         case FLOW_ACTION_VLAN_MANGLE:
 502                 flow_rule_match_vlan(rule, &match);
 503                 /* different vlan id, cannot be merged. */
 504                 if ((match.key->vlan_id & match.mask->vlan_id) ^
 505                     (a_in->vlan.vid & match.mask->vlan_id))
 506                         return -EOPNOTSUPP;
 507
 508                 /* different tpid, cannot be merged. */
 509                 if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^
 510                     (a_in->vlan.proto & match.mask->vlan_tpid))
 511                         return -EOPNOTSUPP;
 512
 513                 /* different priority, cannot be merged. */
 514                 if ((match.key->vlan_priority & match.mask->vlan_priority) ^
 515                     (a_in->vlan.prio & match.mask->vlan_priority))
 516                         return -EOPNOTSUPP;
 517
 518                 break;
 519         default:
 520                 return -EOPNOTSUPP;
 521         }
 522
 523         return 0;
 524 }
 525
 526 /* Extra check for multiple ct-zones merge
 527  * currently surpport nft entries merge check in different zones
 528  */
 529 static int nfp_ct_merge_extra_check(struct nfp_fl_ct_flow_entry *nft_entry,
 530                                     struct nfp_fl_ct_tc_merge *tc_m_entry)
 531 {
 532         struct nfp_fl_nft_tc_merge *prev_nft_m_entry;
 533         struct nfp_fl_ct_flow_entry *pre_ct_entry;
 534
 535         pre_ct_entry = tc_m_entry->pre_ct_parent;
 536         prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1];
 537
 538         return nfp_ct_merge_check(prev_nft_m_entry->nft_parent, nft_entry);
 539 }
 540
 541 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
 542                                   struct nfp_fl_ct_flow_entry *post_ct_entry,
 543                                   struct nfp_fl_ct_flow_entry *nft_entry)
 544 {
 545         struct flow_action_entry *act;
 546         int i, err;
 547
 548         /* Check for pre_ct->action conflicts */
 549         flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
 550                 switch (act->id) {
 551                 case FLOW_ACTION_VLAN_PUSH:
 552                 case FLOW_ACTION_VLAN_POP:
 553                 case FLOW_ACTION_VLAN_MANGLE:
 554                         err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule);
 555                         if (err)
 556                                 return err;
 557                         break;
 558                 case FLOW_ACTION_MPLS_PUSH:
 559                 case FLOW_ACTION_MPLS_POP:
 560                 case FLOW_ACTION_MPLS_MANGLE:
 561                         return -EOPNOTSUPP;
 562                 default:
 563                         break;
 564                 }
 565         }
 566
 567         /* Check for nft->action conflicts */
 568         flow_action_for_each(i, act, &nft_entry->rule->action) {
 569                 switch (act->id) {
 570                 case FLOW_ACTION_VLAN_PUSH:
 571                 case FLOW_ACTION_VLAN_POP:
 572                 case FLOW_ACTION_VLAN_MANGLE:
 573                 case FLOW_ACTION_MPLS_PUSH:
 574                 case FLOW_ACTION_MPLS_POP:
 575                 case FLOW_ACTION_MPLS_MANGLE:
 576                         return -EOPNOTSUPP;
 577                 default:
 578                         break;
 579                 }
 580         }
 581         return 0;
 582 }
 583
 584 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
 585                              struct nfp_fl_ct_flow_entry *nft_entry)
 586 {
 587         struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
 588         struct flow_action_entry *ct_met;
 589         struct flow_match_ct ct;
 590         int i;
 591
 592         ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
 593         if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) {
 594                 u32 *act_lbl;
 595
 596                 act_lbl = ct_met->ct_metadata.labels;
 597                 flow_rule_match_ct(post_ct_entry->rule, &ct);
 598                 for (i = 0; i < 4; i++) {
 599                         if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
 600                             (act_lbl[i] & ct.mask->ct_labels[i]))
 601                                 return -EINVAL;
 602                 }
 603
 604                 if ((ct.key->ct_mark & ct.mask->ct_mark) ^
 605                     (ct_met->ct_metadata.mark & ct.mask->ct_mark))
 606                         return -EINVAL;
 607
 608                 return 0;
 609         } else {
 610                 /* post_ct with ct clear action will not match the
 611                  * ct status when nft is nat entry.
 612                  */
 613                 if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE)
 614                         return 0;
 615         }
 616
 617         return -EINVAL;
 618 }
 619
 620 static int
 621 nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map)
 622 {
 623         int key_size;
 624
 625         /* This field must always be present */
 626         key_size = sizeof(struct nfp_flower_meta_tci);
 627         map[FLOW_PAY_META_TCI] = 0;
 628
 629         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_EXT_META) {
 630                 map[FLOW_PAY_EXT_META] = key_size;
 631                 key_size += sizeof(struct nfp_flower_ext_meta);
 632         }
 633         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_PORT) {
 634                 map[FLOW_PAY_INPORT] = key_size;
 635                 key_size += sizeof(struct nfp_flower_in_port);
 636         }
 637         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_MAC) {
 638                 map[FLOW_PAY_MAC_MPLS] = key_size;
 639                 key_size += sizeof(struct nfp_flower_mac_mpls);
 640         }
 641         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_TP) {
 642                 map[FLOW_PAY_L4] = key_size;
 643                 key_size += sizeof(struct nfp_flower_tp_ports);
 644         }
 645         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV4) {
 646                 map[FLOW_PAY_IPV4] = key_size;
 647                 key_size += sizeof(struct nfp_flower_ipv4);
 648         }
 649         if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV6) {
 650                 map[FLOW_PAY_IPV6] = key_size;
 651                 key_size += sizeof(struct nfp_flower_ipv6);
 652         }
 653
 654         if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_QINQ) {
 655                 map[FLOW_PAY_QINQ] = key_size;
 656                 key_size += sizeof(struct nfp_flower_vlan);
 657         }
 658
 659         if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
 660                 map[FLOW_PAY_GRE] = key_size;
 661                 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
 662                         key_size += sizeof(struct nfp_flower_ipv6_gre_tun);
 663                 else
 664                         key_size += sizeof(struct nfp_flower_ipv4_gre_tun);
 665         }
 666
 667         if ((in_key_ls.key_layer & NFP_FLOWER_LAYER_VXLAN) ||
 668             (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE)) {
 669                 map[FLOW_PAY_UDP_TUN] = key_size;
 670                 if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
 671                         key_size += sizeof(struct nfp_flower_ipv6_udp_tun);
 672                 else
 673                         key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
 674         }
 675
 676         if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
 677                 map[FLOW_PAY_GENEVE_OPT] = key_size;
 678                 key_size += sizeof(struct nfp_flower_geneve_options);
 679         }
 680
 681         return key_size;
 682 }
 683
 684 /* get the csum flag according the ip proto and mangle action. */
 685 static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum)
 686 {
 687         if (a_in->id != FLOW_ACTION_MANGLE)
 688                 return;
 689
 690         switch (a_in->mangle.htype) {
 691         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 692                 *csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR;
 693                 if (ip_proto == IPPROTO_TCP)
 694                         *csum |= TCA_CSUM_UPDATE_FLAG_TCP;
 695                 else if (ip_proto == IPPROTO_UDP)
 696                         *csum |= TCA_CSUM_UPDATE_FLAG_UDP;
 697                 break;
 698         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 699                 *csum |= TCA_CSUM_UPDATE_FLAG_TCP;
 700                 break;
 701         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 702                 *csum |= TCA_CSUM_UPDATE_FLAG_UDP;
 703                 break;
 704         default:
 705                 break;
 706         }
 707 }
 708
 709 static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 710                                         struct nfp_flower_priv *priv,
 711                                         struct net_device *netdev,
 712                                         struct nfp_fl_payload *flow_pay,
 713                                         int num_rules)
 714 {
 715         enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
 716         struct flow_action_entry *a_in;
 717         int i, j, id, num_actions = 0;
 718         struct flow_rule *a_rule;
 719         int err = 0, offset = 0;
 720
 721         for (i = 0; i < num_rules; i++)
 722                 num_actions += rules[i]->action.num_entries;
 723
 724         /* Add one action to make sure there is enough room to add an checksum action
 725          * when do nat.
 726          */
 727         a_rule = flow_rule_alloc(num_actions + (num_rules / 2));
 728         if (!a_rule)
 729                 return -ENOMEM;
 730
 731         /* post_ct entry have one action at least. */
 732         if (rules[num_rules - 1]->action.num_entries != 0)
 733                 tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats;
 734
 735         /* Actions need a BASIC dissector. */
 736         a_rule->match = rules[0]->match;
 737
 738         /* Copy actions */
 739         for (j = 0; j < num_rules; j++) {
 740                 u32 csum_updated = 0;
 741                 u8 ip_proto = 0;
 742
 743                 if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) {
 744                         struct flow_match_basic match;
 745
 746                         /* ip_proto is the only field that is needed in later compile_action,
 747                          * needed to set the correct checksum flags. It doesn't really matter
 748                          * which input rule's ip_proto field we take as the earlier merge checks
 749                          * would have made sure that they don't conflict. We do not know which
 750                          * of the subflows would have the ip_proto filled in, so we need to iterate
 751                          * through the subflows and assign the proper subflow to a_rule
 752                          */
 753                         flow_rule_match_basic(rules[j], &match);
 754                         if (match.mask->ip_proto) {
 755                                 a_rule->match = rules[j]->match;
 756                                 ip_proto = match.key->ip_proto;
 757                         }
 758                 }
 759
 760                 for (i = 0; i < rules[j]->action.num_entries; i++) {
 761                         a_in = &rules[j]->action.entries[i];
 762                         id = a_in->id;
 763
 764                         /* Ignore CT related actions as these would already have
 765                          * been taken care of by previous checks, and we do not send
 766                          * any CT actions to the firmware.
 767                          */
 768                         switch (id) {
 769                         case FLOW_ACTION_CT:
 770                         case FLOW_ACTION_GOTO:
 771                         case FLOW_ACTION_CT_METADATA:
 772                                 continue;
 773                         default:
 774                                 /* nft entry is generated by tc ct, which mangle action do not care
 775                                  * the stats, inherit the post entry stats to meet the
 776                                  * flow_action_hw_stats_check.
 777                                  * nft entry flow rules are at odd array index.
 778                                  */
 779                                 if (j & 0x01) {
 780                                         if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE)
 781                                                 a_in->hw_stats = tmp_stats;
 782                                         nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated);
 783                                 }
 784                                 memcpy(&a_rule->action.entries[offset++],
 785                                        a_in, sizeof(struct flow_action_entry));
 786                                 break;
 787                         }
 788                 }
 789                 /* nft entry have mangle action, but do not have checksum action when do NAT,
 790                  * hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action
 791                  * to meet csum action check.
 792                  */
 793                 if (csum_updated) {
 794                         struct flow_action_entry *csum_action;
 795
 796                         csum_action = &a_rule->action.entries[offset++];
 797                         csum_action->id = FLOW_ACTION_CSUM;
 798                         csum_action->csum_flags = csum_updated;
 799                         csum_action->hw_stats = tmp_stats;
 800                 }
 801         }
 802
 803         /* Some actions would have been ignored, so update the num_entries field */
 804         a_rule->action.num_entries = offset;
 805         err = nfp_flower_compile_action(priv->app, a_rule, netdev, flow_pay, NULL);
 806         kfree(a_rule);
 807
 808         return err;
 809 }
 810
 811 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 812 {
 813         enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
 814         struct nfp_fl_ct_zone_entry *zt = m_entry->zt;
 815         struct flow_rule *rules[NFP_MAX_ENTRY_RULES];
 816         struct nfp_fl_ct_flow_entry *pre_ct_entry;
 817         struct nfp_fl_key_ls key_layer, tmp_layer;
 818         struct nfp_flower_priv *priv = zt->priv;
 819         u16 key_map[_FLOW_PAY_LAYERS_MAX];
 820         struct nfp_fl_payload *flow_pay;
 821         u8 *key, *msk, *kdata, *mdata;
 822         struct nfp_port *port = NULL;
 823         int num_rules, err, i, j = 0;
 824         struct net_device *netdev;
 825         bool qinq_sup;
 826         u32 port_id;
 827         u16 offset;
 828
 829         netdev = m_entry->netdev;
 830         qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ);
 831
 832         pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
 833         num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX;
 834
 835         for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) {
 836                 rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule;
 837                 rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule;
 838         }
 839
 840         rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule;
 841         rules[j++] = m_entry->nft_parent->rule;
 842         rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule;
 843
 844         memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls));
 845         memset(&key_map, 0, sizeof(key_map));
 846
 847         /* Calculate the resultant key layer and size for offload */
 848         for (i = 0; i < num_rules; i++) {
 849                 err = nfp_flower_calculate_key_layers(priv->app,
 850                                                       m_entry->netdev,
 851                                                       &tmp_layer, rules[i],
 852                                                       &tun_type, NULL);
 853                 if (err)
 854                         return err;
 855
 856                 key_layer.key_layer |= tmp_layer.key_layer;
 857                 key_layer.key_layer_two |= tmp_layer.key_layer_two;
 858         }
 859         key_layer.key_size = nfp_fl_calc_key_layers_sz(key_layer, key_map);
 860
 861         flow_pay = nfp_flower_allocate_new(&key_layer);
 862         if (!flow_pay)
 863                 return -ENOMEM;
 864
 865         memset(flow_pay->unmasked_data, 0, key_layer.key_size);
 866         memset(flow_pay->mask_data, 0, key_layer.key_size);
 867
 868         kdata = flow_pay->unmasked_data;
 869         mdata = flow_pay->mask_data;
 870
 871         offset = key_map[FLOW_PAY_META_TCI];
 872         key = kdata + offset;
 873         msk = mdata + offset;
 874         nfp_flower_compile_meta((struct nfp_flower_meta_tci *)key,
 875                                 (struct nfp_flower_meta_tci *)msk,
 876                                 key_layer.key_layer);
 877
 878         if (NFP_FLOWER_LAYER_EXT_META & key_layer.key_layer) {
 879                 offset =  key_map[FLOW_PAY_EXT_META];
 880                 key = kdata + offset;
 881                 msk = mdata + offset;
 882                 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)key,
 883                                             key_layer.key_layer_two);
 884                 nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)msk,
 885                                             key_layer.key_layer_two);
 886         }
 887
 888         /* Using in_port from the -trk rule. The tc merge checks should already
 889          * be checking that the ingress netdevs are the same
 890          */
 891         port_id = nfp_flower_get_port_id_from_netdev(priv->app, netdev);
 892         offset = key_map[FLOW_PAY_INPORT];
 893         key = kdata + offset;
 894         msk = mdata + offset;
 895         err = nfp_flower_compile_port((struct nfp_flower_in_port *)key,
 896                                       port_id, false, tun_type, NULL);
 897         if (err)
 898                 goto ct_offload_err;
 899         err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk,
 900                                       port_id, true, tun_type, NULL);
 901         if (err)
 902                 goto ct_offload_err;
 903
 904         /* This following part works on the assumption that previous checks has
 905          * already filtered out flows that has different values for the different
 906          * layers. Here we iterate through all three rules and merge their respective
 907          * masked value(cared bits), basic method is:
 908          * final_key = (r1_key & r1_mask) | (r2_key & r2_mask) | (r3_key & r3_mask)
 909          * final_mask = r1_mask | r2_mask | r3_mask
 910          * If none of the rules contains a match that is also fine, that simply means
 911          * that the layer is not present.
 912          */
 913         if (!qinq_sup) {
 914                 for (i = 0; i < num_rules; i++) {
 915                         offset = key_map[FLOW_PAY_META_TCI];
 916                         key = kdata + offset;
 917                         msk = mdata + offset;
 918                         nfp_flower_compile_tci((struct nfp_flower_meta_tci *)key,
 919                                                (struct nfp_flower_meta_tci *)msk,
 920                                                rules[i]);
 921                 }
 922         }
 923
 924         if (NFP_FLOWER_LAYER_MAC & key_layer.key_layer) {
 925                 offset = key_map[FLOW_PAY_MAC_MPLS];
 926                 key = kdata + offset;
 927                 msk = mdata + offset;
 928                 for (i = 0; i < num_rules; i++) {
 929                         nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)key,
 930                                                (struct nfp_flower_mac_mpls *)msk,
 931                                                rules[i]);
 932                         err = nfp_flower_compile_mpls((struct nfp_flower_mac_mpls *)key,
 933                                                       (struct nfp_flower_mac_mpls *)msk,
 934                                                       rules[i], NULL);
 935                         if (err)
 936                                 goto ct_offload_err;
 937                 }
 938         }
 939
 940         if (NFP_FLOWER_LAYER_IPV4 & key_layer.key_layer) {
 941                 offset = key_map[FLOW_PAY_IPV4];
 942                 key = kdata + offset;
 943                 msk = mdata + offset;
 944                 for (i = 0; i < num_rules; i++) {
 945                         nfp_flower_compile_ipv4((struct nfp_flower_ipv4 *)key,
 946                                                 (struct nfp_flower_ipv4 *)msk,
 947                                                 rules[i]);
 948                 }
 949         }
 950
 951         if (NFP_FLOWER_LAYER_IPV6 & key_layer.key_layer) {
 952                 offset = key_map[FLOW_PAY_IPV6];
 953                 key = kdata + offset;
 954                 msk = mdata + offset;
 955                 for (i = 0; i < num_rules; i++) {
 956                         nfp_flower_compile_ipv6((struct nfp_flower_ipv6 *)key,
 957                                                 (struct nfp_flower_ipv6 *)msk,
 958                                                 rules[i]);
 959                 }
 960         }
 961
 962         if (NFP_FLOWER_LAYER_TP & key_layer.key_layer) {
 963                 offset = key_map[FLOW_PAY_L4];
 964                 key = kdata + offset;
 965                 msk = mdata + offset;
 966                 for (i = 0; i < num_rules; i++) {
 967                         nfp_flower_compile_tport((struct nfp_flower_tp_ports *)key,
 968                                                  (struct nfp_flower_tp_ports *)msk,
 969                                                  rules[i]);
 970                 }
 971         }
 972
 973         if (NFP_FLOWER_LAYER2_QINQ & key_layer.key_layer_two) {
 974                 offset = key_map[FLOW_PAY_QINQ];
 975                 key = kdata + offset;
 976                 msk = mdata + offset;
 977                 for (i = 0; i < num_rules; i++) {
 978                         nfp_flower_compile_vlan((struct nfp_flower_vlan *)key,
 979                                                 (struct nfp_flower_vlan *)msk,
 980                                                 rules[i]);
 981                 }
 982         }
 983
 984         if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
 985                 offset = key_map[FLOW_PAY_GRE];
 986                 key = kdata + offset;
 987                 msk = mdata + offset;
 988                 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
 989                         struct nfp_flower_ipv6_gre_tun *gre_match;
 990                         struct nfp_ipv6_addr_entry *entry;
 991                         struct in6_addr *dst;
 992
 993                         for (i = 0; i < num_rules; i++) {
 994                                 nfp_flower_compile_ipv6_gre_tun((void *)key,
 995                                                                 (void *)msk, rules[i]);
 996                         }
 997                         gre_match = (struct nfp_flower_ipv6_gre_tun *)key;
 998                         dst = &gre_match->ipv6.dst;
 999
1000                         entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
1001                         if (!entry) {
1002                                 err = -ENOMEM;
1003                                 goto ct_offload_err;
1004                         }
1005
1006                         flow_pay->nfp_tun_ipv6 = entry;
1007                 } else {
1008                         __be32 dst;
1009
1010                         for (i = 0; i < num_rules; i++) {
1011                                 nfp_flower_compile_ipv4_gre_tun((void *)key,
1012                                                                 (void *)msk, rules[i]);
1013                         }
1014                         dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst;
1015
1016                         /* Store the tunnel destination in the rule data.
1017                          * This must be present and be an exact match.
1018                          */
1019                         flow_pay->nfp_tun_ipv4_addr = dst;
1020                         nfp_tunnel_add_ipv4_off(priv->app, dst);
1021                 }
1022         }
1023
1024         if (key_layer.key_layer & NFP_FLOWER_LAYER_VXLAN ||
1025             key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE) {
1026                 offset = key_map[FLOW_PAY_UDP_TUN];
1027                 key = kdata + offset;
1028                 msk = mdata + offset;
1029                 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
1030                         struct nfp_flower_ipv6_udp_tun *udp_match;
1031                         struct nfp_ipv6_addr_entry *entry;
1032                         struct in6_addr *dst;
1033
1034                         for (i = 0; i < num_rules; i++) {
1035                                 nfp_flower_compile_ipv6_udp_tun((void *)key,
1036                                                                 (void *)msk, rules[i]);
1037                         }
1038                         udp_match = (struct nfp_flower_ipv6_udp_tun *)key;
1039                         dst = &udp_match->ipv6.dst;
1040
1041                         entry = nfp_tunnel_add_ipv6_off(priv->app, dst);
1042                         if (!entry) {
1043                                 err = -ENOMEM;
1044                                 goto ct_offload_err;
1045                         }
1046
1047                         flow_pay->nfp_tun_ipv6 = entry;
1048                 } else {
1049                         __be32 dst;
1050
1051                         for (i = 0; i < num_rules; i++) {
1052                                 nfp_flower_compile_ipv4_udp_tun((void *)key,
1053                                                                 (void *)msk, rules[i]);
1054                         }
1055                         dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst;
1056
1057                         /* Store the tunnel destination in the rule data.
1058                          * This must be present and be an exact match.
1059                          */
1060                         flow_pay->nfp_tun_ipv4_addr = dst;
1061                         nfp_tunnel_add_ipv4_off(priv->app, dst);
1062                 }
1063
1064                 if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
1065                         offset = key_map[FLOW_PAY_GENEVE_OPT];
1066                         key = kdata + offset;
1067                         msk = mdata + offset;
1068                         for (i = 0; i < num_rules; i++)
1069                                 nfp_flower_compile_geneve_opt(key, msk, rules[i]);
1070                 }
1071         }
1072
1073         /* Merge actions into flow_pay */
1074         err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules);
1075         if (err)
1076                 goto ct_offload_err;
1077
1078         /* Use the pointer address as the cookie, but set the last bit to 1.
1079          * This is to avoid the 'is_merge_flow' check from detecting this as
1080          * an already merged flow. This works since address alignment means
1081          * that the last bit for pointer addresses will be 0.
1082          */
1083         flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1;
1084         err = nfp_compile_flow_metadata(priv->app, flow_pay->tc_flower_cookie,
1085                                         flow_pay, netdev, NULL);
1086         if (err)
1087                 goto ct_offload_err;
1088
1089         if (nfp_netdev_is_nfp_repr(netdev))
1090                 port = nfp_port_from_netdev(netdev);
1091
1092         err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node,
1093                                      nfp_flower_table_params);
1094         if (err)
1095                 goto ct_release_offload_meta_err;
1096
1097         err = nfp_flower_xmit_flow(priv->app, flow_pay,
1098                                    NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
1099         if (err)
1100                 goto ct_remove_rhash_err;
1101
1102         m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie;
1103         m_entry->flow_pay = flow_pay;
1104
1105         if (port)
1106                 port->tc_offload_cnt++;
1107
1108         return err;
1109
1110 ct_remove_rhash_err:
1111         WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1112                                             &flow_pay->fl_node,
1113                                             nfp_flower_table_params));
1114 ct_release_offload_meta_err:
1115         nfp_modify_flow_metadata(priv->app, flow_pay);
1116 ct_offload_err:
1117         if (flow_pay->nfp_tun_ipv4_addr)
1118                 nfp_tunnel_del_ipv4_off(priv->app, flow_pay->nfp_tun_ipv4_addr);
1119         if (flow_pay->nfp_tun_ipv6)
1120                 nfp_tunnel_put_ipv6_off(priv->app, flow_pay->nfp_tun_ipv6);
1121         kfree(flow_pay->action_data);
1122         kfree(flow_pay->mask_data);
1123         kfree(flow_pay->unmasked_data);
1124         kfree(flow_pay);
1125         return err;
1126 }
1127
1128 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
1129                                  struct net_device *netdev)
1130 {
1131         struct nfp_flower_priv *priv = app->priv;
1132         struct nfp_fl_payload *flow_pay;
1133         struct nfp_port *port = NULL;
1134         int err = 0;
1135
1136         if (nfp_netdev_is_nfp_repr(netdev))
1137                 port = nfp_port_from_netdev(netdev);
1138
1139         flow_pay = nfp_flower_search_fl_table(app, cookie, netdev);
1140         if (!flow_pay)
1141                 return -ENOENT;
1142
1143         err = nfp_modify_flow_metadata(app, flow_pay);
1144         if (err)
1145                 goto err_free_merge_flow;
1146
1147         if (flow_pay->nfp_tun_ipv4_addr)
1148                 nfp_tunnel_del_ipv4_off(app, flow_pay->nfp_tun_ipv4_addr);
1149
1150         if (flow_pay->nfp_tun_ipv6)
1151                 nfp_tunnel_put_ipv6_off(app, flow_pay->nfp_tun_ipv6);
1152
1153         if (!flow_pay->in_hw) {
1154                 err = 0;
1155                 goto err_free_merge_flow;
1156         }
1157
1158         err = nfp_flower_xmit_flow(app, flow_pay,
1159                                    NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
1160
1161 err_free_merge_flow:
1162         nfp_flower_del_linked_merge_flows(app, flow_pay);
1163         if (port)
1164                 port->tc_offload_cnt--;
1165         kfree(flow_pay->action_data);
1166         kfree(flow_pay->mask_data);
1167         kfree(flow_pay->unmasked_data);
1168         WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1169                                             &flow_pay->fl_node,
1170                                             nfp_flower_table_params));
1171         kfree_rcu(flow_pay, rcu);
1172         return err;
1173 }
1174
1175 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
1176                                struct nfp_fl_ct_flow_entry *nft_entry,
1177                                struct nfp_fl_ct_tc_merge *tc_m_entry)
1178 {
1179         struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1180         struct nfp_fl_nft_tc_merge *nft_m_entry;
1181         unsigned long new_cookie[3];
1182         int err;
1183
1184         pre_ct_entry = tc_m_entry->pre_ct_parent;
1185         post_ct_entry = tc_m_entry->post_ct_parent;
1186
1187         err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
1188         if (err)
1189                 return err;
1190
1191         /* Check that the two tc flows are also compatible with
1192          * the nft entry. No need to check the pre_ct and post_ct
1193          * entries as that was already done during pre_merge.
1194          * The nft entry does not have a chain populated, so
1195          * skip this check.
1196          */
1197         err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
1198         if (err)
1199                 return err;
1200         err = nfp_ct_merge_check(nft_entry, post_ct_entry);
1201         if (err)
1202                 return err;
1203         err = nfp_ct_check_meta(post_ct_entry, nft_entry);
1204         if (err)
1205                 return err;
1206
1207         if (pre_ct_entry->num_prev_m_entries > 0) {
1208                 err = nfp_ct_merge_extra_check(nft_entry, tc_m_entry);
1209                 if (err)
1210                         return err;
1211         }
1212
1213         /* Combine tc_merge and nft cookies for this cookie. */
1214         new_cookie[0] = tc_m_entry->cookie[0];
1215         new_cookie[1] = tc_m_entry->cookie[1];
1216         new_cookie[2] = nft_entry->cookie;
1217         nft_m_entry = get_hashentry(&zt->nft_merge_tb,
1218                                     &new_cookie,
1219                                     nfp_nft_ct_merge_params,
1220                                     sizeof(*nft_m_entry));
1221
1222         if (IS_ERR(nft_m_entry))
1223                 return PTR_ERR(nft_m_entry);
1224
1225         /* nft_m_entry already present, not merging again */
1226         if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
1227                 return 0;
1228
1229         memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
1230         nft_m_entry->zt = zt;
1231         nft_m_entry->tc_m_parent = tc_m_entry;
1232         nft_m_entry->nft_parent = nft_entry;
1233         nft_m_entry->tc_flower_cookie = 0;
1234         /* Copy the netdev from the pre_ct entry. When the tc_m_entry was created
1235          * it only combined them if the netdevs were the same, so can use any of them.
1236          */
1237         nft_m_entry->netdev = pre_ct_entry->netdev;
1238
1239         /* Add this entry to the tc_m_list and nft_flow lists */
1240         list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
1241         list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
1242
1243         err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
1244                                      nfp_nft_ct_merge_params);
1245         if (err)
1246                 goto err_nft_ct_merge_insert;
1247
1248         zt->nft_merge_count++;
1249
1250         if (post_ct_entry->goto_chain_index > 0)
1251                 return nfp_fl_create_new_pre_ct(nft_m_entry);
1252
1253         /* Generate offload structure and send to nfp */
1254         err = nfp_fl_ct_add_offload(nft_m_entry);
1255         if (err)
1256                 goto err_nft_ct_offload;
1257
1258         return err;
1259
1260 err_nft_ct_offload:
1261         nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
1262                               nft_m_entry->netdev);
1263 err_nft_ct_merge_insert:
1264         list_del(&nft_m_entry->tc_merge_list);
1265         list_del(&nft_m_entry->nft_flow_list);
1266         kfree(nft_m_entry);
1267         return err;
1268 }
1269
1270 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
1271                               struct nfp_fl_ct_flow_entry *ct_entry1,
1272                               struct nfp_fl_ct_flow_entry *ct_entry2)
1273 {
1274         struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1275         struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
1276         struct nfp_fl_ct_tc_merge *m_entry;
1277         unsigned long new_cookie[2];
1278         int err;
1279
1280         if (ct_entry1->type == CT_TYPE_PRE_CT) {
1281                 pre_ct_entry = ct_entry1;
1282                 post_ct_entry = ct_entry2;
1283         } else {
1284                 post_ct_entry = ct_entry1;
1285                 pre_ct_entry = ct_entry2;
1286         }
1287
1288         /* Checks that the chain_index of the filter matches the
1289          * chain_index of the GOTO action.
1290          */
1291         if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index)
1292                 return -EINVAL;
1293
1294         err = nfp_ct_merge_check(pre_ct_entry, post_ct_entry);
1295         if (err)
1296                 return err;
1297
1298         new_cookie[0] = pre_ct_entry->cookie;
1299         new_cookie[1] = post_ct_entry->cookie;
1300         m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
1301                                 nfp_tc_ct_merge_params, sizeof(*m_entry));
1302         if (IS_ERR(m_entry))
1303                 return PTR_ERR(m_entry);
1304
1305         /* m_entry already present, not merging again */
1306         if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
1307                 return 0;
1308
1309         memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
1310         m_entry->zt = zt;
1311         m_entry->post_ct_parent = post_ct_entry;
1312         m_entry->pre_ct_parent = pre_ct_entry;
1313
1314         /* Add this entry to the pre_ct and post_ct lists */
1315         list_add(&m_entry->post_ct_list, &post_ct_entry->children);
1316         list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
1317         INIT_LIST_HEAD(&m_entry->children);
1318
1319         err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
1320                                      nfp_tc_ct_merge_params);
1321         if (err)
1322                 goto err_ct_tc_merge_insert;
1323         zt->tc_merge_count++;
1324
1325         /* Merge with existing nft flows */
1326         list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
1327                                  list_node) {
1328                 nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
1329         }
1330
1331         return 0;
1332
1333 err_ct_tc_merge_insert:
1334         list_del(&m_entry->post_ct_list);
1335         list_del(&m_entry->pre_ct_list);
1336         kfree(m_entry);
1337         return err;
1338 }
1339
1340 static struct
1341 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
1342                                          u16 zone, bool wildcarded)
1343 {
1344         struct nfp_fl_ct_zone_entry *zt;
1345         int err;
1346
1347         if (wildcarded && priv->ct_zone_wc)
1348                 return priv->ct_zone_wc;
1349
1350         if (!wildcarded) {
1351                 zt = get_hashentry(&priv->ct_zone_table, &zone,
1352                                    nfp_zone_table_params, sizeof(*zt));
1353
1354                 /* If priv is set this is an existing entry, just return it */
1355                 if (IS_ERR(zt) || zt->priv)
1356                         return zt;
1357         } else {
1358                 zt = kzalloc(sizeof(*zt), GFP_KERNEL);
1359                 if (!zt)
1360                         return ERR_PTR(-ENOMEM);
1361         }
1362
1363         zt->zone = zone;
1364         zt->priv = priv;
1365         zt->nft = NULL;
1366
1367         /* init the various hash tables and lists */
1368         INIT_LIST_HEAD(&zt->pre_ct_list);
1369         INIT_LIST_HEAD(&zt->post_ct_list);
1370         INIT_LIST_HEAD(&zt->nft_flows_list);
1371
1372         err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
1373         if (err)
1374                 goto err_tc_merge_tb_init;
1375
1376         err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
1377         if (err)
1378                 goto err_nft_merge_tb_init;
1379
1380         if (wildcarded) {
1381                 priv->ct_zone_wc = zt;
1382         } else {
1383                 err = rhashtable_insert_fast(&priv->ct_zone_table,
1384                                              &zt->hash_node,
1385                                              nfp_zone_table_params);
1386                 if (err)
1387                         goto err_zone_insert;
1388         }
1389
1390         return zt;
1391
1392 err_zone_insert:
1393         rhashtable_destroy(&zt->nft_merge_tb);
1394 err_nft_merge_tb_init:
1395         rhashtable_destroy(&zt->tc_merge_tb);
1396 err_tc_merge_tb_init:
1397         kfree(zt);
1398         return ERR_PTR(err);
1399 }
1400
1401 static struct net_device *get_netdev_from_rule(struct flow_rule *rule)
1402 {
1403         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
1404                 struct flow_match_meta match;
1405
1406                 flow_rule_match_meta(rule, &match);
1407                 if (match.key->ingress_ifindex & match.mask->ingress_ifindex)
1408                         return __dev_get_by_index(&init_net,
1409                                                   match.key->ingress_ifindex);
1410         }
1411
1412         return NULL;
1413 }
1414
1415 static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action)
1416 {
1417         if (mangle_action->id != FLOW_ACTION_MANGLE)
1418                 return;
1419
1420         switch (mangle_action->mangle.htype) {
1421         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1422         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1423                 mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val);
1424                 mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask);
1425                 return;
1426
1427         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
1428         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
1429                 mangle_action->mangle.val = (__force u16)cpu_to_be16(mangle_action->mangle.val);
1430                 mangle_action->mangle.mask = (__force u16)cpu_to_be16(mangle_action->mangle.mask);
1431                 return;
1432
1433         default:
1434                 return;
1435         }
1436 }
1437
1438 static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act,
1439                                     struct nfp_fl_ct_flow_entry *entry)
1440 {
1441         switch (act->id) {
1442         case FLOW_ACTION_CT:
1443                 if (act->ct.action == TCA_CT_ACT_NAT)
1444                         entry->flags |= NFP_FL_ACTION_DO_NAT;
1445                 break;
1446
1447         case FLOW_ACTION_MANGLE:
1448                 entry->flags |= NFP_FL_ACTION_DO_MANGLE;
1449                 break;
1450
1451         default:
1452                 break;
1453         }
1454
1455         return 0;
1456 }
1457
1458 static struct
1459 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
1460                                          struct net_device *netdev,
1461                                          struct flow_cls_offload *flow,
1462                                          bool is_nft, struct netlink_ext_ack *extack)
1463 {
1464         struct nf_flow_match *nft_match = NULL;
1465         struct nfp_fl_ct_flow_entry *entry;
1466         struct nfp_fl_ct_map_entry *map;
1467         struct flow_action_entry *act;
1468         int err, i;
1469
1470         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1471         if (!entry)
1472                 return ERR_PTR(-ENOMEM);
1473
1474         entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
1475         if (!entry->rule) {
1476                 err = -ENOMEM;
1477                 goto err_pre_ct_rule;
1478         }
1479
1480         /* nft flows gets destroyed after callback return, so need
1481          * to do a full copy instead of just a reference.
1482          */
1483         if (is_nft) {
1484                 nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
1485                 if (!nft_match) {
1486                         err = -ENOMEM;
1487                         goto err_pre_ct_act;
1488                 }
1489                 memcpy(&nft_match->dissector, flow->rule->match.dissector,
1490                        sizeof(nft_match->dissector));
1491                 memcpy(&nft_match->mask, flow->rule->match.mask,
1492                        sizeof(nft_match->mask));
1493                 memcpy(&nft_match->key, flow->rule->match.key,
1494                        sizeof(nft_match->key));
1495                 entry->rule->match.dissector = &nft_match->dissector;
1496                 entry->rule->match.mask = &nft_match->mask;
1497                 entry->rule->match.key = &nft_match->key;
1498
1499                 if (!netdev)
1500                         netdev = get_netdev_from_rule(entry->rule);
1501         } else {
1502                 entry->rule->match.dissector = flow->rule->match.dissector;
1503                 entry->rule->match.mask = flow->rule->match.mask;
1504                 entry->rule->match.key = flow->rule->match.key;
1505         }
1506
1507         entry->zt = zt;
1508         entry->netdev = netdev;
1509         entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry;
1510         entry->chain_index = flow->common.chain_index;
1511         entry->tun_offset = NFP_FL_CT_NO_TUN;
1512
1513         /* Copy over action data. Unfortunately we do not get a handle to the
1514          * original tcf_action data, and the flow objects gets destroyed, so we
1515          * cannot just save a pointer to this either, so need to copy over the
1516          * data unfortunately.
1517          */
1518         entry->rule->action.num_entries = flow->rule->action.num_entries;
1519         flow_action_for_each(i, act, &flow->rule->action) {
1520                 struct flow_action_entry *new_act;
1521
1522                 new_act = &entry->rule->action.entries[i];
1523                 memcpy(new_act, act, sizeof(struct flow_action_entry));
1524                 /* nft entry mangle field is host byte order, need translate to
1525                  * network byte order.
1526                  */
1527                 if (is_nft)
1528                         nfp_nft_ct_translate_mangle_action(new_act);
1529
1530                 nfp_nft_ct_set_flow_flag(new_act, entry);
1531                 /* Entunnel is a special case, need to allocate and copy
1532                  * tunnel info.
1533                  */
1534                 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
1535                         struct ip_tunnel_info *tun = act->tunnel;
1536                         size_t tun_size = sizeof(*tun) + tun->options_len;
1537
1538                         new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
1539                         if (!new_act->tunnel) {
1540                                 err = -ENOMEM;
1541                                 goto err_pre_ct_tun_cp;
1542                         }
1543                         entry->tun_offset = i;
1544                 }
1545         }
1546
1547         INIT_LIST_HEAD(&entry->children);
1548
1549         if (flow->cookie == 0)
1550                 return entry;
1551
1552         /* Now add a ct map entry to flower-priv */
1553         map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
1554                             nfp_ct_map_params, sizeof(*map));
1555         if (IS_ERR(map)) {
1556                 NL_SET_ERR_MSG_MOD(extack,
1557                                    "offload error: ct map entry creation failed");
1558                 err = -ENOMEM;
1559                 goto err_ct_flow_insert;
1560         }
1561         map->cookie = flow->cookie;
1562         map->ct_entry = entry;
1563         err = rhashtable_insert_fast(&zt->priv->ct_map_table,
1564                                      &map->hash_node,
1565                                      nfp_ct_map_params);
1566         if (err) {
1567                 NL_SET_ERR_MSG_MOD(extack,
1568                                    "offload error: ct map entry table add failed");
1569                 goto err_map_insert;
1570         }
1571
1572         return entry;
1573
1574 err_map_insert:
1575         kfree(map);
1576 err_ct_flow_insert:
1577         if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1578                 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1579 err_pre_ct_tun_cp:
1580         kfree(nft_match);
1581 err_pre_ct_act:
1582         kfree(entry->rule);
1583 err_pre_ct_rule:
1584         kfree(entry);
1585         return ERR_PTR(err);
1586 }
1587
1588 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
1589 {
1590         struct nfp_fl_ct_zone_entry *zt;
1591         int err;
1592
1593         zt = m_entry->zt;
1594
1595         /* Flow is in HW, need to delete */
1596         if (m_entry->tc_flower_cookie) {
1597                 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
1598                                             m_entry->netdev);
1599                 if (err)
1600                         return;
1601         }
1602
1603         WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
1604                                             &m_entry->hash_node,
1605                                             nfp_nft_ct_merge_params));
1606         zt->nft_merge_count--;
1607         list_del(&m_entry->tc_merge_list);
1608         list_del(&m_entry->nft_flow_list);
1609
1610         if (m_entry->next_pre_ct_entry) {
1611                 struct nfp_fl_ct_map_entry pre_ct_map_ent;
1612
1613                 pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry;
1614                 pre_ct_map_ent.cookie = 0;
1615                 nfp_fl_ct_del_flow(&pre_ct_map_ent);
1616         }
1617
1618         kfree(m_entry);
1619 }
1620
1621 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
1622 {
1623         struct nfp_fl_nft_tc_merge *m_entry, *tmp;
1624
1625         /* These post entries are parts of two lists, one is a list of nft_entries
1626          * and the other is of from a list of tc_merge structures. Iterate
1627          * through the relevant list and cleanup the entries.
1628          */
1629
1630         if (is_nft_flow) {
1631                 /* Need to iterate through list of nft_flow entries */
1632                 struct nfp_fl_ct_flow_entry *ct_entry = entry;
1633
1634                 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1635                                          nft_flow_list) {
1636                         cleanup_nft_merge_entry(m_entry);
1637                 }
1638         } else {
1639                 /* Need to iterate through list of tc_merged_flow entries */
1640                 struct nfp_fl_ct_tc_merge *ct_entry = entry;
1641
1642                 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1643                                          tc_merge_list) {
1644                         cleanup_nft_merge_entry(m_entry);
1645                 }
1646         }
1647 }
1648
1649 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
1650 {
1651         struct nfp_fl_ct_zone_entry *zt;
1652         int err;
1653
1654         zt = m_ent->zt;
1655         err = rhashtable_remove_fast(&zt->tc_merge_tb,
1656                                      &m_ent->hash_node,
1657                                      nfp_tc_ct_merge_params);
1658         if (err)
1659                 pr_warn("WARNING: could not remove merge_entry from hashtable\n");
1660         zt->tc_merge_count--;
1661         list_del(&m_ent->post_ct_list);
1662         list_del(&m_ent->pre_ct_list);
1663
1664         if (!list_empty(&m_ent->children))
1665                 nfp_free_nft_merge_children(m_ent, false);
1666         kfree(m_ent);
1667 }
1668
1669 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
1670 {
1671         struct nfp_fl_ct_tc_merge *m_ent, *tmp;
1672
1673         switch (entry->type) {
1674         case CT_TYPE_PRE_CT:
1675                 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
1676                         nfp_del_tc_merge_entry(m_ent);
1677                 }
1678                 break;
1679         case CT_TYPE_POST_CT:
1680                 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
1681                         nfp_del_tc_merge_entry(m_ent);
1682                 }
1683                 break;
1684         default:
1685                 break;
1686         }
1687 }
1688
1689 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
1690 {
1691         list_del(&entry->list_node);
1692
1693         if (!list_empty(&entry->children)) {
1694                 if (entry->type == CT_TYPE_NFT)
1695                         nfp_free_nft_merge_children(entry, true);
1696                 else
1697                         nfp_free_tc_merge_children(entry);
1698         }
1699
1700         if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1701                 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
1702
1703         if (entry->type == CT_TYPE_NFT) {
1704                 struct nf_flow_match *nft_match;
1705
1706                 nft_match = container_of(entry->rule->match.dissector,
1707                                          struct nf_flow_match, dissector);
1708                 kfree(nft_match);
1709         }
1710
1711         kfree(entry->rule);
1712         kfree(entry);
1713 }
1714
1715 static struct flow_action_entry *get_flow_act_ct(struct flow_rule *rule)
1716 {
1717         struct flow_action_entry *act;
1718         int i;
1719
1720         /* More than one ct action may be present in a flow rule,
1721          * Return the first one that is not a CT clear action
1722          */
1723         flow_action_for_each(i, act, &rule->action) {
1724                 if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR)
1725                         return act;
1726         }
1727
1728         return NULL;
1729 }
1730
1731 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
1732                                               enum flow_action_id act_id)
1733 {
1734         struct flow_action_entry *act = NULL;
1735         int i;
1736
1737         flow_action_for_each(i, act, &rule->action) {
1738                 if (act->id == act_id)
1739                         return act;
1740         }
1741         return NULL;
1742 }
1743
1744 static void
1745 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
1746                         struct nfp_fl_ct_zone_entry *zt_src,
1747                         struct nfp_fl_ct_zone_entry *zt_dst)
1748 {
1749         struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
1750         struct list_head *ct_list;
1751
1752         if (ct_entry1->type == CT_TYPE_PRE_CT)
1753                 ct_list = &zt_src->post_ct_list;
1754         else if (ct_entry1->type == CT_TYPE_POST_CT)
1755                 ct_list = &zt_src->pre_ct_list;
1756         else
1757                 return;
1758
1759         list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
1760                                  list_node) {
1761                 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
1762         }
1763 }
1764
1765 static void
1766 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
1767                          struct nfp_fl_ct_zone_entry *zt)
1768 {
1769         struct nfp_fl_ct_tc_merge *tc_merge_entry;
1770         struct rhashtable_iter iter;
1771
1772         rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
1773         rhashtable_walk_start(&iter);
1774         while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
1775                 if (IS_ERR(tc_merge_entry))
1776                         continue;
1777                 rhashtable_walk_stop(&iter);
1778                 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
1779                 rhashtable_walk_start(&iter);
1780         }
1781         rhashtable_walk_stop(&iter);
1782         rhashtable_walk_exit(&iter);
1783 }
1784
1785 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
1786                             struct net_device *netdev,
1787                             struct flow_cls_offload *flow,
1788                             struct netlink_ext_ack *extack,
1789                             struct nfp_fl_nft_tc_merge *m_entry)
1790 {
1791         struct flow_action_entry *ct_act, *ct_goto;
1792         struct nfp_fl_ct_flow_entry *ct_entry;
1793         struct nfp_fl_ct_zone_entry *zt;
1794         int err;
1795
1796         ct_act = get_flow_act_ct(flow->rule);
1797         if (!ct_act) {
1798                 NL_SET_ERR_MSG_MOD(extack,
1799                                    "unsupported offload: Conntrack action empty in conntrack offload");
1800                 return -EOPNOTSUPP;
1801         }
1802
1803         ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
1804         if (!ct_goto) {
1805                 NL_SET_ERR_MSG_MOD(extack,
1806                                    "unsupported offload: Conntrack requires ACTION_GOTO");
1807                 return -EOPNOTSUPP;
1808         }
1809
1810         zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
1811         if (IS_ERR(zt)) {
1812                 NL_SET_ERR_MSG_MOD(extack,
1813                                    "offload error: Could not create zone table entry");
1814                 return PTR_ERR(zt);
1815         }
1816
1817         if (!zt->nft) {
1818                 zt->nft = ct_act->ct.flow_table;
1819                 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
1820                 if (err) {
1821                         NL_SET_ERR_MSG_MOD(extack,
1822                                            "offload error: Could not register nft_callback");
1823                         return err;
1824                 }
1825         }
1826
1827         /* Add entry to pre_ct_list */
1828         ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1829         if (IS_ERR(ct_entry))
1830                 return PTR_ERR(ct_entry);
1831         ct_entry->type = CT_TYPE_PRE_CT;
1832         ct_entry->chain_index = flow->common.chain_index;
1833         ct_entry->goto_chain_index = ct_goto->chain_index;
1834
1835         if (m_entry) {
1836                 struct nfp_fl_ct_flow_entry *pre_ct_entry;
1837                 int i;
1838
1839                 pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1840                 for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++)
1841                         ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i];
1842                 ct_entry->prev_m_entries[i++] = m_entry;
1843                 ct_entry->num_prev_m_entries = i;
1844
1845                 m_entry->next_pre_ct_entry = ct_entry;
1846         }
1847
1848         list_add(&ct_entry->list_node, &zt->pre_ct_list);
1849         zt->pre_ct_count++;
1850
1851         nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1852
1853         /* Need to check and merge with tables in the wc_zone as well */
1854         if (priv->ct_zone_wc)
1855                 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
1856
1857         return 0;
1858 }
1859
1860 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
1861                              struct net_device *netdev,
1862                              struct flow_cls_offload *flow,
1863                              struct netlink_ext_ack *extack)
1864 {
1865         struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
1866         struct nfp_fl_ct_flow_entry *ct_entry;
1867         struct nfp_fl_ct_zone_entry *zt;
1868         bool wildcarded = false;
1869         struct flow_match_ct ct;
1870         struct flow_action_entry *ct_goto;
1871
1872         flow_rule_match_ct(rule, &ct);
1873         if (!ct.mask->ct_zone) {
1874                 wildcarded = true;
1875         } else if (ct.mask->ct_zone != U16_MAX) {
1876                 NL_SET_ERR_MSG_MOD(extack,
1877                                    "unsupported offload: partially wildcarded ct_zone is not supported");
1878                 return -EOPNOTSUPP;
1879         }
1880
1881         zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1882         if (IS_ERR(zt)) {
1883                 NL_SET_ERR_MSG_MOD(extack,
1884                                    "offload error: Could not create zone table entry");
1885                 return PTR_ERR(zt);
1886         }
1887
1888         /* Add entry to post_ct_list */
1889         ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1890         if (IS_ERR(ct_entry))
1891                 return PTR_ERR(ct_entry);
1892
1893         ct_entry->type = CT_TYPE_POST_CT;
1894         ct_entry->chain_index = flow->common.chain_index;
1895         ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
1896         ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0;
1897         list_add(&ct_entry->list_node, &zt->post_ct_list);
1898         zt->post_ct_count++;
1899
1900         if (wildcarded) {
1901                 /* Iterate through all zone tables if not empty, look for merges with
1902                  * pre_ct entries and merge them.
1903                  */
1904                 struct rhashtable_iter iter;
1905                 struct nfp_fl_ct_zone_entry *zone_table;
1906
1907                 rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1908                 rhashtable_walk_start(&iter);
1909                 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1910                         if (IS_ERR(zone_table))
1911                                 continue;
1912                         rhashtable_walk_stop(&iter);
1913                         nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1914                         rhashtable_walk_start(&iter);
1915                 }
1916                 rhashtable_walk_stop(&iter);
1917                 rhashtable_walk_exit(&iter);
1918         } else {
1919                 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1920         }
1921
1922         return 0;
1923 }
1924
1925 int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry)
1926 {
1927         struct nfp_fl_ct_flow_entry *pre_ct_entry, *post_ct_entry;
1928         struct flow_cls_offload new_pre_ct_flow;
1929         int err;
1930
1931         pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1932         if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1)
1933                 return -1;
1934
1935         post_ct_entry = m_entry->tc_m_parent->post_ct_parent;
1936         memset(&new_pre_ct_flow, 0, sizeof(struct flow_cls_offload));
1937         new_pre_ct_flow.rule = post_ct_entry->rule;
1938         new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index;
1939
1940         err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv,
1941                                       pre_ct_entry->netdev,
1942                                       &new_pre_ct_flow, NULL,
1943                                       m_entry);
1944         return err;
1945 }
1946
1947 static void
1948 nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
1949                     enum ct_entry_type type, u64 *m_pkts,
1950                     u64 *m_bytes, u64 *m_used)
1951 {
1952         struct nfp_flower_priv *priv = nft_merge->zt->priv;
1953         struct nfp_fl_payload *nfp_flow;
1954         u32 ctx_id;
1955
1956         nfp_flow = nft_merge->flow_pay;
1957         if (!nfp_flow)
1958                 return;
1959
1960         ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
1961         *m_pkts += priv->stats[ctx_id].pkts;
1962         *m_bytes += priv->stats[ctx_id].bytes;
1963         *m_used = max_t(u64, *m_used, priv->stats[ctx_id].used);
1964
1965         /* If request is for a sub_flow which is part of a tunnel merged
1966          * flow then update stats from tunnel merged flows first.
1967          */
1968         if (!list_empty(&nfp_flow->linked_flows))
1969                 nfp_flower_update_merge_stats(priv->app, nfp_flow);
1970
1971         if (type != CT_TYPE_NFT) {
1972                 /* Update nft cached stats */
1973                 flow_stats_update(&nft_merge->nft_parent->stats,
1974                                   priv->stats[ctx_id].bytes,
1975                                   priv->stats[ctx_id].pkts,
1976                                   0, priv->stats[ctx_id].used,
1977                                   FLOW_ACTION_HW_STATS_DELAYED);
1978         } else {
1979                 /* Update pre_ct cached stats */
1980                 flow_stats_update(&nft_merge->tc_m_parent->pre_ct_parent->stats,
1981                                   priv->stats[ctx_id].bytes,
1982                                   priv->stats[ctx_id].pkts,
1983                                   0, priv->stats[ctx_id].used,
1984                                   FLOW_ACTION_HW_STATS_DELAYED);
1985                 /* Update post_ct cached stats */
1986                 flow_stats_update(&nft_merge->tc_m_parent->post_ct_parent->stats,
1987                                   priv->stats[ctx_id].bytes,
1988                                   priv->stats[ctx_id].pkts,
1989                                   0, priv->stats[ctx_id].used,
1990                                   FLOW_ACTION_HW_STATS_DELAYED);
1991         }
1992
1993         /* Update previous pre_ct/post_ct/nft flow stats */
1994         if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) {
1995                 struct nfp_fl_nft_tc_merge *tmp_nft_merge;
1996                 int i;
1997
1998                 for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) {
1999                         tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i];
2000                         flow_stats_update(&tmp_nft_merge->tc_m_parent->pre_ct_parent->stats,
2001                                           priv->stats[ctx_id].bytes,
2002                                           priv->stats[ctx_id].pkts,
2003                                           0, priv->stats[ctx_id].used,
2004                                           FLOW_ACTION_HW_STATS_DELAYED);
2005                         flow_stats_update(&tmp_nft_merge->tc_m_parent->post_ct_parent->stats,
2006                                           priv->stats[ctx_id].bytes,
2007                                           priv->stats[ctx_id].pkts,
2008                                           0, priv->stats[ctx_id].used,
2009                                           FLOW_ACTION_HW_STATS_DELAYED);
2010                         flow_stats_update(&tmp_nft_merge->nft_parent->stats,
2011                                           priv->stats[ctx_id].bytes,
2012                                           priv->stats[ctx_id].pkts,
2013                                           0, priv->stats[ctx_id].used,
2014                                           FLOW_ACTION_HW_STATS_DELAYED);
2015                 }
2016         }
2017
2018         /* Reset stats from the nfp */
2019         priv->stats[ctx_id].pkts = 0;
2020         priv->stats[ctx_id].bytes = 0;
2021 }
2022
2023 int nfp_fl_ct_stats(struct flow_cls_offload *flow,
2024                     struct nfp_fl_ct_map_entry *ct_map_ent)
2025 {
2026         struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry;
2027         struct nfp_fl_nft_tc_merge *nft_merge, *nft_m_tmp;
2028         struct nfp_fl_ct_tc_merge *tc_merge, *tc_m_tmp;
2029
2030         u64 pkts = 0, bytes = 0, used = 0;
2031         u64 m_pkts, m_bytes, m_used;
2032
2033         spin_lock_bh(&ct_entry->zt->priv->stats_lock);
2034
2035         if (ct_entry->type == CT_TYPE_PRE_CT) {
2036                 /* Iterate tc_merge entries associated with this flow */
2037                 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2038                                          pre_ct_list) {
2039                         m_pkts = 0;
2040                         m_bytes = 0;
2041                         m_used = 0;
2042                         /* Iterate nft_merge entries associated with this tc_merge flow */
2043                         list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2044                                                  tc_merge_list) {
2045                                 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_PRE_CT,
2046                                                     &m_pkts, &m_bytes, &m_used);
2047                         }
2048                         pkts += m_pkts;
2049                         bytes += m_bytes;
2050                         used = max_t(u64, used, m_used);
2051                         /* Update post_ct partner */
2052                         flow_stats_update(&tc_merge->post_ct_parent->stats,
2053                                           m_bytes, m_pkts, 0, m_used,
2054                                           FLOW_ACTION_HW_STATS_DELAYED);
2055                 }
2056         } else if (ct_entry->type == CT_TYPE_POST_CT) {
2057                 /* Iterate tc_merge entries associated with this flow */
2058                 list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2059                                          post_ct_list) {
2060                         m_pkts = 0;
2061                         m_bytes = 0;
2062                         m_used = 0;
2063                         /* Iterate nft_merge entries associated with this tc_merge flow */
2064                         list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2065                                                  tc_merge_list) {
2066                                 nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_POST_CT,
2067                                                     &m_pkts, &m_bytes, &m_used);
2068                         }
2069                         pkts += m_pkts;
2070                         bytes += m_bytes;
2071                         used = max_t(u64, used, m_used);
2072                         /* Update pre_ct partner */
2073                         flow_stats_update(&tc_merge->pre_ct_parent->stats,
2074                                           m_bytes, m_pkts, 0, m_used,
2075                                           FLOW_ACTION_HW_STATS_DELAYED);
2076                 }
2077         } else  {
2078                 /* Iterate nft_merge entries associated with this nft flow */
2079                 list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children,
2080                                          nft_flow_list) {
2081                         nfp_fl_ct_sub_stats(nft_merge, CT_TYPE_NFT,
2082                                             &pkts, &bytes, &used);
2083                 }
2084         }
2085
2086         /* Add stats from this request to stats potentially cached by
2087          * previous requests.
2088          */
2089         flow_stats_update(&ct_entry->stats, bytes, pkts, 0, used,
2090                           FLOW_ACTION_HW_STATS_DELAYED);
2091         /* Finally update the flow stats from the original stats request */
2092         flow_stats_update(&flow->stats, ct_entry->stats.bytes,
2093                           ct_entry->stats.pkts, 0,
2094                           ct_entry->stats.lastused,
2095                           FLOW_ACTION_HW_STATS_DELAYED);
2096         /* Stats has been synced to original flow, can now clear
2097          * the cache.
2098          */
2099         ct_entry->stats.pkts = 0;
2100         ct_entry->stats.bytes = 0;
2101         spin_unlock_bh(&ct_entry->zt->priv->stats_lock);
2102
2103         return 0;
2104 }
2105
2106 static bool
2107 nfp_fl_ct_offload_nft_supported(struct flow_cls_offload *flow)
2108 {
2109         struct flow_rule *flow_rule = flow->rule;
2110         struct flow_action *flow_action =
2111                 &flow_rule->action;
2112         struct flow_action_entry *act;
2113         int i;
2114
2115         flow_action_for_each(i, act, flow_action) {
2116                 if (act->id == FLOW_ACTION_CT_METADATA) {
2117                         enum ip_conntrack_info ctinfo =
2118                                 act->ct_metadata.cookie & NFCT_INFOMASK;
2119
2120                         return ctinfo != IP_CT_NEW;
2121                 }
2122         }
2123
2124         return false;
2125 }
2126
2127 static int
2128 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
2129 {
2130         struct nfp_fl_ct_map_entry *ct_map_ent;
2131         struct nfp_fl_ct_flow_entry *ct_entry;
2132         struct netlink_ext_ack *extack = NULL;
2133
2134         extack = flow->common.extack;
2135         switch (flow->command) {
2136         case FLOW_CLS_REPLACE:
2137                 if (!nfp_fl_ct_offload_nft_supported(flow))
2138                         return -EOPNOTSUPP;
2139
2140                 /* Netfilter can request offload multiple times for the same
2141                  * flow - protect against adding duplicates.
2142                  */
2143                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2144                                                     nfp_ct_map_params);
2145                 if (!ct_map_ent) {
2146                         ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
2147                         if (IS_ERR(ct_entry))
2148                                 return PTR_ERR(ct_entry);
2149                         ct_entry->type = CT_TYPE_NFT;
2150                         list_add(&ct_entry->list_node, &zt->nft_flows_list);
2151                         zt->nft_flows_count++;
2152                         nfp_ct_merge_nft_with_tc(ct_entry, zt);
2153                 }
2154                 return 0;
2155         case FLOW_CLS_DESTROY:
2156                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2157                                                     nfp_ct_map_params);
2158                 return nfp_fl_ct_del_flow(ct_map_ent);
2159         case FLOW_CLS_STATS:
2160                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
2161                                                     nfp_ct_map_params);
2162                 if (ct_map_ent)
2163                         return nfp_fl_ct_stats(flow, ct_map_ent);
2164                 break;
2165         default:
2166                 break;
2167         }
2168         return -EINVAL;
2169 }
2170
2171 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
2172 {
2173         struct flow_cls_offload *flow = type_data;
2174         struct nfp_fl_ct_zone_entry *zt = cb_priv;
2175         int err = -EOPNOTSUPP;
2176
2177         switch (type) {
2178         case TC_SETUP_CLSFLOWER:
2179                 while (!mutex_trylock(&zt->priv->nfp_fl_lock)) {
2180                         if (!zt->nft) /* avoid deadlock */
2181                                 return err;
2182                         msleep(20);
2183                 }
2184                 err = nfp_fl_ct_offload_nft_flow(zt, flow);
2185                 mutex_unlock(&zt->priv->nfp_fl_lock);
2186                 break;
2187         default:
2188                 return -EOPNOTSUPP;
2189         }
2190         return err;
2191 }
2192
2193 static void
2194 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
2195 {
2196         struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
2197         struct nfp_fl_ct_map_entry *ct_map_ent;
2198
2199         list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
2200                                  list_node) {
2201                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
2202                                                     &nft_entry->cookie,
2203                                                     nfp_ct_map_params);
2204                 nfp_fl_ct_del_flow(ct_map_ent);
2205         }
2206 }
2207
2208 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
2209 {
2210         struct nfp_fl_ct_flow_entry *ct_entry;
2211         struct nfp_fl_ct_zone_entry *zt;
2212         struct rhashtable *m_table;
2213         struct nf_flowtable *nft;
2214
2215         if (!ct_map_ent)
2216                 return -ENOENT;
2217
2218         zt = ct_map_ent->ct_entry->zt;
2219         ct_entry = ct_map_ent->ct_entry;
2220         m_table = &zt->priv->ct_map_table;
2221
2222         switch (ct_entry->type) {
2223         case CT_TYPE_PRE_CT:
2224                 zt->pre_ct_count--;
2225                 if (ct_map_ent->cookie > 0)
2226                         rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2227                                                nfp_ct_map_params);
2228                 nfp_fl_ct_clean_flow_entry(ct_entry);
2229                 if (ct_map_ent->cookie > 0)
2230                         kfree(ct_map_ent);
2231
2232                 if (!zt->pre_ct_count && zt->nft) {
2233                         nft = zt->nft;
2234                         zt->nft = NULL; /* avoid deadlock */
2235                         nf_flow_table_offload_del_cb(nft,
2236                                                      nfp_fl_ct_handle_nft_flow,
2237                                                      zt);
2238                         nfp_fl_ct_clean_nft_entries(zt);
2239                 }
2240                 break;
2241         case CT_TYPE_POST_CT:
2242                 zt->post_ct_count--;
2243                 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2244                                        nfp_ct_map_params);
2245                 nfp_fl_ct_clean_flow_entry(ct_entry);
2246                 kfree(ct_map_ent);
2247                 break;
2248         case CT_TYPE_NFT:
2249                 zt->nft_flows_count--;
2250                 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
2251                                        nfp_ct_map_params);
2252                 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
2253                 kfree(ct_map_ent);
2254                 break;
2255         default:
2256                 break;
2257         }
2258
2259         return 0;
2260 }