drivers/net/ethernet/netronome/nfp/flower/metadata.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
   3
   4 #include <linux/hash.h>
   5 #include <linux/hashtable.h>
   6 #include <linux/jhash.h>
   7 #include <linux/math64.h>
   8 #include <linux/vmalloc.h>
   9 #include <net/pkt_cls.h>
  10
  11 #include "cmsg.h"
  12 #include "conntrack.h"
  13 #include "main.h"
  14 #include "../nfp_app.h"
  15
  16 struct nfp_mask_id_table {
  17         struct hlist_node link;
  18         u32 hash_key;
  19         u32 ref_cnt;
  20         u8 mask_id;
  21 };
  22
  23 struct nfp_fl_flow_table_cmp_arg {
  24         struct net_device *netdev;
  25         unsigned long cookie;
  26 };
  27
  28 struct nfp_fl_stats_ctx_to_flow {
  29         struct rhash_head ht_node;
  30         u32 stats_cxt;
  31         struct nfp_fl_payload *flow;
  32 };
  33
  34 static const struct rhashtable_params stats_ctx_table_params = {
  35         .key_offset     = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt),
  36         .head_offset    = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node),
  37         .key_len        = sizeof(u32),
  38 };
  39
  40 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id)
  41 {
  42         struct nfp_flower_priv *priv = app->priv;
  43         struct circ_buf *ring;
  44
  45         ring = &priv->stats_ids.free_list;
  46         /* Check if buffer is full, stats_ring_size must be power of 2 */
  47         if (!CIRC_SPACE(ring->head, ring->tail, priv->stats_ring_size))
  48                 return -ENOBUFS;
  49
  50         /* Each increment of head represents size of NFP_FL_STATS_ELEM_RS */
  51         memcpy(&ring->buf[ring->head * NFP_FL_STATS_ELEM_RS],
  52                &stats_context_id, NFP_FL_STATS_ELEM_RS);
  53         ring->head = (ring->head + 1) & (priv->stats_ring_size - 1);
  54
  55         return 0;
  56 }
  57
  58 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
  59 {
  60         struct nfp_flower_priv *priv = app->priv;
  61         u32 freed_stats_id, temp_stats_id;
  62         struct circ_buf *ring;
  63
  64         ring = &priv->stats_ids.free_list;
  65         freed_stats_id = priv->stats_ring_size;
  66         /* Check for unallocated entries first. */
  67         if (priv->stats_ids.init_unalloc > 0) {
  68                 *stats_context_id =
  69                         FIELD_PREP(NFP_FL_STAT_ID_STAT,
  70                                    priv->stats_ids.init_unalloc - 1) |
  71                         FIELD_PREP(NFP_FL_STAT_ID_MU_NUM,
  72                                    priv->active_mem_unit);
  73
  74                 if (++priv->active_mem_unit == priv->total_mem_units) {
  75                         priv->stats_ids.init_unalloc--;
  76                         priv->active_mem_unit = 0;
  77                 }
  78
  79                 return 0;
  80         }
  81
  82         /* Check if buffer is empty. */
  83         if (ring->head == ring->tail) {
  84                 *stats_context_id = freed_stats_id;
  85                 return -ENOENT;
  86         }
  87
  88         /* Each increment of tail represents size of NFP_FL_STATS_ELEM_RS */
  89         memcpy(&temp_stats_id, &ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS],
  90                NFP_FL_STATS_ELEM_RS);
  91         *stats_context_id = temp_stats_id;
  92         memcpy(&ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], &freed_stats_id,
  93                NFP_FL_STATS_ELEM_RS);
  94         /* stats_ring_size must be power of 2 */
  95         ring->tail = (ring->tail + 1) & (priv->stats_ring_size - 1);
  96
  97         return 0;
  98 }
  99
 100 /* Must be called with either RTNL or rcu_read_lock */
 101 struct nfp_fl_payload *
 102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
 103                            struct net_device *netdev)
 104 {
 105         struct nfp_fl_flow_table_cmp_arg flower_cmp_arg;
 106         struct nfp_flower_priv *priv = app->priv;
 107
 108         flower_cmp_arg.netdev = netdev;
 109         flower_cmp_arg.cookie = tc_flower_cookie;
 110
 111         return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg,
 112                                       nfp_flower_table_params);
 113 }
 114
 115 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb)
 116 {
 117         unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb);
 118         struct nfp_flower_priv *priv = app->priv;
 119         struct nfp_fl_stats_frame *stats;
 120         unsigned char *msg;
 121         u32 ctx_id;
 122         int i;
 123
 124         msg = nfp_flower_cmsg_get_data(skb);
 125
 126         spin_lock(&priv->stats_lock);
 127         for (i = 0; i < msg_len / sizeof(*stats); i++) {
 128                 stats = (struct nfp_fl_stats_frame *)msg + i;
 129                 ctx_id = be32_to_cpu(stats->stats_con_id);
 130                 priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count);
 131                 priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count);
 132                 priv->stats[ctx_id].used = jiffies;
 133         }
 134         spin_unlock(&priv->stats_lock);
 135 }
 136
 137 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id)
 138 {
 139         struct nfp_flower_priv *priv = app->priv;
 140         struct circ_buf *ring;
 141
 142         ring = &priv->mask_ids.mask_id_free_list;
 143         /* Checking if buffer is full,
 144          * NFP_FLOWER_MASK_ENTRY_RS must be power of 2
 145          */
 146         if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0)
 147                 return -ENOBUFS;
 148
 149         /* Each increment of head represents size of
 150          * NFP_FLOWER_MASK_ELEMENT_RS
 151          */
 152         memcpy(&ring->buf[ring->head * NFP_FLOWER_MASK_ELEMENT_RS], &mask_id,
 153                NFP_FLOWER_MASK_ELEMENT_RS);
 154         ring->head = (ring->head + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1);
 155
 156         priv->mask_ids.last_used[mask_id] = ktime_get();
 157
 158         return 0;
 159 }
 160
 161 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id)
 162 {
 163         struct nfp_flower_priv *priv = app->priv;
 164         ktime_t reuse_timeout;
 165         struct circ_buf *ring;
 166         u8 temp_id, freed_id;
 167
 168         ring = &priv->mask_ids.mask_id_free_list;
 169         freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1;
 170         /* Checking for unallocated entries first. */
 171         if (priv->mask_ids.init_unallocated > 0) {
 172                 *mask_id = priv->mask_ids.init_unallocated;
 173                 priv->mask_ids.init_unallocated--;
 174                 return 0;
 175         }
 176
 177         /* Checking if buffer is empty. */
 178         if (ring->head == ring->tail)
 179                 goto err_not_found;
 180
 181         /* Each increment of tail represents size of
 182          * NFP_FLOWER_MASK_ELEMENT_RS
 183          */
 184         memcpy(&temp_id, &ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS],
 185                NFP_FLOWER_MASK_ELEMENT_RS);
 186         *mask_id = temp_id;
 187
 188         reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id],
 189                                      NFP_FL_MASK_REUSE_TIME_NS);
 190
 191         if (ktime_before(ktime_get(), reuse_timeout))
 192                 goto err_not_found;
 193
 194         memcpy(&ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], &freed_id,
 195                NFP_FLOWER_MASK_ELEMENT_RS);
 196         /* NFP_FLOWER_MASK_ENTRY_RS must be power of 2 */
 197         ring->tail = (ring->tail + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1);
 198
 199         return 0;
 200
 201 err_not_found:
 202         *mask_id = freed_id;
 203         return -ENOENT;
 204 }
 205
 206 static int
 207 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
 208 {
 209         struct nfp_flower_priv *priv = app->priv;
 210         struct nfp_mask_id_table *mask_entry;
 211         unsigned long hash_key;
 212         u8 mask_id;
 213
 214         if (nfp_mask_alloc(app, &mask_id))
 215                 return -ENOENT;
 216
 217         mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL);
 218         if (!mask_entry) {
 219                 nfp_release_mask_id(app, mask_id);
 220                 return -ENOMEM;
 221         }
 222
 223         INIT_HLIST_NODE(&mask_entry->link);
 224         mask_entry->mask_id = mask_id;
 225         hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
 226         mask_entry->hash_key = hash_key;
 227         mask_entry->ref_cnt = 1;
 228         hash_add(priv->mask_table, &mask_entry->link, hash_key);
 229
 230         return mask_id;
 231 }
 232
 233 static struct nfp_mask_id_table *
 234 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
 235 {
 236         struct nfp_flower_priv *priv = app->priv;
 237         struct nfp_mask_id_table *mask_entry;
 238         unsigned long hash_key;
 239
 240         hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
 241
 242         hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key)
 243                 if (mask_entry->hash_key == hash_key)
 244                         return mask_entry;
 245
 246         return NULL;
 247 }
 248
 249 static int
 250 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
 251 {
 252         struct nfp_mask_id_table *mask_entry;
 253
 254         mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
 255         if (!mask_entry)
 256                 return -ENOENT;
 257
 258         mask_entry->ref_cnt++;
 259
 260         /* Casting u8 to int for later use. */
 261         return mask_entry->mask_id;
 262 }
 263
 264 static bool
 265 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len,
 266                    u8 *meta_flags, u8 *mask_id)
 267 {
 268         int id;
 269
 270         id = nfp_find_in_mask_table(app, mask_data, mask_len);
 271         if (id < 0) {
 272                 id = nfp_add_mask_table(app, mask_data, mask_len);
 273                 if (id < 0)
 274                         return false;
 275                 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
 276         }
 277         *mask_id = id;
 278
 279         return true;
 280 }
 281
 282 static bool
 283 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
 284                       u8 *meta_flags, u8 *mask_id)
 285 {
 286         struct nfp_mask_id_table *mask_entry;
 287
 288         mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
 289         if (!mask_entry)
 290                 return false;
 291
 292         *mask_id = mask_entry->mask_id;
 293         mask_entry->ref_cnt--;
 294         if (!mask_entry->ref_cnt) {
 295                 hash_del(&mask_entry->link);
 296                 nfp_release_mask_id(app, *mask_id);
 297                 kfree(mask_entry);
 298                 if (meta_flags)
 299                         *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
 300         }
 301
 302         return true;
 303 }
 304
 305 int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie,
 306                               struct nfp_fl_payload *nfp_flow,
 307                               struct net_device *netdev,
 308                               struct netlink_ext_ack *extack)
 309 {
 310         struct nfp_fl_stats_ctx_to_flow *ctx_entry;
 311         struct nfp_flower_priv *priv = app->priv;
 312         struct nfp_fl_payload *check_entry;
 313         u8 new_mask_id;
 314         u32 stats_cxt;
 315         int err;
 316
 317         err = nfp_get_stats_entry(app, &stats_cxt);
 318         if (err) {
 319                 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context");
 320                 return err;
 321         }
 322
 323         nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt);
 324         nfp_flow->meta.host_cookie = cpu_to_be64(cookie);
 325         nfp_flow->ingress_dev = netdev;
 326
 327         ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL);
 328         if (!ctx_entry) {
 329                 err = -ENOMEM;
 330                 goto err_release_stats;
 331         }
 332
 333         ctx_entry->stats_cxt = stats_cxt;
 334         ctx_entry->flow = nfp_flow;
 335
 336         if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node,
 337                                    stats_ctx_table_params)) {
 338                 err = -ENOMEM;
 339                 goto err_free_ctx_entry;
 340         }
 341
 342         /* Do not allocate a mask-id for pre_tun_rules. These flows are used to
 343          * configure the pre_tun table and are never actually send to the
 344          * firmware as an add-flow message. This causes the mask-id allocation
 345          * on the firmware to get out of sync if allocated here.
 346          */
 347         new_mask_id = 0;
 348         if (!nfp_flow->pre_tun_rule.dev &&
 349             !nfp_check_mask_add(app, nfp_flow->mask_data,
 350                                 nfp_flow->meta.mask_len,
 351                                 &nfp_flow->meta.flags, &new_mask_id)) {
 352                 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id");
 353                 err = -ENOENT;
 354                 goto err_remove_rhash;
 355         }
 356
 357         nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
 358         priv->flower_version++;
 359
 360         /* Update flow payload with mask ids. */
 361         nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
 362         priv->stats[stats_cxt].pkts = 0;
 363         priv->stats[stats_cxt].bytes = 0;
 364         priv->stats[stats_cxt].used = jiffies;
 365
 366         check_entry = nfp_flower_search_fl_table(app, cookie, netdev);
 367         if (check_entry) {
 368                 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry");
 369                 err = -EEXIST;
 370                 goto err_remove_mask;
 371         }
 372
 373         return 0;
 374
 375 err_remove_mask:
 376         if (!nfp_flow->pre_tun_rule.dev)
 377                 nfp_check_mask_remove(app, nfp_flow->mask_data,
 378                                       nfp_flow->meta.mask_len,
 379                                       NULL, &new_mask_id);
 380 err_remove_rhash:
 381         WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table,
 382                                             &ctx_entry->ht_node,
 383                                             stats_ctx_table_params));
 384 err_free_ctx_entry:
 385         kfree(ctx_entry);
 386 err_release_stats:
 387         nfp_release_stats_entry(app, stats_cxt);
 388
 389         return err;
 390 }
 391
 392 void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv,
 393                                 struct nfp_fl_payload *nfp_flow)
 394 {
 395         nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK;
 396         nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
 397         priv->flower_version++;
 398 }
 399
 400 int nfp_modify_flow_metadata(struct nfp_app *app,
 401                              struct nfp_fl_payload *nfp_flow)
 402 {
 403         struct nfp_fl_stats_ctx_to_flow *ctx_entry;
 404         struct nfp_flower_priv *priv = app->priv;
 405         u8 new_mask_id = 0;
 406         u32 temp_ctx_id;
 407
 408         __nfp_modify_flow_metadata(priv, nfp_flow);
 409
 410         if (!nfp_flow->pre_tun_rule.dev)
 411                 nfp_check_mask_remove(app, nfp_flow->mask_data,
 412                                       nfp_flow->meta.mask_len, &nfp_flow->meta.flags,
 413                                       &new_mask_id);
 414
 415         /* Update flow payload with mask ids. */
 416         nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
 417
 418         /* Release the stats ctx id and ctx to flow table entry. */
 419         temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
 420
 421         ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id,
 422                                            stats_ctx_table_params);
 423         if (!ctx_entry)
 424                 return -ENOENT;
 425
 426         WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table,
 427                                             &ctx_entry->ht_node,
 428                                             stats_ctx_table_params));
 429         kfree(ctx_entry);
 430
 431         return nfp_release_stats_entry(app, temp_ctx_id);
 432 }
 433
 434 struct nfp_fl_payload *
 435 nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id)
 436 {
 437         struct nfp_fl_stats_ctx_to_flow *ctx_entry;
 438         struct nfp_flower_priv *priv = app->priv;
 439
 440         ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id,
 441                                            stats_ctx_table_params);
 442         if (!ctx_entry)
 443                 return NULL;
 444
 445         return ctx_entry->flow;
 446 }
 447
 448 static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg,
 449                             const void *obj)
 450 {
 451         const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key;
 452         const struct nfp_fl_payload *flow_entry = obj;
 453
 454         if (flow_entry->ingress_dev == cmp_arg->netdev)
 455                 return flow_entry->tc_flower_cookie != cmp_arg->cookie;
 456
 457         return 1;
 458 }
 459
 460 static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed)
 461 {
 462         const struct nfp_fl_payload *flower_entry = data;
 463
 464         return jhash2((u32 *)&flower_entry->tc_flower_cookie,
 465                       sizeof(flower_entry->tc_flower_cookie) / sizeof(u32),
 466                       seed);
 467 }
 468
 469 static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed)
 470 {
 471         const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data;
 472
 473         return jhash2((u32 *)&cmp_arg->cookie,
 474                       sizeof(cmp_arg->cookie) / sizeof(u32), seed);
 475 }
 476
 477 const struct rhashtable_params nfp_flower_table_params = {
 478         .head_offset            = offsetof(struct nfp_fl_payload, fl_node),
 479         .hashfn                 = nfp_fl_key_hashfn,
 480         .obj_cmpfn              = nfp_fl_obj_cmpfn,
 481         .obj_hashfn             = nfp_fl_obj_hashfn,
 482         .automatic_shrinking    = true,
 483 };
 484
 485 const struct rhashtable_params merge_table_params = {
 486         .key_offset     = offsetof(struct nfp_merge_info, parent_ctx),
 487         .head_offset    = offsetof(struct nfp_merge_info, ht_node),
 488         .key_len        = sizeof(u64),
 489 };
 490
 491 const struct rhashtable_params nfp_zone_table_params = {
 492         .head_offset            = offsetof(struct nfp_fl_ct_zone_entry, hash_node),
 493         .key_len                = sizeof(u16),
 494         .key_offset             = offsetof(struct nfp_fl_ct_zone_entry, zone),
 495         .automatic_shrinking    = false,
 496 };
 497
 498 const struct rhashtable_params nfp_ct_map_params = {
 499         .head_offset            = offsetof(struct nfp_fl_ct_map_entry, hash_node),
 500         .key_len                = sizeof(unsigned long),
 501         .key_offset             = offsetof(struct nfp_fl_ct_map_entry, cookie),
 502         .automatic_shrinking    = true,
 503 };
 504
 505 const struct rhashtable_params neigh_table_params = {
 506         .key_offset     = offsetof(struct nfp_neigh_entry, neigh_cookie),
 507         .head_offset    = offsetof(struct nfp_neigh_entry, ht_node),
 508         .key_len        = sizeof(unsigned long),
 509 };
 510
 511 int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count,
 512                              unsigned int host_num_mems)
 513 {
 514         struct nfp_flower_priv *priv = app->priv;
 515         int err, stats_size;
 516
 517         hash_init(priv->mask_table);
 518
 519         err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params);
 520         if (err)
 521                 return err;
 522
 523         err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params);
 524         if (err)
 525                 goto err_free_flow_table;
 526
 527         err = rhashtable_init(&priv->merge_table, &merge_table_params);
 528         if (err)
 529                 goto err_free_stats_ctx_table;
 530
 531         mutex_init(&priv->nfp_fl_lock);
 532
 533         err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params);
 534         if (err)
 535                 goto err_free_merge_table;
 536
 537         err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params);
 538         if (err)
 539                 goto err_free_ct_zone_table;
 540
 541         err = rhashtable_init(&priv->neigh_table, &neigh_table_params);
 542         if (err)
 543                 goto err_free_ct_map_table;
 544
 545         INIT_LIST_HEAD(&priv->predt_list);
 546
 547         get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
 548
 549         /* Init ring buffer and unallocated mask_ids. */
 550         priv->mask_ids.mask_id_free_list.buf =
 551                 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
 552                               NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
 553         if (!priv->mask_ids.mask_id_free_list.buf)
 554                 goto err_free_neigh_table;
 555
 556         priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
 557
 558         /* Init timestamps for mask id*/
 559         priv->mask_ids.last_used =
 560                 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
 561                               sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
 562         if (!priv->mask_ids.last_used)
 563                 goto err_free_mask_id;
 564
 565         /* Init ring buffer and unallocated stats_ids. */
 566         priv->stats_ids.free_list.buf =
 567                 vmalloc(array_size(NFP_FL_STATS_ELEM_RS,
 568                                    priv->stats_ring_size));
 569         if (!priv->stats_ids.free_list.buf)
 570                 goto err_free_last_used;
 571
 572         priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems);
 573
 574         stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) |
 575                      FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1);
 576         priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats),
 577                                      GFP_KERNEL);
 578         if (!priv->stats)
 579                 goto err_free_ring_buf;
 580
 581         spin_lock_init(&priv->stats_lock);
 582         spin_lock_init(&priv->predt_lock);
 583
 584         return 0;
 585
 586 err_free_ring_buf:
 587         vfree(priv->stats_ids.free_list.buf);
 588 err_free_last_used:
 589         kfree(priv->mask_ids.last_used);
 590 err_free_mask_id:
 591         kfree(priv->mask_ids.mask_id_free_list.buf);
 592 err_free_neigh_table:
 593         rhashtable_destroy(&priv->neigh_table);
 594 err_free_ct_map_table:
 595         rhashtable_destroy(&priv->ct_map_table);
 596 err_free_ct_zone_table:
 597         rhashtable_destroy(&priv->ct_zone_table);
 598 err_free_merge_table:
 599         rhashtable_destroy(&priv->merge_table);
 600 err_free_stats_ctx_table:
 601         rhashtable_destroy(&priv->stats_ctx_table);
 602 err_free_flow_table:
 603         rhashtable_destroy(&priv->flow_table);
 604         return -ENOMEM;
 605 }
 606
 607 static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt)
 608 {
 609         if (!zt)
 610                 return;
 611
 612         if (!list_empty(&zt->pre_ct_list)) {
 613                 struct rhashtable *m_table = &zt->priv->ct_map_table;
 614                 struct nfp_fl_ct_flow_entry *entry, *tmp;
 615                 struct nfp_fl_ct_map_entry *map;
 616
 617                 WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n");
 618                 list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list,
 619                                          list_node) {
 620                         map = rhashtable_lookup_fast(m_table,
 621                                                      &entry->cookie,
 622                                                      nfp_ct_map_params);
 623                         WARN_ON_ONCE(rhashtable_remove_fast(m_table,
 624                                                             &map->hash_node,
 625                                                             nfp_ct_map_params));
 626                         nfp_fl_ct_clean_flow_entry(entry);
 627                         kfree(map);
 628                 }
 629         }
 630
 631         if (!list_empty(&zt->post_ct_list)) {
 632                 struct rhashtable *m_table = &zt->priv->ct_map_table;
 633                 struct nfp_fl_ct_flow_entry *entry, *tmp;
 634                 struct nfp_fl_ct_map_entry *map;
 635
 636                 WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n");
 637                 list_for_each_entry_safe(entry, tmp, &zt->post_ct_list,
 638                                          list_node) {
 639                         map = rhashtable_lookup_fast(m_table,
 640                                                      &entry->cookie,
 641                                                      nfp_ct_map_params);
 642                         WARN_ON_ONCE(rhashtable_remove_fast(m_table,
 643                                                             &map->hash_node,
 644                                                             nfp_ct_map_params));
 645                         nfp_fl_ct_clean_flow_entry(entry);
 646                         kfree(map);
 647                 }
 648         }
 649
 650         if (zt->nft) {
 651                 nf_flow_table_offload_del_cb(zt->nft,
 652                                              nfp_fl_ct_handle_nft_flow,
 653                                              zt);
 654                 zt->nft = NULL;
 655         }
 656
 657         if (!list_empty(&zt->nft_flows_list)) {
 658                 struct rhashtable *m_table = &zt->priv->ct_map_table;
 659                 struct nfp_fl_ct_flow_entry *entry, *tmp;
 660                 struct nfp_fl_ct_map_entry *map;
 661
 662                 WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n");
 663                 list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list,
 664                                          list_node) {
 665                         map = rhashtable_lookup_fast(m_table,
 666                                                      &entry->cookie,
 667                                                      nfp_ct_map_params);
 668                         WARN_ON_ONCE(rhashtable_remove_fast(m_table,
 669                                                             &map->hash_node,
 670                                                             nfp_ct_map_params));
 671                         nfp_fl_ct_clean_flow_entry(entry);
 672                         kfree(map);
 673                 }
 674         }
 675
 676         rhashtable_free_and_destroy(&zt->tc_merge_tb,
 677                                     nfp_check_rhashtable_empty, NULL);
 678         rhashtable_free_and_destroy(&zt->nft_merge_tb,
 679                                     nfp_check_rhashtable_empty, NULL);
 680
 681         kfree(zt);
 682 }
 683
 684 static void nfp_free_zone_table_entry(void *ptr, void *arg)
 685 {
 686         struct nfp_fl_ct_zone_entry *zt = ptr;
 687
 688         nfp_zone_table_entry_destroy(zt);
 689 }
 690
 691 static void nfp_free_map_table_entry(void *ptr, void *arg)
 692 {
 693         struct nfp_fl_ct_map_entry *map = ptr;
 694
 695         if (!map)
 696                 return;
 697
 698         kfree(map);
 699 }
 700
 701 void nfp_flower_metadata_cleanup(struct nfp_app *app)
 702 {
 703         struct nfp_flower_priv *priv = app->priv;
 704
 705         if (!priv)
 706                 return;
 707
 708         rhashtable_free_and_destroy(&priv->flow_table,
 709                                     nfp_check_rhashtable_empty, NULL);
 710         rhashtable_free_and_destroy(&priv->stats_ctx_table,
 711                                     nfp_check_rhashtable_empty, NULL);
 712         rhashtable_free_and_destroy(&priv->merge_table,
 713                                     nfp_check_rhashtable_empty, NULL);
 714         rhashtable_free_and_destroy(&priv->ct_zone_table,
 715                                     nfp_free_zone_table_entry, NULL);
 716         nfp_zone_table_entry_destroy(priv->ct_zone_wc);
 717
 718         rhashtable_free_and_destroy(&priv->ct_map_table,
 719                                     nfp_free_map_table_entry, NULL);
 720         rhashtable_free_and_destroy(&priv->neigh_table,
 721                                     nfp_check_rhashtable_empty, NULL);
 722         kvfree(priv->stats);
 723         kfree(priv->mask_ids.mask_id_free_list.buf);
 724         kfree(priv->mask_ids.last_used);
 725         vfree(priv->stats_ids.free_list.buf);
 726 }