1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
4 #include <linux/kernel.h>
5 #include <linux/slab.h>
6 #include <linux/errno.h>
7 #include <linux/bitops.h>
8 #include <linux/list.h>
9 #include <linux/rhashtable.h>
10 #include <linux/netdevice.h>
11 #include <linux/mutex.h>
12 #include <net/devlink.h>
13 #include <trace/events/mlxsw.h>
17 #include "resources.h"
19 #include "spectrum_acl_tcam.h"
20 #include "core_acl_flex_keys.h"
22 size_t mlxsw_sp_acl_tcam_priv_size(struct mlxsw_sp *mlxsw_sp)
24 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
26 return ops->priv_size;
29 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT 5000 /* ms */
30 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN 3000 /* ms */
31 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS 100 /* number of entries */
33 int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp,
34 struct mlxsw_sp_acl_rule_info *rulei,
35 u32 *priority, bool fillup_priority)
39 if (!fillup_priority) {
44 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, KVD_SIZE))
47 /* Priority range is 1..cap_kvd_size-1. */
48 max_priority = MLXSW_CORE_RES_GET(mlxsw_sp->core, KVD_SIZE) - 1;
49 if (rulei->priority >= max_priority)
52 /* Unlike in TC, in HW, higher number means higher priority. */
53 *priority = max_priority - rulei->priority;
57 static int mlxsw_sp_acl_tcam_region_id_get(struct mlxsw_sp_acl_tcam *tcam,
62 id = find_first_zero_bit(tcam->used_regions, tcam->max_regions);
63 if (id < tcam->max_regions) {
64 __set_bit(id, tcam->used_regions);
71 static void mlxsw_sp_acl_tcam_region_id_put(struct mlxsw_sp_acl_tcam *tcam,
74 __clear_bit(id, tcam->used_regions);
77 static int mlxsw_sp_acl_tcam_group_id_get(struct mlxsw_sp_acl_tcam *tcam,
82 id = find_first_zero_bit(tcam->used_groups, tcam->max_groups);
83 if (id < tcam->max_groups) {
84 __set_bit(id, tcam->used_groups);
91 static void mlxsw_sp_acl_tcam_group_id_put(struct mlxsw_sp_acl_tcam *tcam,
94 __clear_bit(id, tcam->used_groups);
97 struct mlxsw_sp_acl_tcam_pattern {
98 const enum mlxsw_afk_element *elements;
99 unsigned int elements_count;
102 struct mlxsw_sp_acl_tcam_group {
103 struct mlxsw_sp_acl_tcam *tcam;
105 struct mutex lock; /* guards region list updates */
106 struct list_head region_list;
107 unsigned int region_count;
110 struct mlxsw_sp_acl_tcam_vgroup {
111 struct mlxsw_sp_acl_tcam_group group;
112 struct list_head vregion_list;
113 struct rhashtable vchunk_ht;
114 const struct mlxsw_sp_acl_tcam_pattern *patterns;
115 unsigned int patterns_count;
116 bool tmplt_elusage_set;
117 struct mlxsw_afk_element_usage tmplt_elusage;
118 bool vregion_rehash_enabled;
119 unsigned int *p_min_prio;
120 unsigned int *p_max_prio;
123 struct mlxsw_sp_acl_tcam_rehash_ctx {
125 bool this_is_rollback;
126 struct mlxsw_sp_acl_tcam_vchunk *current_vchunk; /* vchunk being
127 * currently migrated.
129 struct mlxsw_sp_acl_tcam_ventry *start_ventry; /* ventry to start
132 * currently migrated.
134 struct mlxsw_sp_acl_tcam_ventry *stop_ventry; /* ventry to stop
137 * currently migrated.
141 struct mlxsw_sp_acl_tcam_vregion {
142 struct mutex lock; /* Protects consistency of region, region2 pointers
145 struct mlxsw_sp_acl_tcam_region *region;
146 struct mlxsw_sp_acl_tcam_region *region2; /* Used during migration */
147 struct list_head list; /* Member of a TCAM group */
148 struct list_head tlist; /* Member of a TCAM */
149 struct list_head vchunk_list; /* List of vchunks under this vregion */
150 struct mlxsw_afk_key_info *key_info;
151 struct mlxsw_sp_acl_tcam *tcam;
152 struct mlxsw_sp_acl_tcam_vgroup *vgroup;
154 struct delayed_work dw;
155 struct mlxsw_sp_acl_tcam_rehash_ctx ctx;
157 struct mlxsw_sp *mlxsw_sp;
158 unsigned int ref_count;
161 struct mlxsw_sp_acl_tcam_vchunk;
163 struct mlxsw_sp_acl_tcam_chunk {
164 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
165 struct mlxsw_sp_acl_tcam_region *region;
166 unsigned long priv[];
167 /* priv has to be always the last item */
170 struct mlxsw_sp_acl_tcam_vchunk {
171 struct mlxsw_sp_acl_tcam_chunk *chunk;
172 struct mlxsw_sp_acl_tcam_chunk *chunk2; /* Used during migration */
173 struct list_head list; /* Member of a TCAM vregion */
174 struct rhash_head ht_node; /* Member of a chunk HT */
175 struct list_head ventry_list;
176 unsigned int priority; /* Priority within the vregion and group */
177 struct mlxsw_sp_acl_tcam_vgroup *vgroup;
178 struct mlxsw_sp_acl_tcam_vregion *vregion;
179 unsigned int ref_count;
182 struct mlxsw_sp_acl_tcam_entry {
183 struct mlxsw_sp_acl_tcam_ventry *ventry;
184 struct mlxsw_sp_acl_tcam_chunk *chunk;
185 unsigned long priv[];
186 /* priv has to be always the last item */
189 struct mlxsw_sp_acl_tcam_ventry {
190 struct mlxsw_sp_acl_tcam_entry *entry;
191 struct list_head list; /* Member of a TCAM vchunk */
192 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
193 struct mlxsw_sp_acl_rule_info *rulei;
196 static const struct rhashtable_params mlxsw_sp_acl_tcam_vchunk_ht_params = {
197 .key_len = sizeof(unsigned int),
198 .key_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, priority),
199 .head_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, ht_node),
200 .automatic_shrinking = true,
203 static int mlxsw_sp_acl_tcam_group_update(struct mlxsw_sp *mlxsw_sp,
204 struct mlxsw_sp_acl_tcam_group *group)
206 struct mlxsw_sp_acl_tcam_region *region;
207 char pagt_pl[MLXSW_REG_PAGT_LEN];
210 mlxsw_reg_pagt_pack(pagt_pl, group->id);
211 list_for_each_entry(region, &group->region_list, list) {
214 /* Check if the next entry in the list has the same vregion. */
215 if (region->list.next != &group->region_list &&
216 list_next_entry(region, list)->vregion == region->vregion)
218 mlxsw_reg_pagt_acl_id_pack(pagt_pl, acl_index++,
221 mlxsw_reg_pagt_size_set(pagt_pl, acl_index);
222 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pagt), pagt_pl);
226 mlxsw_sp_acl_tcam_group_add(struct mlxsw_sp_acl_tcam *tcam,
227 struct mlxsw_sp_acl_tcam_group *group)
232 INIT_LIST_HEAD(&group->region_list);
234 err = mlxsw_sp_acl_tcam_group_id_get(tcam, &group->id);
238 mutex_init(&group->lock);
243 static void mlxsw_sp_acl_tcam_group_del(struct mlxsw_sp_acl_tcam_group *group)
245 struct mlxsw_sp_acl_tcam *tcam = group->tcam;
247 mutex_destroy(&group->lock);
248 mlxsw_sp_acl_tcam_group_id_put(tcam, group->id);
249 WARN_ON(!list_empty(&group->region_list));
253 mlxsw_sp_acl_tcam_vgroup_add(struct mlxsw_sp *mlxsw_sp,
254 struct mlxsw_sp_acl_tcam *tcam,
255 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
256 const struct mlxsw_sp_acl_tcam_pattern *patterns,
257 unsigned int patterns_count,
258 struct mlxsw_afk_element_usage *tmplt_elusage,
259 bool vregion_rehash_enabled,
260 unsigned int *p_min_prio,
261 unsigned int *p_max_prio)
265 vgroup->patterns = patterns;
266 vgroup->patterns_count = patterns_count;
267 vgroup->vregion_rehash_enabled = vregion_rehash_enabled;
268 vgroup->p_min_prio = p_min_prio;
269 vgroup->p_max_prio = p_max_prio;
272 vgroup->tmplt_elusage_set = true;
273 memcpy(&vgroup->tmplt_elusage, tmplt_elusage,
274 sizeof(vgroup->tmplt_elusage));
276 INIT_LIST_HEAD(&vgroup->vregion_list);
278 err = mlxsw_sp_acl_tcam_group_add(tcam, &vgroup->group);
282 err = rhashtable_init(&vgroup->vchunk_ht,
283 &mlxsw_sp_acl_tcam_vchunk_ht_params);
285 goto err_rhashtable_init;
290 mlxsw_sp_acl_tcam_group_del(&vgroup->group);
295 mlxsw_sp_acl_tcam_vgroup_del(struct mlxsw_sp_acl_tcam_vgroup *vgroup)
297 rhashtable_destroy(&vgroup->vchunk_ht);
298 mlxsw_sp_acl_tcam_group_del(&vgroup->group);
299 WARN_ON(!list_empty(&vgroup->vregion_list));
303 mlxsw_sp_acl_tcam_group_bind(struct mlxsw_sp *mlxsw_sp,
304 struct mlxsw_sp_acl_tcam_group *group,
305 struct mlxsw_sp_port *mlxsw_sp_port,
308 char ppbt_pl[MLXSW_REG_PPBT_LEN];
310 mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
311 MLXSW_REG_PXBT_E_EACL,
312 MLXSW_REG_PXBT_OP_BIND, mlxsw_sp_port->local_port,
314 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
318 mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
319 struct mlxsw_sp_acl_tcam_group *group,
320 struct mlxsw_sp_port *mlxsw_sp_port,
323 char ppbt_pl[MLXSW_REG_PPBT_LEN];
325 mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
326 MLXSW_REG_PXBT_E_EACL,
327 MLXSW_REG_PXBT_OP_UNBIND, mlxsw_sp_port->local_port,
329 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
333 mlxsw_sp_acl_tcam_group_id(struct mlxsw_sp_acl_tcam_group *group)
339 mlxsw_sp_acl_tcam_vregion_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
341 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
343 if (list_empty(&vregion->vchunk_list))
345 /* As a priority of a vregion, return priority of the first vchunk */
346 vchunk = list_first_entry(&vregion->vchunk_list,
347 typeof(*vchunk), list);
348 return vchunk->priority;
352 mlxsw_sp_acl_tcam_vregion_max_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
354 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
356 if (list_empty(&vregion->vchunk_list))
358 vchunk = list_last_entry(&vregion->vchunk_list,
359 typeof(*vchunk), list);
360 return vchunk->priority;
364 mlxsw_sp_acl_tcam_vgroup_prio_update(struct mlxsw_sp_acl_tcam_vgroup *vgroup)
366 struct mlxsw_sp_acl_tcam_vregion *vregion;
368 if (list_empty(&vgroup->vregion_list))
370 vregion = list_first_entry(&vgroup->vregion_list,
371 typeof(*vregion), list);
372 *vgroup->p_min_prio = mlxsw_sp_acl_tcam_vregion_prio(vregion);
373 vregion = list_last_entry(&vgroup->vregion_list,
374 typeof(*vregion), list);
375 *vgroup->p_max_prio = mlxsw_sp_acl_tcam_vregion_max_prio(vregion);
379 mlxsw_sp_acl_tcam_group_region_attach(struct mlxsw_sp *mlxsw_sp,
380 struct mlxsw_sp_acl_tcam_group *group,
381 struct mlxsw_sp_acl_tcam_region *region,
382 unsigned int priority,
383 struct mlxsw_sp_acl_tcam_region *next_region)
385 struct mlxsw_sp_acl_tcam_region *region2;
386 struct list_head *pos;
389 mutex_lock(&group->lock);
390 if (group->region_count == group->tcam->max_group_size) {
392 goto err_region_count_check;
396 /* If the next region is defined, place the new one
397 * before it. The next one is a sibling.
399 pos = &next_region->list;
401 /* Position the region inside the list according to priority */
402 list_for_each(pos, &group->region_list) {
403 region2 = list_entry(pos, typeof(*region2), list);
404 if (mlxsw_sp_acl_tcam_vregion_prio(region2->vregion) >
409 list_add_tail(®ion->list, pos);
410 region->group = group;
412 err = mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
414 goto err_group_update;
416 group->region_count++;
417 mutex_unlock(&group->lock);
421 list_del(®ion->list);
422 err_region_count_check:
423 mutex_unlock(&group->lock);
428 mlxsw_sp_acl_tcam_group_region_detach(struct mlxsw_sp *mlxsw_sp,
429 struct mlxsw_sp_acl_tcam_region *region)
431 struct mlxsw_sp_acl_tcam_group *group = region->group;
433 mutex_lock(&group->lock);
434 list_del(®ion->list);
435 group->region_count--;
436 mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
437 mutex_unlock(&group->lock);
441 mlxsw_sp_acl_tcam_vgroup_vregion_attach(struct mlxsw_sp *mlxsw_sp,
442 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
443 struct mlxsw_sp_acl_tcam_vregion *vregion,
444 unsigned int priority)
446 struct mlxsw_sp_acl_tcam_vregion *vregion2;
447 struct list_head *pos;
450 /* Position the vregion inside the list according to priority */
451 list_for_each(pos, &vgroup->vregion_list) {
452 vregion2 = list_entry(pos, typeof(*vregion2), list);
453 if (mlxsw_sp_acl_tcam_vregion_prio(vregion2) > priority)
456 list_add_tail(&vregion->list, pos);
458 err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp, &vgroup->group,
462 goto err_region_attach;
467 list_del(&vregion->list);
472 mlxsw_sp_acl_tcam_vgroup_vregion_detach(struct mlxsw_sp *mlxsw_sp,
473 struct mlxsw_sp_acl_tcam_vregion *vregion)
475 list_del(&vregion->list);
476 if (vregion->region2)
477 mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp,
479 mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, vregion->region);
482 static struct mlxsw_sp_acl_tcam_vregion *
483 mlxsw_sp_acl_tcam_vgroup_vregion_find(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
484 unsigned int priority,
485 struct mlxsw_afk_element_usage *elusage,
488 struct mlxsw_sp_acl_tcam_vregion *vregion, *vregion2;
489 struct list_head *pos;
492 list_for_each(pos, &vgroup->vregion_list) {
493 vregion = list_entry(pos, typeof(*vregion), list);
495 /* First, check if the requested priority does not rather belong
496 * under some of the next vregions.
498 if (pos->next != &vgroup->vregion_list) { /* not last */
499 vregion2 = list_entry(pos->next, typeof(*vregion2),
502 mlxsw_sp_acl_tcam_vregion_prio(vregion2))
506 issubset = mlxsw_afk_key_info_subset(vregion->key_info,
509 /* If requested element usage would not fit and the priority
510 * is lower than the currently inspected vregion we cannot
511 * use this region, so return NULL to indicate new vregion has
515 priority < mlxsw_sp_acl_tcam_vregion_prio(vregion))
518 /* If requested element usage would not fit and the priority
519 * is higher than the currently inspected vregion we cannot
520 * use this vregion. There is still some hope that the next
521 * vregion would be the fit. So let it be processed and
522 * eventually break at the check right above this.
525 priority > mlxsw_sp_acl_tcam_vregion_max_prio(vregion))
528 /* Indicate if the vregion needs to be split in order to add
529 * the requested priority. Split is needed when requested
530 * element usage won't fit into the found vregion.
532 *p_need_split = !issubset;
535 return NULL; /* New vregion has to be created. */
539 mlxsw_sp_acl_tcam_vgroup_use_patterns(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
540 struct mlxsw_afk_element_usage *elusage,
541 struct mlxsw_afk_element_usage *out)
543 const struct mlxsw_sp_acl_tcam_pattern *pattern;
546 /* In case the template is set, we don't have to look up the pattern
547 * and just use the template.
549 if (vgroup->tmplt_elusage_set) {
550 memcpy(out, &vgroup->tmplt_elusage, sizeof(*out));
551 WARN_ON(!mlxsw_afk_element_usage_subset(elusage, out));
555 for (i = 0; i < vgroup->patterns_count; i++) {
556 pattern = &vgroup->patterns[i];
557 mlxsw_afk_element_usage_fill(out, pattern->elements,
558 pattern->elements_count);
559 if (mlxsw_afk_element_usage_subset(elusage, out))
562 memcpy(out, elusage, sizeof(*out));
566 mlxsw_sp_acl_tcam_region_alloc(struct mlxsw_sp *mlxsw_sp,
567 struct mlxsw_sp_acl_tcam_region *region)
569 struct mlxsw_afk_key_info *key_info = region->key_info;
570 char ptar_pl[MLXSW_REG_PTAR_LEN];
571 unsigned int encodings_count;
575 mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_ALLOC,
577 MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT,
578 region->id, region->tcam_region_info);
579 encodings_count = mlxsw_afk_key_info_blocks_count_get(key_info);
580 for (i = 0; i < encodings_count; i++) {
583 encoding = mlxsw_afk_key_info_block_encoding_get(key_info, i);
584 mlxsw_reg_ptar_key_id_pack(ptar_pl, i, encoding);
586 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
589 mlxsw_reg_ptar_unpack(ptar_pl, region->tcam_region_info);
594 mlxsw_sp_acl_tcam_region_free(struct mlxsw_sp *mlxsw_sp,
595 struct mlxsw_sp_acl_tcam_region *region)
597 char ptar_pl[MLXSW_REG_PTAR_LEN];
599 mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_FREE,
600 region->key_type, 0, region->id,
601 region->tcam_region_info);
602 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
606 mlxsw_sp_acl_tcam_region_enable(struct mlxsw_sp *mlxsw_sp,
607 struct mlxsw_sp_acl_tcam_region *region)
609 char pacl_pl[MLXSW_REG_PACL_LEN];
611 mlxsw_reg_pacl_pack(pacl_pl, region->id, true,
612 region->tcam_region_info);
613 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
617 mlxsw_sp_acl_tcam_region_disable(struct mlxsw_sp *mlxsw_sp,
618 struct mlxsw_sp_acl_tcam_region *region)
620 char pacl_pl[MLXSW_REG_PACL_LEN];
622 mlxsw_reg_pacl_pack(pacl_pl, region->id, false,
623 region->tcam_region_info);
624 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
627 static struct mlxsw_sp_acl_tcam_region *
628 mlxsw_sp_acl_tcam_region_create(struct mlxsw_sp *mlxsw_sp,
629 struct mlxsw_sp_acl_tcam *tcam,
630 struct mlxsw_sp_acl_tcam_vregion *vregion,
633 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
634 struct mlxsw_sp_acl_tcam_region *region;
637 region = kzalloc(sizeof(*region) + ops->region_priv_size, GFP_KERNEL);
639 return ERR_PTR(-ENOMEM);
640 region->mlxsw_sp = mlxsw_sp;
641 region->vregion = vregion;
642 region->key_info = vregion->key_info;
644 err = mlxsw_sp_acl_tcam_region_id_get(tcam, ®ion->id);
646 goto err_region_id_get;
648 err = ops->region_associate(mlxsw_sp, region);
650 goto err_tcam_region_associate;
652 region->key_type = ops->key_type;
653 err = mlxsw_sp_acl_tcam_region_alloc(mlxsw_sp, region);
655 goto err_tcam_region_alloc;
657 err = mlxsw_sp_acl_tcam_region_enable(mlxsw_sp, region);
659 goto err_tcam_region_enable;
661 err = ops->region_init(mlxsw_sp, region->priv, tcam->priv,
664 goto err_tcam_region_init;
668 err_tcam_region_init:
669 mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
670 err_tcam_region_enable:
671 mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
672 err_tcam_region_alloc:
673 err_tcam_region_associate:
674 mlxsw_sp_acl_tcam_region_id_put(tcam, region->id);
681 mlxsw_sp_acl_tcam_region_destroy(struct mlxsw_sp *mlxsw_sp,
682 struct mlxsw_sp_acl_tcam_region *region)
684 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
686 ops->region_fini(mlxsw_sp, region->priv);
687 mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
688 mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
689 mlxsw_sp_acl_tcam_region_id_put(region->group->tcam,
695 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(struct mlxsw_sp_acl_tcam_vregion *vregion)
697 unsigned long interval = vregion->tcam->vregion_rehash_intrvl;
701 mlxsw_core_schedule_dw(&vregion->rehash.dw,
702 msecs_to_jiffies(interval));
706 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
707 struct mlxsw_sp_acl_tcam_vregion *vregion,
710 static void mlxsw_sp_acl_tcam_vregion_rehash_work(struct work_struct *work)
712 struct mlxsw_sp_acl_tcam_vregion *vregion =
713 container_of(work, struct mlxsw_sp_acl_tcam_vregion,
715 int credits = MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS;
717 mlxsw_sp_acl_tcam_vregion_rehash(vregion->mlxsw_sp, vregion, &credits);
719 /* Rehash gone out of credits so it was interrupted.
720 * Schedule the work as soon as possible to continue.
722 mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
724 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
728 mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(struct mlxsw_sp_acl_tcam_vchunk *vchunk)
730 struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
732 /* If a rule was added or deleted from vchunk which is currently
733 * under rehash migration, we have to reset the ventry pointers
734 * to make sure all rules are properly migrated.
736 if (vregion->rehash.ctx.current_vchunk == vchunk) {
737 vregion->rehash.ctx.start_ventry = NULL;
738 vregion->rehash.ctx.stop_ventry = NULL;
743 mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(struct mlxsw_sp_acl_tcam_vregion *vregion)
745 /* If a chunk was added or deleted from vregion we have to reset
746 * the current chunk pointer to make sure all chunks
747 * are properly migrated.
749 vregion->rehash.ctx.current_vchunk = NULL;
752 static struct mlxsw_sp_acl_tcam_vregion *
753 mlxsw_sp_acl_tcam_vregion_create(struct mlxsw_sp *mlxsw_sp,
754 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
755 unsigned int priority,
756 struct mlxsw_afk_element_usage *elusage)
758 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
759 struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
760 struct mlxsw_sp_acl_tcam *tcam = vgroup->group.tcam;
761 struct mlxsw_sp_acl_tcam_vregion *vregion;
764 vregion = kzalloc(sizeof(*vregion), GFP_KERNEL);
766 return ERR_PTR(-ENOMEM);
767 INIT_LIST_HEAD(&vregion->vchunk_list);
768 mutex_init(&vregion->lock);
769 vregion->tcam = tcam;
770 vregion->mlxsw_sp = mlxsw_sp;
771 vregion->vgroup = vgroup;
772 vregion->ref_count = 1;
774 vregion->key_info = mlxsw_afk_key_info_get(afk, elusage);
775 if (IS_ERR(vregion->key_info)) {
776 err = PTR_ERR(vregion->key_info);
777 goto err_key_info_get;
780 vregion->region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, tcam,
782 if (IS_ERR(vregion->region)) {
783 err = PTR_ERR(vregion->region);
784 goto err_region_create;
787 err = mlxsw_sp_acl_tcam_vgroup_vregion_attach(mlxsw_sp, vgroup, vregion,
790 goto err_vgroup_vregion_attach;
792 if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
793 /* Create the delayed work for vregion periodic rehash */
794 INIT_DELAYED_WORK(&vregion->rehash.dw,
795 mlxsw_sp_acl_tcam_vregion_rehash_work);
796 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
797 mutex_lock(&tcam->lock);
798 list_add_tail(&vregion->tlist, &tcam->vregion_list);
799 mutex_unlock(&tcam->lock);
804 err_vgroup_vregion_attach:
805 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
807 mlxsw_afk_key_info_put(vregion->key_info);
814 mlxsw_sp_acl_tcam_vregion_destroy(struct mlxsw_sp *mlxsw_sp,
815 struct mlxsw_sp_acl_tcam_vregion *vregion)
817 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
818 struct mlxsw_sp_acl_tcam_vgroup *vgroup = vregion->vgroup;
819 struct mlxsw_sp_acl_tcam *tcam = vregion->tcam;
821 if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
822 mutex_lock(&tcam->lock);
823 list_del(&vregion->tlist);
824 mutex_unlock(&tcam->lock);
825 cancel_delayed_work_sync(&vregion->rehash.dw);
827 mlxsw_sp_acl_tcam_vgroup_vregion_detach(mlxsw_sp, vregion);
828 if (vregion->region2)
829 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region2);
830 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
831 mlxsw_afk_key_info_put(vregion->key_info);
832 mutex_destroy(&vregion->lock);
836 static struct mlxsw_sp_acl_tcam_vregion *
837 mlxsw_sp_acl_tcam_vregion_get(struct mlxsw_sp *mlxsw_sp,
838 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
839 unsigned int priority,
840 struct mlxsw_afk_element_usage *elusage)
842 struct mlxsw_afk_element_usage vregion_elusage;
843 struct mlxsw_sp_acl_tcam_vregion *vregion;
846 vregion = mlxsw_sp_acl_tcam_vgroup_vregion_find(vgroup, priority,
847 elusage, &need_split);
850 /* According to priority, new vchunk should belong to
851 * an existing vregion. However, this vchunk needs
852 * elements that vregion does not contain. We need
853 * to split the existing vregion into two and create
854 * a new vregion for the new vchunk in between.
855 * This is not supported now.
857 return ERR_PTR(-EOPNOTSUPP);
859 vregion->ref_count++;
863 mlxsw_sp_acl_tcam_vgroup_use_patterns(vgroup, elusage,
866 return mlxsw_sp_acl_tcam_vregion_create(mlxsw_sp, vgroup, priority,
871 mlxsw_sp_acl_tcam_vregion_put(struct mlxsw_sp *mlxsw_sp,
872 struct mlxsw_sp_acl_tcam_vregion *vregion)
874 if (--vregion->ref_count)
876 mlxsw_sp_acl_tcam_vregion_destroy(mlxsw_sp, vregion);
879 static struct mlxsw_sp_acl_tcam_chunk *
880 mlxsw_sp_acl_tcam_chunk_create(struct mlxsw_sp *mlxsw_sp,
881 struct mlxsw_sp_acl_tcam_vchunk *vchunk,
882 struct mlxsw_sp_acl_tcam_region *region)
884 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
885 struct mlxsw_sp_acl_tcam_chunk *chunk;
887 chunk = kzalloc(sizeof(*chunk) + ops->chunk_priv_size, GFP_KERNEL);
889 return ERR_PTR(-ENOMEM);
890 chunk->vchunk = vchunk;
891 chunk->region = region;
893 ops->chunk_init(region->priv, chunk->priv, vchunk->priority);
898 mlxsw_sp_acl_tcam_chunk_destroy(struct mlxsw_sp *mlxsw_sp,
899 struct mlxsw_sp_acl_tcam_chunk *chunk)
901 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
903 ops->chunk_fini(chunk->priv);
907 static struct mlxsw_sp_acl_tcam_vchunk *
908 mlxsw_sp_acl_tcam_vchunk_create(struct mlxsw_sp *mlxsw_sp,
909 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
910 unsigned int priority,
911 struct mlxsw_afk_element_usage *elusage)
913 struct mlxsw_sp_acl_tcam_vchunk *vchunk, *vchunk2;
914 struct mlxsw_sp_acl_tcam_vregion *vregion;
915 struct list_head *pos;
918 if (priority == MLXSW_SP_ACL_TCAM_CATCHALL_PRIO)
919 return ERR_PTR(-EINVAL);
921 vchunk = kzalloc(sizeof(*vchunk), GFP_KERNEL);
923 return ERR_PTR(-ENOMEM);
924 INIT_LIST_HEAD(&vchunk->ventry_list);
925 vchunk->priority = priority;
926 vchunk->vgroup = vgroup;
927 vchunk->ref_count = 1;
929 vregion = mlxsw_sp_acl_tcam_vregion_get(mlxsw_sp, vgroup,
931 if (IS_ERR(vregion)) {
932 err = PTR_ERR(vregion);
933 goto err_vregion_get;
936 vchunk->vregion = vregion;
938 err = rhashtable_insert_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
939 mlxsw_sp_acl_tcam_vchunk_ht_params);
941 goto err_rhashtable_insert;
943 mutex_lock(&vregion->lock);
944 vchunk->chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk,
945 vchunk->vregion->region);
946 if (IS_ERR(vchunk->chunk)) {
947 mutex_unlock(&vregion->lock);
948 err = PTR_ERR(vchunk->chunk);
949 goto err_chunk_create;
952 mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
954 /* Position the vchunk inside the list according to priority */
955 list_for_each(pos, &vregion->vchunk_list) {
956 vchunk2 = list_entry(pos, typeof(*vchunk2), list);
957 if (vchunk2->priority > priority)
960 list_add_tail(&vchunk->list, pos);
961 mutex_unlock(&vregion->lock);
962 mlxsw_sp_acl_tcam_vgroup_prio_update(vgroup);
967 rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
968 mlxsw_sp_acl_tcam_vchunk_ht_params);
969 err_rhashtable_insert:
970 mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vregion);
977 mlxsw_sp_acl_tcam_vchunk_destroy(struct mlxsw_sp *mlxsw_sp,
978 struct mlxsw_sp_acl_tcam_vchunk *vchunk)
980 struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
981 struct mlxsw_sp_acl_tcam_vgroup *vgroup = vchunk->vgroup;
983 mutex_lock(&vregion->lock);
984 mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
985 list_del(&vchunk->list);
987 mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
988 mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk);
989 mutex_unlock(&vregion->lock);
990 rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
991 mlxsw_sp_acl_tcam_vchunk_ht_params);
992 mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vchunk->vregion);
994 mlxsw_sp_acl_tcam_vgroup_prio_update(vgroup);
997 static struct mlxsw_sp_acl_tcam_vchunk *
998 mlxsw_sp_acl_tcam_vchunk_get(struct mlxsw_sp *mlxsw_sp,
999 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1000 unsigned int priority,
1001 struct mlxsw_afk_element_usage *elusage)
1003 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1005 vchunk = rhashtable_lookup_fast(&vgroup->vchunk_ht, &priority,
1006 mlxsw_sp_acl_tcam_vchunk_ht_params);
1008 if (WARN_ON(!mlxsw_afk_key_info_subset(vchunk->vregion->key_info,
1010 return ERR_PTR(-EINVAL);
1011 vchunk->ref_count++;
1014 return mlxsw_sp_acl_tcam_vchunk_create(mlxsw_sp, vgroup,
1019 mlxsw_sp_acl_tcam_vchunk_put(struct mlxsw_sp *mlxsw_sp,
1020 struct mlxsw_sp_acl_tcam_vchunk *vchunk)
1022 if (--vchunk->ref_count)
1024 mlxsw_sp_acl_tcam_vchunk_destroy(mlxsw_sp, vchunk);
1027 static struct mlxsw_sp_acl_tcam_entry *
1028 mlxsw_sp_acl_tcam_entry_create(struct mlxsw_sp *mlxsw_sp,
1029 struct mlxsw_sp_acl_tcam_ventry *ventry,
1030 struct mlxsw_sp_acl_tcam_chunk *chunk)
1032 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1033 struct mlxsw_sp_acl_tcam_entry *entry;
1036 entry = kzalloc(sizeof(*entry) + ops->entry_priv_size, GFP_KERNEL);
1038 return ERR_PTR(-ENOMEM);
1039 entry->ventry = ventry;
1040 entry->chunk = chunk;
1042 err = ops->entry_add(mlxsw_sp, chunk->region->priv, chunk->priv,
1043 entry->priv, ventry->rulei);
1051 return ERR_PTR(err);
1054 static void mlxsw_sp_acl_tcam_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1055 struct mlxsw_sp_acl_tcam_entry *entry)
1057 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1059 ops->entry_del(mlxsw_sp, entry->chunk->region->priv,
1060 entry->chunk->priv, entry->priv);
1065 mlxsw_sp_acl_tcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
1066 struct mlxsw_sp_acl_tcam_region *region,
1067 struct mlxsw_sp_acl_tcam_entry *entry,
1068 struct mlxsw_sp_acl_rule_info *rulei)
1070 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1072 return ops->entry_action_replace(mlxsw_sp, region->priv,
1073 entry->priv, rulei);
1077 mlxsw_sp_acl_tcam_entry_activity_get(struct mlxsw_sp *mlxsw_sp,
1078 struct mlxsw_sp_acl_tcam_entry *entry,
1081 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1083 return ops->entry_activity_get(mlxsw_sp, entry->chunk->region->priv,
1084 entry->priv, activity);
1087 static int mlxsw_sp_acl_tcam_ventry_add(struct mlxsw_sp *mlxsw_sp,
1088 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1089 struct mlxsw_sp_acl_tcam_ventry *ventry,
1090 struct mlxsw_sp_acl_rule_info *rulei)
1092 struct mlxsw_sp_acl_tcam_vregion *vregion;
1093 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1096 vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp, vgroup, rulei->priority,
1097 &rulei->values.elusage);
1099 return PTR_ERR(vchunk);
1101 ventry->vchunk = vchunk;
1102 ventry->rulei = rulei;
1103 vregion = vchunk->vregion;
1105 mutex_lock(&vregion->lock);
1106 ventry->entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry,
1108 if (IS_ERR(ventry->entry)) {
1109 mutex_unlock(&vregion->lock);
1110 err = PTR_ERR(ventry->entry);
1111 goto err_entry_create;
1114 list_add_tail(&ventry->list, &vchunk->ventry_list);
1115 mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1116 mutex_unlock(&vregion->lock);
1121 mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1125 static void mlxsw_sp_acl_tcam_ventry_del(struct mlxsw_sp *mlxsw_sp,
1126 struct mlxsw_sp_acl_tcam_ventry *ventry)
1128 struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1129 struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
1131 mutex_lock(&vregion->lock);
1132 mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1133 list_del(&ventry->list);
1134 mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1135 mutex_unlock(&vregion->lock);
1136 mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1140 mlxsw_sp_acl_tcam_ventry_action_replace(struct mlxsw_sp *mlxsw_sp,
1141 struct mlxsw_sp_acl_tcam_ventry *ventry,
1142 struct mlxsw_sp_acl_rule_info *rulei)
1144 struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1146 return mlxsw_sp_acl_tcam_entry_action_replace(mlxsw_sp,
1147 vchunk->vregion->region,
1148 ventry->entry, rulei);
1152 mlxsw_sp_acl_tcam_ventry_activity_get(struct mlxsw_sp *mlxsw_sp,
1153 struct mlxsw_sp_acl_tcam_ventry *ventry,
1156 return mlxsw_sp_acl_tcam_entry_activity_get(mlxsw_sp,
1157 ventry->entry, activity);
1161 mlxsw_sp_acl_tcam_ventry_migrate(struct mlxsw_sp *mlxsw_sp,
1162 struct mlxsw_sp_acl_tcam_ventry *ventry,
1163 struct mlxsw_sp_acl_tcam_chunk *chunk,
1166 struct mlxsw_sp_acl_tcam_entry *new_entry;
1168 /* First check if the entry is not already where we want it to be. */
1169 if (ventry->entry->chunk == chunk)
1172 if (--(*credits) < 0)
1175 new_entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry, chunk);
1176 if (IS_ERR(new_entry))
1177 return PTR_ERR(new_entry);
1178 mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1179 ventry->entry = new_entry;
1184 mlxsw_sp_acl_tcam_vchunk_migrate_start(struct mlxsw_sp *mlxsw_sp,
1185 struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1186 struct mlxsw_sp_acl_tcam_region *region,
1187 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1189 struct mlxsw_sp_acl_tcam_chunk *new_chunk;
1191 new_chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk, region);
1192 if (IS_ERR(new_chunk))
1193 return PTR_ERR(new_chunk);
1194 vchunk->chunk2 = vchunk->chunk;
1195 vchunk->chunk = new_chunk;
1196 ctx->current_vchunk = vchunk;
1197 ctx->start_ventry = NULL;
1198 ctx->stop_ventry = NULL;
1203 mlxsw_sp_acl_tcam_vchunk_migrate_end(struct mlxsw_sp *mlxsw_sp,
1204 struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1205 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1207 mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
1208 vchunk->chunk2 = NULL;
1209 ctx->current_vchunk = NULL;
1213 mlxsw_sp_acl_tcam_vchunk_migrate_one(struct mlxsw_sp *mlxsw_sp,
1214 struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1215 struct mlxsw_sp_acl_tcam_region *region,
1216 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1219 struct mlxsw_sp_acl_tcam_ventry *ventry;
1222 if (vchunk->chunk->region != region) {
1223 err = mlxsw_sp_acl_tcam_vchunk_migrate_start(mlxsw_sp, vchunk,
1227 } else if (!vchunk->chunk2) {
1228 /* The chunk is already as it should be, nothing to do. */
1232 /* If the migration got interrupted, we have the ventry to start from
1233 * stored in context.
1235 if (ctx->start_ventry)
1236 ventry = ctx->start_ventry;
1238 ventry = list_first_entry(&vchunk->ventry_list,
1239 typeof(*ventry), list);
1241 list_for_each_entry_from(ventry, &vchunk->ventry_list, list) {
1242 /* During rollback, once we reach the ventry that failed
1243 * to migrate, we are done.
1245 if (ventry == ctx->stop_ventry)
1248 err = mlxsw_sp_acl_tcam_ventry_migrate(mlxsw_sp, ventry,
1249 vchunk->chunk, credits);
1251 if (ctx->this_is_rollback) {
1252 /* Save the ventry which we ended with and try
1253 * to continue later on.
1255 ctx->start_ventry = ventry;
1258 /* Swap the chunk and chunk2 pointers so the follow-up
1259 * rollback call will see the original chunk pointer
1262 swap(vchunk->chunk, vchunk->chunk2);
1263 /* The rollback has to be done from beginning of the
1264 * chunk, that is why we have to null the start_ventry.
1265 * However, we know where to stop the rollback,
1266 * at the current ventry.
1268 ctx->start_ventry = NULL;
1269 ctx->stop_ventry = ventry;
1271 } else if (*credits < 0) {
1272 /* We are out of credits, the rest of the ventries
1273 * will be migrated later. Save the ventry
1274 * which we ended with.
1276 ctx->start_ventry = ventry;
1281 mlxsw_sp_acl_tcam_vchunk_migrate_end(mlxsw_sp, vchunk, ctx);
1286 mlxsw_sp_acl_tcam_vchunk_migrate_all(struct mlxsw_sp *mlxsw_sp,
1287 struct mlxsw_sp_acl_tcam_vregion *vregion,
1288 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1291 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1294 /* If the migration got interrupted, we have the vchunk
1295 * we are working on stored in context.
1297 if (ctx->current_vchunk)
1298 vchunk = ctx->current_vchunk;
1300 vchunk = list_first_entry(&vregion->vchunk_list,
1301 typeof(*vchunk), list);
1303 list_for_each_entry_from(vchunk, &vregion->vchunk_list, list) {
1304 err = mlxsw_sp_acl_tcam_vchunk_migrate_one(mlxsw_sp, vchunk,
1307 if (err || *credits < 0)
1314 mlxsw_sp_acl_tcam_vregion_migrate(struct mlxsw_sp *mlxsw_sp,
1315 struct mlxsw_sp_acl_tcam_vregion *vregion,
1316 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1321 trace_mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion);
1322 mutex_lock(&vregion->lock);
1323 err = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1326 /* In case migration was not successful, we need to swap
1327 * so the original region pointer is assigned again
1328 * to vregion->region.
1330 swap(vregion->region, vregion->region2);
1331 ctx->current_vchunk = NULL;
1332 ctx->this_is_rollback = true;
1333 err2 = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1336 trace_mlxsw_sp_acl_tcam_vregion_rehash_rollback_failed(mlxsw_sp,
1338 dev_err(mlxsw_sp->bus_info->dev, "Failed to rollback during vregion migration fail\n");
1339 /* Let the rollback to be continued later on. */
1342 mutex_unlock(&vregion->lock);
1343 trace_mlxsw_sp_acl_tcam_vregion_migrate_end(mlxsw_sp, vregion);
1348 mlxsw_sp_acl_tcam_vregion_rehash_in_progress(const struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1350 return ctx->hints_priv;
1354 mlxsw_sp_acl_tcam_vregion_rehash_start(struct mlxsw_sp *mlxsw_sp,
1355 struct mlxsw_sp_acl_tcam_vregion *vregion,
1356 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1358 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1359 unsigned int priority = mlxsw_sp_acl_tcam_vregion_prio(vregion);
1360 struct mlxsw_sp_acl_tcam_region *new_region;
1364 trace_mlxsw_sp_acl_tcam_vregion_rehash(mlxsw_sp, vregion);
1366 hints_priv = ops->region_rehash_hints_get(vregion->region->priv);
1367 if (IS_ERR(hints_priv))
1368 return PTR_ERR(hints_priv);
1370 new_region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, vregion->tcam,
1371 vregion, hints_priv);
1372 if (IS_ERR(new_region)) {
1373 err = PTR_ERR(new_region);
1374 goto err_region_create;
1377 /* vregion->region contains the pointer to the new region
1378 * we are going to migrate to.
1380 vregion->region2 = vregion->region;
1381 vregion->region = new_region;
1382 err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp,
1383 vregion->region2->group,
1384 new_region, priority,
1387 goto err_group_region_attach;
1389 ctx->hints_priv = hints_priv;
1390 ctx->this_is_rollback = false;
1394 err_group_region_attach:
1395 vregion->region = vregion->region2;
1396 vregion->region2 = NULL;
1397 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, new_region);
1399 ops->region_rehash_hints_put(hints_priv);
1404 mlxsw_sp_acl_tcam_vregion_rehash_end(struct mlxsw_sp *mlxsw_sp,
1405 struct mlxsw_sp_acl_tcam_vregion *vregion,
1406 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1408 struct mlxsw_sp_acl_tcam_region *unused_region = vregion->region2;
1409 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1411 vregion->region2 = NULL;
1412 mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, unused_region);
1413 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, unused_region);
1414 ops->region_rehash_hints_put(ctx->hints_priv);
1415 ctx->hints_priv = NULL;
1419 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
1420 struct mlxsw_sp_acl_tcam_vregion *vregion,
1423 struct mlxsw_sp_acl_tcam_rehash_ctx *ctx = &vregion->rehash.ctx;
1426 /* Check if the previous rehash work was interrupted
1427 * which means we have to continue it now.
1428 * If not, start a new rehash.
1430 if (!mlxsw_sp_acl_tcam_vregion_rehash_in_progress(ctx)) {
1431 err = mlxsw_sp_acl_tcam_vregion_rehash_start(mlxsw_sp,
1435 dev_err(mlxsw_sp->bus_info->dev, "Failed get rehash hints\n");
1440 err = mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion,
1443 dev_err(mlxsw_sp->bus_info->dev, "Failed to migrate vregion\n");
1447 mlxsw_sp_acl_tcam_vregion_rehash_end(mlxsw_sp, vregion, ctx);
1451 mlxsw_sp_acl_tcam_region_rehash_intrvl_get(struct devlink *devlink, u32 id,
1452 struct devlink_param_gset_ctx *ctx)
1454 struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1455 struct mlxsw_sp_acl_tcam *tcam;
1456 struct mlxsw_sp *mlxsw_sp;
1458 mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1459 tcam = mlxsw_sp_acl_to_tcam(mlxsw_sp->acl);
1460 ctx->val.vu32 = tcam->vregion_rehash_intrvl;
1466 mlxsw_sp_acl_tcam_region_rehash_intrvl_set(struct devlink *devlink, u32 id,
1467 struct devlink_param_gset_ctx *ctx)
1469 struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1470 struct mlxsw_sp_acl_tcam_vregion *vregion;
1471 struct mlxsw_sp_acl_tcam *tcam;
1472 struct mlxsw_sp *mlxsw_sp;
1473 u32 val = ctx->val.vu32;
1475 if (val < MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN && val)
1478 mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1479 tcam = mlxsw_sp_acl_to_tcam(mlxsw_sp->acl);
1480 tcam->vregion_rehash_intrvl = val;
1481 mutex_lock(&tcam->lock);
1482 list_for_each_entry(vregion, &tcam->vregion_list, tlist) {
1484 mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
1486 cancel_delayed_work_sync(&vregion->rehash.dw);
1488 mutex_unlock(&tcam->lock);
1492 static const struct devlink_param mlxsw_sp_acl_tcam_rehash_params[] = {
1493 DEVLINK_PARAM_DRIVER(MLXSW_DEVLINK_PARAM_ID_ACL_REGION_REHASH_INTERVAL,
1494 "acl_region_rehash_interval",
1495 DEVLINK_PARAM_TYPE_U32,
1496 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
1497 mlxsw_sp_acl_tcam_region_rehash_intrvl_get,
1498 mlxsw_sp_acl_tcam_region_rehash_intrvl_set,
1502 static int mlxsw_sp_acl_tcam_rehash_params_register(struct mlxsw_sp *mlxsw_sp)
1504 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1506 if (!mlxsw_sp->acl_tcam_ops->region_rehash_hints_get)
1509 return devl_params_register(devlink, mlxsw_sp_acl_tcam_rehash_params,
1510 ARRAY_SIZE(mlxsw_sp_acl_tcam_rehash_params));
1514 mlxsw_sp_acl_tcam_rehash_params_unregister(struct mlxsw_sp *mlxsw_sp)
1516 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1518 if (!mlxsw_sp->acl_tcam_ops->region_rehash_hints_get)
1521 devl_params_unregister(devlink, mlxsw_sp_acl_tcam_rehash_params,
1522 ARRAY_SIZE(mlxsw_sp_acl_tcam_rehash_params));
1525 int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp,
1526 struct mlxsw_sp_acl_tcam *tcam)
1528 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1529 u64 max_tcam_regions;
1534 mutex_init(&tcam->lock);
1535 tcam->vregion_rehash_intrvl =
1536 MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT;
1537 INIT_LIST_HEAD(&tcam->vregion_list);
1539 err = mlxsw_sp_acl_tcam_rehash_params_register(mlxsw_sp);
1541 goto err_rehash_params_register;
1543 max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1544 ACL_MAX_TCAM_REGIONS);
1545 max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
1547 /* Use 1:1 mapping between ACL region and TCAM region */
1548 if (max_tcam_regions < max_regions)
1549 max_regions = max_tcam_regions;
1551 tcam->used_regions = bitmap_zalloc(max_regions, GFP_KERNEL);
1552 if (!tcam->used_regions) {
1554 goto err_alloc_used_regions;
1556 tcam->max_regions = max_regions;
1558 max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS);
1559 tcam->used_groups = bitmap_zalloc(max_groups, GFP_KERNEL);
1560 if (!tcam->used_groups) {
1562 goto err_alloc_used_groups;
1564 tcam->max_groups = max_groups;
1565 tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1566 ACL_MAX_GROUP_SIZE);
1568 err = ops->init(mlxsw_sp, tcam->priv, tcam);
1575 bitmap_free(tcam->used_groups);
1576 err_alloc_used_groups:
1577 bitmap_free(tcam->used_regions);
1578 err_alloc_used_regions:
1579 mlxsw_sp_acl_tcam_rehash_params_unregister(mlxsw_sp);
1580 err_rehash_params_register:
1581 mutex_destroy(&tcam->lock);
1585 void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp,
1586 struct mlxsw_sp_acl_tcam *tcam)
1588 const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1590 ops->fini(mlxsw_sp, tcam->priv);
1591 bitmap_free(tcam->used_groups);
1592 bitmap_free(tcam->used_regions);
1593 mlxsw_sp_acl_tcam_rehash_params_unregister(mlxsw_sp);
1594 mutex_destroy(&tcam->lock);
1597 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = {
1598 MLXSW_AFK_ELEMENT_SRC_SYS_PORT,
1599 MLXSW_AFK_ELEMENT_DMAC_32_47,
1600 MLXSW_AFK_ELEMENT_DMAC_0_31,
1601 MLXSW_AFK_ELEMENT_SMAC_32_47,
1602 MLXSW_AFK_ELEMENT_SMAC_0_31,
1603 MLXSW_AFK_ELEMENT_ETHERTYPE,
1604 MLXSW_AFK_ELEMENT_IP_PROTO,
1605 MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1606 MLXSW_AFK_ELEMENT_DST_IP_0_31,
1607 MLXSW_AFK_ELEMENT_DST_L4_PORT,
1608 MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1609 MLXSW_AFK_ELEMENT_VID,
1610 MLXSW_AFK_ELEMENT_PCP,
1611 MLXSW_AFK_ELEMENT_TCP_FLAGS,
1612 MLXSW_AFK_ELEMENT_IP_TTL_,
1613 MLXSW_AFK_ELEMENT_IP_ECN,
1614 MLXSW_AFK_ELEMENT_IP_DSCP,
1617 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
1618 MLXSW_AFK_ELEMENT_ETHERTYPE,
1619 MLXSW_AFK_ELEMENT_IP_PROTO,
1620 MLXSW_AFK_ELEMENT_SRC_IP_96_127,
1621 MLXSW_AFK_ELEMENT_SRC_IP_64_95,
1622 MLXSW_AFK_ELEMENT_SRC_IP_32_63,
1623 MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1624 MLXSW_AFK_ELEMENT_DST_IP_96_127,
1625 MLXSW_AFK_ELEMENT_DST_IP_64_95,
1626 MLXSW_AFK_ELEMENT_DST_IP_32_63,
1627 MLXSW_AFK_ELEMENT_DST_IP_0_31,
1628 MLXSW_AFK_ELEMENT_DST_L4_PORT,
1629 MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1632 static const struct mlxsw_sp_acl_tcam_pattern mlxsw_sp_acl_tcam_patterns[] = {
1634 .elements = mlxsw_sp_acl_tcam_pattern_ipv4,
1635 .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv4),
1638 .elements = mlxsw_sp_acl_tcam_pattern_ipv6,
1639 .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv6),
1643 #define MLXSW_SP_ACL_TCAM_PATTERNS_COUNT \
1644 ARRAY_SIZE(mlxsw_sp_acl_tcam_patterns)
1646 struct mlxsw_sp_acl_tcam_flower_ruleset {
1647 struct mlxsw_sp_acl_tcam_vgroup vgroup;
1650 struct mlxsw_sp_acl_tcam_flower_rule {
1651 struct mlxsw_sp_acl_tcam_ventry ventry;
1655 mlxsw_sp_acl_tcam_flower_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1656 struct mlxsw_sp_acl_tcam *tcam,
1658 struct mlxsw_afk_element_usage *tmplt_elusage,
1659 unsigned int *p_min_prio,
1660 unsigned int *p_max_prio)
1662 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1664 return mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1665 mlxsw_sp_acl_tcam_patterns,
1666 MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1667 tmplt_elusage, true,
1668 p_min_prio, p_max_prio);
1672 mlxsw_sp_acl_tcam_flower_ruleset_del(struct mlxsw_sp *mlxsw_sp,
1675 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1677 mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1681 mlxsw_sp_acl_tcam_flower_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
1683 struct mlxsw_sp_port *mlxsw_sp_port,
1686 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1688 return mlxsw_sp_acl_tcam_group_bind(mlxsw_sp, &ruleset->vgroup.group,
1689 mlxsw_sp_port, ingress);
1693 mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1695 struct mlxsw_sp_port *mlxsw_sp_port,
1698 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1700 mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->vgroup.group,
1701 mlxsw_sp_port, ingress);
1705 mlxsw_sp_acl_tcam_flower_ruleset_group_id(void *ruleset_priv)
1707 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1709 return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1713 mlxsw_sp_acl_tcam_flower_rule_add(struct mlxsw_sp *mlxsw_sp,
1714 void *ruleset_priv, void *rule_priv,
1715 struct mlxsw_sp_acl_rule_info *rulei)
1717 struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1718 struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1720 return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1721 &rule->ventry, rulei);
1725 mlxsw_sp_acl_tcam_flower_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1727 struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1729 mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1733 mlxsw_sp_acl_tcam_flower_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1735 struct mlxsw_sp_acl_rule_info *rulei)
1741 mlxsw_sp_acl_tcam_flower_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1742 void *rule_priv, bool *activity)
1744 struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1746 return mlxsw_sp_acl_tcam_ventry_activity_get(mlxsw_sp, &rule->ventry,
1750 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_flower_ops = {
1751 .ruleset_priv_size = sizeof(struct mlxsw_sp_acl_tcam_flower_ruleset),
1752 .ruleset_add = mlxsw_sp_acl_tcam_flower_ruleset_add,
1753 .ruleset_del = mlxsw_sp_acl_tcam_flower_ruleset_del,
1754 .ruleset_bind = mlxsw_sp_acl_tcam_flower_ruleset_bind,
1755 .ruleset_unbind = mlxsw_sp_acl_tcam_flower_ruleset_unbind,
1756 .ruleset_group_id = mlxsw_sp_acl_tcam_flower_ruleset_group_id,
1757 .rule_priv_size = sizeof(struct mlxsw_sp_acl_tcam_flower_rule),
1758 .rule_add = mlxsw_sp_acl_tcam_flower_rule_add,
1759 .rule_del = mlxsw_sp_acl_tcam_flower_rule_del,
1760 .rule_action_replace = mlxsw_sp_acl_tcam_flower_rule_action_replace,
1761 .rule_activity_get = mlxsw_sp_acl_tcam_flower_rule_activity_get,
1764 struct mlxsw_sp_acl_tcam_mr_ruleset {
1765 struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1766 struct mlxsw_sp_acl_tcam_vgroup vgroup;
1769 struct mlxsw_sp_acl_tcam_mr_rule {
1770 struct mlxsw_sp_acl_tcam_ventry ventry;
1774 mlxsw_sp_acl_tcam_mr_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1775 struct mlxsw_sp_acl_tcam *tcam,
1777 struct mlxsw_afk_element_usage *tmplt_elusage,
1778 unsigned int *p_min_prio,
1779 unsigned int *p_max_prio)
1781 struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1784 err = mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1785 mlxsw_sp_acl_tcam_patterns,
1786 MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1787 tmplt_elusage, false,
1788 p_min_prio, p_max_prio);
1792 /* For most of the TCAM clients it would make sense to take a tcam chunk
1793 * only when the first rule is written. This is not the case for
1794 * multicast router as it is required to bind the multicast router to a
1795 * specific ACL Group ID which must exist in HW before multicast router
1798 ruleset->vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp,
1799 &ruleset->vgroup, 1,
1801 if (IS_ERR(ruleset->vchunk)) {
1802 err = PTR_ERR(ruleset->vchunk);
1809 mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1814 mlxsw_sp_acl_tcam_mr_ruleset_del(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv)
1816 struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1818 mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, ruleset->vchunk);
1819 mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1823 mlxsw_sp_acl_tcam_mr_ruleset_bind(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1824 struct mlxsw_sp_port *mlxsw_sp_port,
1827 /* Binding is done when initializing multicast router */
1832 mlxsw_sp_acl_tcam_mr_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1834 struct mlxsw_sp_port *mlxsw_sp_port,
1840 mlxsw_sp_acl_tcam_mr_ruleset_group_id(void *ruleset_priv)
1842 struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1844 return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1848 mlxsw_sp_acl_tcam_mr_rule_add(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1850 struct mlxsw_sp_acl_rule_info *rulei)
1852 struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1853 struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1855 return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1856 &rule->ventry, rulei);
1860 mlxsw_sp_acl_tcam_mr_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1862 struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1864 mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1868 mlxsw_sp_acl_tcam_mr_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1870 struct mlxsw_sp_acl_rule_info *rulei)
1872 struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1874 return mlxsw_sp_acl_tcam_ventry_action_replace(mlxsw_sp, &rule->ventry,
1879 mlxsw_sp_acl_tcam_mr_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1880 void *rule_priv, bool *activity)
1887 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_mr_ops = {
1888 .ruleset_priv_size = sizeof(struct mlxsw_sp_acl_tcam_mr_ruleset),
1889 .ruleset_add = mlxsw_sp_acl_tcam_mr_ruleset_add,
1890 .ruleset_del = mlxsw_sp_acl_tcam_mr_ruleset_del,
1891 .ruleset_bind = mlxsw_sp_acl_tcam_mr_ruleset_bind,
1892 .ruleset_unbind = mlxsw_sp_acl_tcam_mr_ruleset_unbind,
1893 .ruleset_group_id = mlxsw_sp_acl_tcam_mr_ruleset_group_id,
1894 .rule_priv_size = sizeof(struct mlxsw_sp_acl_tcam_mr_rule),
1895 .rule_add = mlxsw_sp_acl_tcam_mr_rule_add,
1896 .rule_del = mlxsw_sp_acl_tcam_mr_rule_del,
1897 .rule_action_replace = mlxsw_sp_acl_tcam_mr_rule_action_replace,
1898 .rule_activity_get = mlxsw_sp_acl_tcam_mr_rule_activity_get,
1901 static const struct mlxsw_sp_acl_profile_ops *
1902 mlxsw_sp_acl_tcam_profile_ops_arr[] = {
1903 [MLXSW_SP_ACL_PROFILE_FLOWER] = &mlxsw_sp_acl_tcam_flower_ops,
1904 [MLXSW_SP_ACL_PROFILE_MR] = &mlxsw_sp_acl_tcam_mr_ops,
1907 const struct mlxsw_sp_acl_profile_ops *
1908 mlxsw_sp_acl_tcam_profile_ops(struct mlxsw_sp *mlxsw_sp,
1909 enum mlxsw_sp_acl_profile profile)
1911 const struct mlxsw_sp_acl_profile_ops *ops;
1913 if (WARN_ON(profile >= ARRAY_SIZE(mlxsw_sp_acl_tcam_profile_ops_arr)))
1915 ops = mlxsw_sp_acl_tcam_profile_ops_arr[profile];