1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
4 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/refcount.h>
9 #include <linux/rhashtable.h>
14 #include "spectrum_acl_tcam.h"
15 #include "core_acl_flex_keys.h"
17 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_START 6
18 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_END 11
20 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
21 char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
25 struct mlxsw_sp_acl_atcam_lkey_id {
26 struct rhash_head ht_node;
27 struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
32 struct mlxsw_sp_acl_atcam_region_ops {
33 int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
34 void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
35 struct mlxsw_sp_acl_atcam_lkey_id *
36 (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
37 struct mlxsw_sp_acl_rule_info *rulei, u8 erp_id);
38 void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
39 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
42 struct mlxsw_sp_acl_atcam_region_generic {
43 struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
46 struct mlxsw_sp_acl_atcam_region_12kb {
47 struct rhashtable lkey_ht;
48 unsigned int max_lkey_id;
49 unsigned long *used_lkey_id;
52 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
53 .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
54 .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
55 .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
58 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
59 .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
60 .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
61 .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
65 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
67 return mlxsw_sp_acl_erp_is_ctcam_erp(aentry->erp);
71 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
73 struct mlxsw_sp_acl_atcam_region_generic *region_generic;
75 region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
79 refcount_set(®ion_generic->dummy_lkey_id.refcnt, 1);
80 aregion->priv = region_generic;
86 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
91 static struct mlxsw_sp_acl_atcam_lkey_id *
92 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
93 struct mlxsw_sp_acl_rule_info *rulei,
96 struct mlxsw_sp_acl_atcam_region_generic *region_generic;
98 region_generic = aregion->priv;
99 return ®ion_generic->dummy_lkey_id;
103 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
104 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
108 static const struct mlxsw_sp_acl_atcam_region_ops
109 mlxsw_sp_acl_atcam_region_generic_ops = {
110 .init = mlxsw_sp_acl_atcam_region_generic_init,
111 .fini = mlxsw_sp_acl_atcam_region_generic_fini,
112 .lkey_id_get = mlxsw_sp_acl_atcam_generic_lkey_id_get,
113 .lkey_id_put = mlxsw_sp_acl_atcam_generic_lkey_id_put,
117 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
119 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
120 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
125 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
128 max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
129 region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
133 alloc_size = BITS_TO_LONGS(max_lkey_id) * sizeof(unsigned long);
134 region_12kb->used_lkey_id = kzalloc(alloc_size, GFP_KERNEL);
135 if (!region_12kb->used_lkey_id) {
137 goto err_used_lkey_id_alloc;
140 err = rhashtable_init(®ion_12kb->lkey_ht,
141 &mlxsw_sp_acl_atcam_lkey_id_ht_params);
143 goto err_rhashtable_init;
145 region_12kb->max_lkey_id = max_lkey_id;
146 aregion->priv = region_12kb;
151 kfree(region_12kb->used_lkey_id);
152 err_used_lkey_id_alloc:
158 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
160 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
162 rhashtable_destroy(®ion_12kb->lkey_ht);
163 kfree(region_12kb->used_lkey_id);
167 static struct mlxsw_sp_acl_atcam_lkey_id *
168 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
169 struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
171 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
172 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
176 id = find_first_zero_bit(region_12kb->used_lkey_id,
177 region_12kb->max_lkey_id);
178 if (id < region_12kb->max_lkey_id)
179 __set_bit(id, region_12kb->used_lkey_id);
181 return ERR_PTR(-ENOBUFS);
183 lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
186 goto err_lkey_id_alloc;
190 memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
191 refcount_set(&lkey_id->refcnt, 1);
193 err = rhashtable_insert_fast(®ion_12kb->lkey_ht,
195 mlxsw_sp_acl_atcam_lkey_id_ht_params);
197 goto err_rhashtable_insert;
201 err_rhashtable_insert:
204 __clear_bit(id, region_12kb->used_lkey_id);
209 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
210 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
212 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
213 u32 id = lkey_id->id;
215 rhashtable_remove_fast(®ion_12kb->lkey_ht, &lkey_id->ht_node,
216 mlxsw_sp_acl_atcam_lkey_id_ht_params);
218 __clear_bit(id, region_12kb->used_lkey_id);
221 static struct mlxsw_sp_acl_atcam_lkey_id *
222 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
223 struct mlxsw_sp_acl_rule_info *rulei,
226 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
227 struct mlxsw_sp_acl_tcam_region *region = aregion->region;
228 struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
229 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
230 struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
231 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
233 mlxsw_afk_encode(afk, region->key_info, &rulei->values, ht_key.enc_key,
234 NULL, MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_START,
235 MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_END);
236 ht_key.erp_id = erp_id;
237 lkey_id = rhashtable_lookup_fast(®ion_12kb->lkey_ht, &ht_key,
238 mlxsw_sp_acl_atcam_lkey_id_ht_params);
240 refcount_inc(&lkey_id->refcnt);
244 return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
248 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
249 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
251 if (refcount_dec_and_test(&lkey_id->refcnt))
252 mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
255 static const struct mlxsw_sp_acl_atcam_region_ops
256 mlxsw_sp_acl_atcam_region_12kb_ops = {
257 .init = mlxsw_sp_acl_atcam_region_12kb_init,
258 .fini = mlxsw_sp_acl_atcam_region_12kb_fini,
259 .lkey_id_get = mlxsw_sp_acl_atcam_12kb_lkey_id_get,
260 .lkey_id_put = mlxsw_sp_acl_atcam_12kb_lkey_id_put,
263 static const struct mlxsw_sp_acl_atcam_region_ops *
264 mlxsw_sp_acl_atcam_region_ops_arr[] = {
265 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] =
266 &mlxsw_sp_acl_atcam_region_generic_ops,
267 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] =
268 &mlxsw_sp_acl_atcam_region_generic_ops,
269 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] =
270 &mlxsw_sp_acl_atcam_region_generic_ops,
271 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] =
272 &mlxsw_sp_acl_atcam_region_12kb_ops,
275 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
278 char perar_pl[MLXSW_REG_PERAR_LEN];
279 /* For now, just assume that every region has 12 key blocks */
280 u16 hw_region = region_id * 3;
283 max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
284 if (hw_region >= max_regions)
287 mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
288 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
292 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
294 struct mlxsw_sp_acl_tcam_region *region = aregion->region;
295 enum mlxsw_sp_acl_atcam_region_type region_type;
296 unsigned int blocks_count;
298 /* We already know the blocks count can not exceed the maximum
301 blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
302 if (blocks_count <= 2)
303 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
304 else if (blocks_count <= 4)
305 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
306 else if (blocks_count <= 8)
307 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
309 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
311 aregion->type = region_type;
312 aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
316 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
317 struct mlxsw_sp_acl_atcam *atcam,
318 struct mlxsw_sp_acl_atcam_region *aregion,
319 struct mlxsw_sp_acl_tcam_region *region,
320 const struct mlxsw_sp_acl_ctcam_region_ops *ops)
324 aregion->region = region;
325 aregion->atcam = atcam;
326 mlxsw_sp_acl_atcam_region_type_init(aregion);
328 err = rhashtable_init(&aregion->entries_ht,
329 &mlxsw_sp_acl_atcam_entries_ht_params);
332 err = aregion->ops->init(aregion);
335 err = mlxsw_sp_acl_erp_region_init(aregion);
337 goto err_erp_region_init;
338 err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
341 goto err_ctcam_region_init;
345 err_ctcam_region_init:
346 mlxsw_sp_acl_erp_region_fini(aregion);
348 aregion->ops->fini(aregion);
350 rhashtable_destroy(&aregion->entries_ht);
354 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
356 mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
357 mlxsw_sp_acl_erp_region_fini(aregion);
358 aregion->ops->fini(aregion);
359 rhashtable_destroy(&aregion->entries_ht);
362 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
363 struct mlxsw_sp_acl_atcam_chunk *achunk,
364 unsigned int priority)
366 mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
370 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
372 mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
376 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
377 struct mlxsw_sp_acl_atcam_region *aregion,
378 struct mlxsw_sp_acl_atcam_entry *aentry,
379 struct mlxsw_sp_acl_rule_info *rulei)
381 struct mlxsw_sp_acl_tcam_region *region = aregion->region;
382 u8 erp_id = mlxsw_sp_acl_erp_id(aentry->erp);
383 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
384 char ptce3_pl[MLXSW_REG_PTCE3_LEN];
385 u32 kvdl_index, priority;
388 err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
392 lkey_id = aregion->ops->lkey_id_get(aregion, rulei, erp_id);
394 return PTR_ERR(lkey_id);
395 aentry->lkey_id = lkey_id;
397 kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
398 mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
399 priority, region->tcam_region_info,
400 aentry->ht_key.enc_key, erp_id,
401 refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
403 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
405 goto err_ptce3_write;
410 aregion->ops->lkey_id_put(aregion, lkey_id);
415 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
416 struct mlxsw_sp_acl_atcam_region *aregion,
417 struct mlxsw_sp_acl_atcam_entry *aentry)
419 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
420 struct mlxsw_sp_acl_tcam_region *region = aregion->region;
421 u8 erp_id = mlxsw_sp_acl_erp_id(aentry->erp);
422 char ptce3_pl[MLXSW_REG_PTCE3_LEN];
424 mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
425 region->tcam_region_info, aentry->ht_key.enc_key,
426 erp_id, refcount_read(&lkey_id->refcnt) != 1,
428 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
429 aregion->ops->lkey_id_put(aregion, lkey_id);
433 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
434 struct mlxsw_sp_acl_atcam_region *aregion,
435 struct mlxsw_sp_acl_atcam_entry *aentry,
436 struct mlxsw_sp_acl_rule_info *rulei)
438 struct mlxsw_sp_acl_tcam_region *region = aregion->region;
439 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
440 struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
441 struct mlxsw_sp_acl_erp *erp;
442 unsigned int blocks_count;
445 blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
446 mlxsw_afk_encode(afk, region->key_info, &rulei->values,
447 aentry->ht_key.enc_key, mask, 0, blocks_count - 1);
449 erp = mlxsw_sp_acl_erp_get(aregion, mask, false);
453 aentry->ht_key.erp_id = mlxsw_sp_acl_erp_id(erp);
455 /* We can't insert identical rules into the A-TCAM, so fail and
456 * let the rule spill into C-TCAM
458 err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
460 mlxsw_sp_acl_atcam_entries_ht_params);
462 goto err_rhashtable_insert;
464 err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
467 goto err_rule_insert;
472 rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
473 mlxsw_sp_acl_atcam_entries_ht_params);
474 err_rhashtable_insert:
475 mlxsw_sp_acl_erp_put(aregion, erp);
480 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
481 struct mlxsw_sp_acl_atcam_region *aregion,
482 struct mlxsw_sp_acl_atcam_entry *aentry)
484 mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
485 rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
486 mlxsw_sp_acl_atcam_entries_ht_params);
487 mlxsw_sp_acl_erp_put(aregion, aentry->erp);
490 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
491 struct mlxsw_sp_acl_atcam_region *aregion,
492 struct mlxsw_sp_acl_atcam_chunk *achunk,
493 struct mlxsw_sp_acl_atcam_entry *aentry,
494 struct mlxsw_sp_acl_rule_info *rulei)
498 err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
502 /* It is possible we failed to add the rule to the A-TCAM due to
503 * exceeded number of masks. Try to spill into C-TCAM.
505 err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
506 &achunk->cchunk, &aentry->centry,
514 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
515 struct mlxsw_sp_acl_atcam_region *aregion,
516 struct mlxsw_sp_acl_atcam_chunk *achunk,
517 struct mlxsw_sp_acl_atcam_entry *aentry)
519 if (mlxsw_sp_acl_atcam_is_centry(aentry))
520 mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
521 &achunk->cchunk, &aentry->centry);
523 __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
526 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
527 struct mlxsw_sp_acl_atcam *atcam)
529 return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
532 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
533 struct mlxsw_sp_acl_atcam *atcam)
535 mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);