drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/slab.h>
   6 #include <linux/list.h>
   7 #include <linux/errno.h>
   8
   9 #include "item.h"
  10 #include "core_acl_flex_keys.h"
  11
  12 /* For the purpose of the driver, define an internal storage scratchpad
  13  * that will be used to store key/mask values. For each defined element type
  14  * define an internal storage geometry.
  15  *
  16  * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased
  17  * accordingly.
  18  */
  19 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
  20         MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16),
  21         MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2),
  22         MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4),
  23         MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2),
  24         MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4),
  25         MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16),
  26         MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8),
  27         MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
  28         MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
  29         MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9),
  30         MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16),
  31         MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16),
  32         MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
  33         MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
  34         MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
  35         MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 3),
  36         MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 20, 8),
  37         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4),
  38         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4),
  39         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4),
  40         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4),
  41         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4),
  42         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4),
  43         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4),
  44         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4),
  45         MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1),
  46 };
  47
  48 struct mlxsw_afk {
  49         struct list_head key_info_list;
  50         unsigned int max_blocks;
  51         const struct mlxsw_afk_ops *ops;
  52         const struct mlxsw_afk_block *blocks;
  53         unsigned int blocks_count;
  54 };
  55
  56 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk)
  57 {
  58         int i;
  59         int j;
  60
  61         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
  62                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
  63
  64                 for (j = 0; j < block->instances_count; j++) {
  65                         const struct mlxsw_afk_element_info *elinfo;
  66                         struct mlxsw_afk_element_inst *elinst;
  67
  68                         elinst = &block->instances[j];
  69                         elinfo = &mlxsw_afk_element_infos[elinst->element];
  70                         if (elinst->type != elinfo->type ||
  71                             (!elinst->avoid_size_check &&
  72                              elinst->item.size.bits !=
  73                              elinfo->item.size.bits))
  74                                 return false;
  75                 }
  76         }
  77         return true;
  78 }
  79
  80 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks,
  81                                    const struct mlxsw_afk_ops *ops)
  82 {
  83         struct mlxsw_afk *mlxsw_afk;
  84
  85         mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL);
  86         if (!mlxsw_afk)
  87                 return NULL;
  88         INIT_LIST_HEAD(&mlxsw_afk->key_info_list);
  89         mlxsw_afk->max_blocks = max_blocks;
  90         mlxsw_afk->ops = ops;
  91         mlxsw_afk->blocks = ops->blocks;
  92         mlxsw_afk->blocks_count = ops->blocks_count;
  93         WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk));
  94         return mlxsw_afk;
  95 }
  96 EXPORT_SYMBOL(mlxsw_afk_create);
  97
  98 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk)
  99 {
 100         WARN_ON(!list_empty(&mlxsw_afk->key_info_list));
 101         kfree(mlxsw_afk);
 102 }
 103 EXPORT_SYMBOL(mlxsw_afk_destroy);
 104
 105 struct mlxsw_afk_key_info {
 106         struct list_head list;
 107         unsigned int ref_count;
 108         unsigned int blocks_count;
 109         int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value
 110                                                       * is index inside "blocks"
 111                                                       */
 112         struct mlxsw_afk_element_usage elusage;
 113         const struct mlxsw_afk_block *blocks[];
 114 };
 115
 116 static bool
 117 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info,
 118                                struct mlxsw_afk_element_usage *elusage)
 119 {
 120         return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0;
 121 }
 122
 123 static struct mlxsw_afk_key_info *
 124 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk,
 125                         struct mlxsw_afk_element_usage *elusage)
 126 {
 127         struct mlxsw_afk_key_info *key_info;
 128
 129         list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) {
 130                 if (mlxsw_afk_key_info_elements_eq(key_info, elusage))
 131                         return key_info;
 132         }
 133         return NULL;
 134 }
 135
 136 struct mlxsw_afk_picker {
 137         DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX);
 138         unsigned int total;
 139 };
 140
 141 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk,
 142                                         struct mlxsw_afk_picker *picker,
 143                                         enum mlxsw_afk_element element)
 144 {
 145         int i;
 146         int j;
 147
 148         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 149                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
 150
 151                 for (j = 0; j < block->instances_count; j++) {
 152                         struct mlxsw_afk_element_inst *elinst;
 153
 154                         elinst = &block->instances[j];
 155                         if (elinst->element == element) {
 156                                 __set_bit(element, picker[i].element);
 157                                 picker[i].total++;
 158                         }
 159                 }
 160         }
 161 }
 162
 163 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk,
 164                                            struct mlxsw_afk_picker *picker,
 165                                            int block_index)
 166 {
 167         DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX);
 168         int i;
 169         int j;
 170
 171         memcpy(&hits_element, &picker[block_index].element,
 172                sizeof(hits_element));
 173
 174         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 175                 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) {
 176                         if (__test_and_clear_bit(j, picker[i].element))
 177                                 picker[i].total--;
 178                 }
 179         }
 180 }
 181
 182 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk,
 183                                           struct mlxsw_afk_picker *picker)
 184 {
 185         int most_index = -EINVAL; /* Should never happen to return this */
 186         int most_hits = 0;
 187         int i;
 188
 189         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 190                 if (picker[i].total > most_hits) {
 191                         most_hits = picker[i].total;
 192                         most_index = i;
 193                 }
 194         }
 195         return most_index;
 196 }
 197
 198 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk,
 199                                          struct mlxsw_afk_picker *picker,
 200                                          int block_index,
 201                                          struct mlxsw_afk_key_info *key_info)
 202 {
 203         enum mlxsw_afk_element element;
 204
 205         if (key_info->blocks_count == mlxsw_afk->max_blocks)
 206                 return -EINVAL;
 207
 208         for_each_set_bit(element, picker[block_index].element,
 209                          MLXSW_AFK_ELEMENT_MAX) {
 210                 key_info->element_to_block[element] = key_info->blocks_count;
 211                 mlxsw_afk_element_usage_add(&key_info->elusage, element);
 212         }
 213
 214         key_info->blocks[key_info->blocks_count] =
 215                                         &mlxsw_afk->blocks[block_index];
 216         key_info->blocks_count++;
 217         return 0;
 218 }
 219
 220 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk,
 221                             struct mlxsw_afk_key_info *key_info,
 222                             struct mlxsw_afk_element_usage *elusage)
 223 {
 224         struct mlxsw_afk_picker *picker;
 225         enum mlxsw_afk_element element;
 226         int err;
 227
 228         picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL);
 229         if (!picker)
 230                 return -ENOMEM;
 231
 232         /* Since the same elements could be present in multiple blocks,
 233          * we must find out optimal block list in order to make the
 234          * block count as low as possible.
 235          *
 236          * First, we count hits. We go over all available blocks and count
 237          * how many of requested elements are covered by each.
 238          *
 239          * Then in loop, we find block with most hits and add it to
 240          * output key_info. Then we have to subtract this block hits so
 241          * the next iteration will find most suitable block for
 242          * the rest of requested elements.
 243          */
 244
 245         mlxsw_afk_element_usage_for_each(element, elusage)
 246                 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element);
 247
 248         do {
 249                 int block_index;
 250
 251                 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker);
 252                 if (block_index < 0) {
 253                         err = block_index;
 254                         goto out;
 255                 }
 256                 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker,
 257                                                     block_index, key_info);
 258                 if (err)
 259                         goto out;
 260                 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index);
 261         } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage));
 262
 263         err = 0;
 264 out:
 265         kfree(picker);
 266         return err;
 267 }
 268
 269 static struct mlxsw_afk_key_info *
 270 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk,
 271                           struct mlxsw_afk_element_usage *elusage)
 272 {
 273         struct mlxsw_afk_key_info *key_info;
 274         int err;
 275
 276         key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks),
 277                            GFP_KERNEL);
 278         if (!key_info)
 279                 return ERR_PTR(-ENOMEM);
 280         err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage);
 281         if (err)
 282                 goto err_picker;
 283         list_add(&key_info->list, &mlxsw_afk->key_info_list);
 284         key_info->ref_count = 1;
 285         return key_info;
 286
 287 err_picker:
 288         kfree(key_info);
 289         return ERR_PTR(err);
 290 }
 291
 292 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info)
 293 {
 294         list_del(&key_info->list);
 295         kfree(key_info);
 296 }
 297
 298 struct mlxsw_afk_key_info *
 299 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk,
 300                        struct mlxsw_afk_element_usage *elusage)
 301 {
 302         struct mlxsw_afk_key_info *key_info;
 303
 304         key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage);
 305         if (key_info) {
 306                 key_info->ref_count++;
 307                 return key_info;
 308         }
 309         return mlxsw_afk_key_info_create(mlxsw_afk, elusage);
 310 }
 311 EXPORT_SYMBOL(mlxsw_afk_key_info_get);
 312
 313 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info)
 314 {
 315         if (--key_info->ref_count)
 316                 return;
 317         mlxsw_afk_key_info_destroy(key_info);
 318 }
 319 EXPORT_SYMBOL(mlxsw_afk_key_info_put);
 320
 321 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info,
 322                                struct mlxsw_afk_element_usage *elusage)
 323 {
 324         return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage);
 325 }
 326 EXPORT_SYMBOL(mlxsw_afk_key_info_subset);
 327
 328 static const struct mlxsw_afk_element_inst *
 329 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block,
 330                            enum mlxsw_afk_element element)
 331 {
 332         int i;
 333
 334         for (i = 0; i < block->instances_count; i++) {
 335                 struct mlxsw_afk_element_inst *elinst;
 336
 337                 elinst = &block->instances[i];
 338                 if (elinst->element == element)
 339                         return elinst;
 340         }
 341         return NULL;
 342 }
 343
 344 static const struct mlxsw_afk_element_inst *
 345 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info,
 346                               enum mlxsw_afk_element element,
 347                               int *p_block_index)
 348 {
 349         const struct mlxsw_afk_element_inst *elinst;
 350         const struct mlxsw_afk_block *block;
 351         int block_index;
 352
 353         if (WARN_ON(!test_bit(element, key_info->elusage.usage)))
 354                 return NULL;
 355         block_index = key_info->element_to_block[element];
 356         block = key_info->blocks[block_index];
 357
 358         elinst = mlxsw_afk_block_elinst_get(block, element);
 359         if (WARN_ON(!elinst))
 360                 return NULL;
 361
 362         *p_block_index = block_index;
 363         return elinst;
 364 }
 365
 366 u16
 367 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info,
 368                                       int block_index)
 369 {
 370         return key_info->blocks[block_index]->encoding;
 371 }
 372 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get);
 373
 374 unsigned int
 375 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info)
 376 {
 377         return key_info->blocks_count;
 378 }
 379 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get);
 380
 381 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values,
 382                               enum mlxsw_afk_element element,
 383                               u32 key_value, u32 mask_value)
 384 {
 385         const struct mlxsw_afk_element_info *elinfo =
 386                                 &mlxsw_afk_element_infos[element];
 387         const struct mlxsw_item *storage_item = &elinfo->item;
 388
 389         if (!mask_value)
 390                 return;
 391         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32))
 392                 return;
 393         __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value);
 394         __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value);
 395         mlxsw_afk_element_usage_add(&values->elusage, element);
 396 }
 397 EXPORT_SYMBOL(mlxsw_afk_values_add_u32);
 398
 399 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values,
 400                               enum mlxsw_afk_element element,
 401                               const char *key_value, const char *mask_value,
 402                               unsigned int len)
 403 {
 404         const struct mlxsw_afk_element_info *elinfo =
 405                                 &mlxsw_afk_element_infos[element];
 406         const struct mlxsw_item *storage_item = &elinfo->item;
 407
 408         if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */
 409                 return;
 410         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) ||
 411             WARN_ON(elinfo->item.size.bytes != len))
 412                 return;
 413         __mlxsw_item_memcpy_to(values->storage.key, key_value,
 414                                storage_item, 0);
 415         __mlxsw_item_memcpy_to(values->storage.mask, mask_value,
 416                                storage_item, 0);
 417         mlxsw_afk_element_usage_add(&values->elusage, element);
 418 }
 419 EXPORT_SYMBOL(mlxsw_afk_values_add_buf);
 420
 421 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item,
 422                                     const struct mlxsw_item *output_item,
 423                                     char *storage, char *output, int diff)
 424 {
 425         u32 value;
 426
 427         value = __mlxsw_item_get32(storage, storage_item, 0);
 428         __mlxsw_item_set32(output, output_item, 0, value + diff);
 429 }
 430
 431 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item,
 432                                     const struct mlxsw_item *output_item,
 433                                     char *storage, char *output)
 434 {
 435         char *storage_data = __mlxsw_item_data(storage, storage_item, 0);
 436         char *output_data = __mlxsw_item_data(output, output_item, 0);
 437         size_t len = output_item->size.bytes;
 438
 439         memcpy(output_data, storage_data, len);
 440 }
 441
 442 static void
 443 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst,
 444                         char *output, char *storage, int u32_diff)
 445 {
 446         const struct mlxsw_item *output_item = &elinst->item;
 447         const struct mlxsw_afk_element_info *elinfo;
 448         const struct mlxsw_item *storage_item;
 449
 450         elinfo = &mlxsw_afk_element_infos[elinst->element];
 451         storage_item = &elinfo->item;
 452         if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32)
 453                 mlxsw_sp_afk_encode_u32(storage_item, output_item,
 454                                         storage, output, u32_diff);
 455         else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF)
 456                 mlxsw_sp_afk_encode_buf(storage_item, output_item,
 457                                         storage, output);
 458 }
 459
 460 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16
 461
 462 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk,
 463                       struct mlxsw_afk_key_info *key_info,
 464                       struct mlxsw_afk_element_values *values,
 465                       char *key, char *mask)
 466 {
 467         unsigned int blocks_count =
 468                         mlxsw_afk_key_info_blocks_count_get(key_info);
 469         char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 470         char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 471         const struct mlxsw_afk_element_inst *elinst;
 472         enum mlxsw_afk_element element;
 473         int block_index, i;
 474
 475         for (i = 0; i < blocks_count; i++) {
 476                 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 477                 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 478
 479                 mlxsw_afk_element_usage_for_each(element, &values->elusage) {
 480                         elinst = mlxsw_afk_key_info_elinst_get(key_info,
 481                                                                element,
 482                                                                &block_index);
 483                         if (!elinst || block_index != i)
 484                                 continue;
 485
 486                         mlxsw_sp_afk_encode_one(elinst, block_key,
 487                                                 values->storage.key,
 488                                                 elinst->u32_key_diff);
 489                         mlxsw_sp_afk_encode_one(elinst, block_mask,
 490                                                 values->storage.mask, 0);
 491                 }
 492
 493                 mlxsw_afk->ops->encode_block(key, i, block_key);
 494                 mlxsw_afk->ops->encode_block(mask, i, block_mask);
 495         }
 496 }
 497 EXPORT_SYMBOL(mlxsw_afk_encode);
 498
 499 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key,
 500                      int block_start, int block_end)
 501 {
 502         int i;
 503
 504         for (i = block_start; i <= block_end; i++)
 505                 mlxsw_afk->ops->clear_block(key, i);
 506 }
 507 EXPORT_SYMBOL(mlxsw_afk_clear);