lib/objagg.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/module.h>
   5 #include <linux/slab.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/idr.h>
   8 #include <linux/list.h>
   9 #include <linux/sort.h>
  10 #include <linux/objagg.h>
  11
  12 #define CREATE_TRACE_POINTS
  13 #include <trace/events/objagg.h>
  14
  15 struct objagg_hints {
  16         struct rhashtable node_ht;
  17         struct rhashtable_params ht_params;
  18         struct list_head node_list;
  19         unsigned int node_count;
  20         unsigned int root_count;
  21         unsigned int refcount;
  22         const struct objagg_ops *ops;
  23 };
  24
  25 struct objagg_hints_node {
  26         struct rhash_head ht_node; /* member of objagg_hints->node_ht */
  27         struct list_head list; /* member of objagg_hints->node_list */
  28         struct objagg_hints_node *parent;
  29         unsigned int root_id;
  30         struct objagg_obj_stats_info stats_info;
  31         unsigned long obj[0];
  32 };
  33
  34 static struct objagg_hints_node *
  35 objagg_hints_lookup(struct objagg_hints *objagg_hints, void *obj)
  36 {
  37         if (!objagg_hints)
  38                 return NULL;
  39         return rhashtable_lookup_fast(&objagg_hints->node_ht, obj,
  40                                       objagg_hints->ht_params);
  41 }
  42
  43 struct objagg {
  44         const struct objagg_ops *ops;
  45         void *priv;
  46         struct rhashtable obj_ht;
  47         struct rhashtable_params ht_params;
  48         struct list_head obj_list;
  49         unsigned int obj_count;
  50         struct ida root_ida;
  51         struct objagg_hints *hints;
  52 };
  53
  54 struct objagg_obj {
  55         struct rhash_head ht_node; /* member of objagg->obj_ht */
  56         struct list_head list; /* member of objagg->obj_list */
  57         struct objagg_obj *parent; /* if the object is nested, this
  58                                     * holds pointer to parent, otherwise NULL
  59                                     */
  60         union {
  61                 void *delta_priv; /* user delta private */
  62                 void *root_priv; /* user root private */
  63         };
  64         unsigned int root_id;
  65         unsigned int refcount; /* counts number of users of this object
  66                                 * including nested objects
  67                                 */
  68         struct objagg_obj_stats stats;
  69         unsigned long obj[0];
  70 };
  71
  72 static unsigned int objagg_obj_ref_inc(struct objagg_obj *objagg_obj)
  73 {
  74         return ++objagg_obj->refcount;
  75 }
  76
  77 static unsigned int objagg_obj_ref_dec(struct objagg_obj *objagg_obj)
  78 {
  79         return --objagg_obj->refcount;
  80 }
  81
  82 static void objagg_obj_stats_inc(struct objagg_obj *objagg_obj)
  83 {
  84         objagg_obj->stats.user_count++;
  85         objagg_obj->stats.delta_user_count++;
  86         if (objagg_obj->parent)
  87                 objagg_obj->parent->stats.delta_user_count++;
  88 }
  89
  90 static void objagg_obj_stats_dec(struct objagg_obj *objagg_obj)
  91 {
  92         objagg_obj->stats.user_count--;
  93         objagg_obj->stats.delta_user_count--;
  94         if (objagg_obj->parent)
  95                 objagg_obj->parent->stats.delta_user_count--;
  96 }
  97
  98 static bool objagg_obj_is_root(const struct objagg_obj *objagg_obj)
  99 {
 100         /* Nesting is not supported, so we can use ->parent
 101          * to figure out if the object is root.
 102          */
 103         return !objagg_obj->parent;
 104 }
 105
 106 /**
 107  * objagg_obj_root_priv - obtains root private for an object
 108  * @objagg_obj: objagg object instance
 109  *
 110  * Note: all locking must be provided by the caller.
 111  *
 112  * Either the object is root itself when the private is returned
 113  * directly, or the parent is root and its private is returned
 114  * instead.
 115  *
 116  * Returns a user private root pointer.
 117  */
 118 const void *objagg_obj_root_priv(const struct objagg_obj *objagg_obj)
 119 {
 120         if (objagg_obj_is_root(objagg_obj))
 121                 return objagg_obj->root_priv;
 122         WARN_ON(!objagg_obj_is_root(objagg_obj->parent));
 123         return objagg_obj->parent->root_priv;
 124 }
 125 EXPORT_SYMBOL(objagg_obj_root_priv);
 126
 127 /**
 128  * objagg_obj_delta_priv - obtains delta private for an object
 129  * @objagg_obj: objagg object instance
 130  *
 131  * Note: all locking must be provided by the caller.
 132  *
 133  * Returns user private delta pointer or NULL in case the passed
 134  * object is root.
 135  */
 136 const void *objagg_obj_delta_priv(const struct objagg_obj *objagg_obj)
 137 {
 138         if (objagg_obj_is_root(objagg_obj))
 139                 return NULL;
 140         return objagg_obj->delta_priv;
 141 }
 142 EXPORT_SYMBOL(objagg_obj_delta_priv);
 143
 144 /**
 145  * objagg_obj_raw - obtains object user private pointer
 146  * @objagg_obj: objagg object instance
 147  *
 148  * Note: all locking must be provided by the caller.
 149  *
 150  * Returns user private pointer as was passed to objagg_obj_get() by "obj" arg.
 151  */
 152 const void *objagg_obj_raw(const struct objagg_obj *objagg_obj)
 153 {
 154         return objagg_obj->obj;
 155 }
 156 EXPORT_SYMBOL(objagg_obj_raw);
 157
 158 static struct objagg_obj *objagg_obj_lookup(struct objagg *objagg, void *obj)
 159 {
 160         return rhashtable_lookup_fast(&objagg->obj_ht, obj, objagg->ht_params);
 161 }
 162
 163 static int objagg_obj_parent_assign(struct objagg *objagg,
 164                                     struct objagg_obj *objagg_obj,
 165                                     struct objagg_obj *parent,
 166                                     bool take_parent_ref)
 167 {
 168         void *delta_priv;
 169
 170         delta_priv = objagg->ops->delta_create(objagg->priv, parent->obj,
 171                                                objagg_obj->obj);
 172         if (IS_ERR(delta_priv))
 173                 return PTR_ERR(delta_priv);
 174
 175         /* User returned a delta private, that means that
 176          * our object can be aggregated into the parent.
 177          */
 178         objagg_obj->parent = parent;
 179         objagg_obj->delta_priv = delta_priv;
 180         if (take_parent_ref)
 181                 objagg_obj_ref_inc(objagg_obj->parent);
 182         trace_objagg_obj_parent_assign(objagg, objagg_obj,
 183                                        parent,
 184                                        parent->refcount);
 185         return 0;
 186 }
 187
 188 static int objagg_obj_parent_lookup_assign(struct objagg *objagg,
 189                                            struct objagg_obj *objagg_obj)
 190 {
 191         struct objagg_obj *objagg_obj_cur;
 192         int err;
 193
 194         list_for_each_entry(objagg_obj_cur, &objagg->obj_list, list) {
 195                 /* Nesting is not supported. In case the object
 196                  * is not root, it cannot be assigned as parent.
 197                  */
 198                 if (!objagg_obj_is_root(objagg_obj_cur))
 199                         continue;
 200                 err = objagg_obj_parent_assign(objagg, objagg_obj,
 201                                                objagg_obj_cur, true);
 202                 if (!err)
 203                         return 0;
 204         }
 205         return -ENOENT;
 206 }
 207
 208 static void __objagg_obj_put(struct objagg *objagg,
 209                              struct objagg_obj *objagg_obj);
 210
 211 static void objagg_obj_parent_unassign(struct objagg *objagg,
 212                                        struct objagg_obj *objagg_obj)
 213 {
 214         trace_objagg_obj_parent_unassign(objagg, objagg_obj,
 215                                          objagg_obj->parent,
 216                                          objagg_obj->parent->refcount);
 217         objagg->ops->delta_destroy(objagg->priv, objagg_obj->delta_priv);
 218         __objagg_obj_put(objagg, objagg_obj->parent);
 219 }
 220
 221 static int objagg_obj_root_id_alloc(struct objagg *objagg,
 222                                     struct objagg_obj *objagg_obj,
 223                                     struct objagg_hints_node *hnode)
 224 {
 225         unsigned int min, max;
 226         int root_id;
 227
 228         /* In case there are no hints available, the root id is invalid. */
 229         if (!objagg->hints) {
 230                 objagg_obj->root_id = OBJAGG_OBJ_ROOT_ID_INVALID;
 231                 return 0;
 232         }
 233
 234         if (hnode) {
 235                 min = hnode->root_id;
 236                 max = hnode->root_id;
 237         } else {
 238                 /* For objects with no hint, start after the last
 239                  * hinted root_id.
 240                  */
 241                 min = objagg->hints->root_count;
 242                 max = ~0;
 243         }
 244
 245         root_id = ida_alloc_range(&objagg->root_ida, min, max, GFP_KERNEL);
 246
 247         if (root_id < 0)
 248                 return root_id;
 249         objagg_obj->root_id = root_id;
 250         return 0;
 251 }
 252
 253 static void objagg_obj_root_id_free(struct objagg *objagg,
 254                                     struct objagg_obj *objagg_obj)
 255 {
 256         if (!objagg->hints)
 257                 return;
 258         ida_free(&objagg->root_ida, objagg_obj->root_id);
 259 }
 260
 261 static int objagg_obj_root_create(struct objagg *objagg,
 262                                   struct objagg_obj *objagg_obj,
 263                                   struct objagg_hints_node *hnode)
 264 {
 265         int err;
 266
 267         err = objagg_obj_root_id_alloc(objagg, objagg_obj, hnode);
 268         if (err)
 269                 return err;
 270         objagg_obj->root_priv = objagg->ops->root_create(objagg->priv,
 271                                                          objagg_obj->obj,
 272                                                          objagg_obj->root_id);
 273         if (IS_ERR(objagg_obj->root_priv)) {
 274                 err = PTR_ERR(objagg_obj->root_priv);
 275                 goto err_root_create;
 276         }
 277         trace_objagg_obj_root_create(objagg, objagg_obj);
 278         return 0;
 279
 280 err_root_create:
 281         objagg_obj_root_id_free(objagg, objagg_obj);
 282         return err;
 283 }
 284
 285 static void objagg_obj_root_destroy(struct objagg *objagg,
 286                                     struct objagg_obj *objagg_obj)
 287 {
 288         trace_objagg_obj_root_destroy(objagg, objagg_obj);
 289         objagg->ops->root_destroy(objagg->priv, objagg_obj->root_priv);
 290         objagg_obj_root_id_free(objagg, objagg_obj);
 291 }
 292
 293 static struct objagg_obj *__objagg_obj_get(struct objagg *objagg, void *obj);
 294
 295 static int objagg_obj_init_with_hints(struct objagg *objagg,
 296                                       struct objagg_obj *objagg_obj,
 297                                       bool *hint_found)
 298 {
 299         struct objagg_hints_node *hnode;
 300         struct objagg_obj *parent;
 301         int err;
 302
 303         hnode = objagg_hints_lookup(objagg->hints, objagg_obj->obj);
 304         if (!hnode) {
 305                 *hint_found = false;
 306                 return 0;
 307         }
 308         *hint_found = true;
 309
 310         if (!hnode->parent)
 311                 return objagg_obj_root_create(objagg, objagg_obj, hnode);
 312
 313         parent = __objagg_obj_get(objagg, hnode->parent->obj);
 314         if (IS_ERR(parent))
 315                 return PTR_ERR(parent);
 316
 317         err = objagg_obj_parent_assign(objagg, objagg_obj, parent, false);
 318         if (err) {
 319                 *hint_found = false;
 320                 err = 0;
 321                 goto err_parent_assign;
 322         }
 323
 324         return 0;
 325
 326 err_parent_assign:
 327         objagg_obj_put(objagg, parent);
 328         return err;
 329 }
 330
 331 static int objagg_obj_init(struct objagg *objagg,
 332                            struct objagg_obj *objagg_obj)
 333 {
 334         bool hint_found;
 335         int err;
 336
 337         /* First, try to use hints if they are available and
 338          * if they provide result.
 339          */
 340         err = objagg_obj_init_with_hints(objagg, objagg_obj, &hint_found);
 341         if (err)
 342                 return err;
 343
 344         if (hint_found)
 345                 return 0;
 346
 347         /* Try to find if the object can be aggregated under an existing one. */
 348         err = objagg_obj_parent_lookup_assign(objagg, objagg_obj);
 349         if (!err)
 350                 return 0;
 351         /* If aggregation is not possible, make the object a root. */
 352         return objagg_obj_root_create(objagg, objagg_obj, NULL);
 353 }
 354
 355 static void objagg_obj_fini(struct objagg *objagg,
 356                             struct objagg_obj *objagg_obj)
 357 {
 358         if (!objagg_obj_is_root(objagg_obj))
 359                 objagg_obj_parent_unassign(objagg, objagg_obj);
 360         else
 361                 objagg_obj_root_destroy(objagg, objagg_obj);
 362 }
 363
 364 static struct objagg_obj *objagg_obj_create(struct objagg *objagg, void *obj)
 365 {
 366         struct objagg_obj *objagg_obj;
 367         int err;
 368
 369         objagg_obj = kzalloc(sizeof(*objagg_obj) + objagg->ops->obj_size,
 370                              GFP_KERNEL);
 371         if (!objagg_obj)
 372                 return ERR_PTR(-ENOMEM);
 373         objagg_obj_ref_inc(objagg_obj);
 374         memcpy(objagg_obj->obj, obj, objagg->ops->obj_size);
 375
 376         err = objagg_obj_init(objagg, objagg_obj);
 377         if (err)
 378                 goto err_obj_init;
 379
 380         err = rhashtable_insert_fast(&objagg->obj_ht, &objagg_obj->ht_node,
 381                                      objagg->ht_params);
 382         if (err)
 383                 goto err_ht_insert;
 384         list_add(&objagg_obj->list, &objagg->obj_list);
 385         objagg->obj_count++;
 386         trace_objagg_obj_create(objagg, objagg_obj);
 387
 388         return objagg_obj;
 389
 390 err_ht_insert:
 391         objagg_obj_fini(objagg, objagg_obj);
 392 err_obj_init:
 393         kfree(objagg_obj);
 394         return ERR_PTR(err);
 395 }
 396
 397 static struct objagg_obj *__objagg_obj_get(struct objagg *objagg, void *obj)
 398 {
 399         struct objagg_obj *objagg_obj;
 400
 401         /* First, try to find the object exactly as user passed it,
 402          * perhaps it is already in use.
 403          */
 404         objagg_obj = objagg_obj_lookup(objagg, obj);
 405         if (objagg_obj) {
 406                 objagg_obj_ref_inc(objagg_obj);
 407                 return objagg_obj;
 408         }
 409
 410         return objagg_obj_create(objagg, obj);
 411 }
 412
 413 /**
 414  * objagg_obj_get - gets an object within objagg instance
 415  * @objagg:     objagg instance
 416  * @obj:        user-specific private object pointer
 417  *
 418  * Note: all locking must be provided by the caller.
 419  *
 420  * Size of the "obj" memory is specified in "objagg->ops".
 421  *
 422  * There are 3 main options this function wraps:
 423  * 1) The object according to "obj" already exist. In that case
 424  *    the reference counter is incrementes and the object is returned.
 425  * 2) The object does not exist, but it can be aggregated within
 426  *    another object. In that case, user ops->delta_create() is called
 427  *    to obtain delta data and a new object is created with returned
 428  *    user-delta private pointer.
 429  * 3) The object does not exist and cannot be aggregated into
 430  *    any of the existing objects. In that case, user ops->root_create()
 431  *    is called to create the root and a new object is created with
 432  *    returned user-root private pointer.
 433  *
 434  * Returns a pointer to objagg object instance in case of success,
 435  * otherwise it returns pointer error using ERR_PTR macro.
 436  */
 437 struct objagg_obj *objagg_obj_get(struct objagg *objagg, void *obj)
 438 {
 439         struct objagg_obj *objagg_obj;
 440
 441         objagg_obj = __objagg_obj_get(objagg, obj);
 442         if (IS_ERR(objagg_obj))
 443                 return objagg_obj;
 444         objagg_obj_stats_inc(objagg_obj);
 445         trace_objagg_obj_get(objagg, objagg_obj, objagg_obj->refcount);
 446         return objagg_obj;
 447 }
 448 EXPORT_SYMBOL(objagg_obj_get);
 449
 450 static void objagg_obj_destroy(struct objagg *objagg,
 451                                struct objagg_obj *objagg_obj)
 452 {
 453         trace_objagg_obj_destroy(objagg, objagg_obj);
 454         --objagg->obj_count;
 455         list_del(&objagg_obj->list);
 456         rhashtable_remove_fast(&objagg->obj_ht, &objagg_obj->ht_node,
 457                                objagg->ht_params);
 458         objagg_obj_fini(objagg, objagg_obj);
 459         kfree(objagg_obj);
 460 }
 461
 462 static void __objagg_obj_put(struct objagg *objagg,
 463                              struct objagg_obj *objagg_obj)
 464 {
 465         if (!objagg_obj_ref_dec(objagg_obj))
 466                 objagg_obj_destroy(objagg, objagg_obj);
 467 }
 468
 469 /**
 470  * objagg_obj_put - puts an object within objagg instance
 471  * @objagg:     objagg instance
 472  * @objagg_obj: objagg object instance
 473  *
 474  * Note: all locking must be provided by the caller.
 475  *
 476  * Symmetric to objagg_obj_get().
 477  */
 478 void objagg_obj_put(struct objagg *objagg, struct objagg_obj *objagg_obj)
 479 {
 480         trace_objagg_obj_put(objagg, objagg_obj, objagg_obj->refcount);
 481         objagg_obj_stats_dec(objagg_obj);
 482         __objagg_obj_put(objagg, objagg_obj);
 483 }
 484 EXPORT_SYMBOL(objagg_obj_put);
 485
 486 /**
 487  * objagg_create - creates a new objagg instance
 488  * @ops:                user-specific callbacks
 489  * @objagg_hints:       hints, can be NULL
 490  * @priv:               pointer to a private data passed to the ops
 491  *
 492  * Note: all locking must be provided by the caller.
 493  *
 494  * The purpose of the library is to provide an infrastructure to
 495  * aggregate user-specified objects. Library does not care about the type
 496  * of the object. User fills-up ops which take care of the specific
 497  * user object manipulation.
 498  *
 499  * As a very stupid example, consider integer numbers. For example
 500  * number 8 as a root object. That can aggregate number 9 with delta 1,
 501  * number 10 with delta 2, etc. This example is implemented as
 502  * a part of a testing module in test_objagg.c file.
 503  *
 504  * Each objagg instance contains multiple trees. Each tree node is
 505  * represented by "an object". In the current implementation there can be
 506  * only roots and leafs nodes. Leaf nodes are called deltas.
 507  * But in general, this can be easily extended for intermediate nodes.
 508  * In that extension, a delta would be associated with all non-root
 509  * nodes.
 510  *
 511  * Returns a pointer to newly created objagg instance in case of success,
 512  * otherwise it returns pointer error using ERR_PTR macro.
 513  */
 514 struct objagg *objagg_create(const struct objagg_ops *ops,
 515                              struct objagg_hints *objagg_hints, void *priv)
 516 {
 517         struct objagg *objagg;
 518         int err;
 519
 520         if (WARN_ON(!ops || !ops->root_create || !ops->root_destroy ||
 521                     !ops->delta_check || !ops->delta_create ||
 522                     !ops->delta_destroy))
 523                 return ERR_PTR(-EINVAL);
 524
 525         objagg = kzalloc(sizeof(*objagg), GFP_KERNEL);
 526         if (!objagg)
 527                 return ERR_PTR(-ENOMEM);
 528         objagg->ops = ops;
 529         if (objagg_hints) {
 530                 objagg->hints = objagg_hints;
 531                 objagg_hints->refcount++;
 532         }
 533         objagg->priv = priv;
 534         INIT_LIST_HEAD(&objagg->obj_list);
 535
 536         objagg->ht_params.key_len = ops->obj_size;
 537         objagg->ht_params.key_offset = offsetof(struct objagg_obj, obj);
 538         objagg->ht_params.head_offset = offsetof(struct objagg_obj, ht_node);
 539
 540         err = rhashtable_init(&objagg->obj_ht, &objagg->ht_params);
 541         if (err)
 542                 goto err_rhashtable_init;
 543
 544         ida_init(&objagg->root_ida);
 545
 546         trace_objagg_create(objagg);
 547         return objagg;
 548
 549 err_rhashtable_init:
 550         kfree(objagg);
 551         return ERR_PTR(err);
 552 }
 553 EXPORT_SYMBOL(objagg_create);
 554
 555 /**
 556  * objagg_destroy - destroys a new objagg instance
 557  * @objagg:     objagg instance
 558  *
 559  * Note: all locking must be provided by the caller.
 560  */
 561 void objagg_destroy(struct objagg *objagg)
 562 {
 563         trace_objagg_destroy(objagg);
 564         ida_destroy(&objagg->root_ida);
 565         WARN_ON(!list_empty(&objagg->obj_list));
 566         rhashtable_destroy(&objagg->obj_ht);
 567         if (objagg->hints)
 568                 objagg_hints_put(objagg->hints);
 569         kfree(objagg);
 570 }
 571 EXPORT_SYMBOL(objagg_destroy);
 572
 573 static int objagg_stats_info_sort_cmp_func(const void *a, const void *b)
 574 {
 575         const struct objagg_obj_stats_info *stats_info1 = a;
 576         const struct objagg_obj_stats_info *stats_info2 = b;
 577
 578         if (stats_info1->is_root != stats_info2->is_root)
 579                 return stats_info2->is_root - stats_info1->is_root;
 580         if (stats_info1->stats.delta_user_count !=
 581             stats_info2->stats.delta_user_count)
 582                 return stats_info2->stats.delta_user_count -
 583                        stats_info1->stats.delta_user_count;
 584         return stats_info2->stats.user_count - stats_info1->stats.user_count;
 585 }
 586
 587 /**
 588  * objagg_stats_get - obtains stats of the objagg instance
 589  * @objagg:     objagg instance
 590  *
 591  * Note: all locking must be provided by the caller.
 592  *
 593  * The returned structure contains statistics of all object
 594  * currently in use, ordered by following rules:
 595  * 1) Root objects are always on lower indexes than the rest.
 596  * 2) Objects with higher delta user count are always on lower
 597  *    indexes.
 598  * 3) In case more objects have the same delta user count,
 599  *    the objects are ordered by user count.
 600  *
 601  * Returns a pointer to stats instance in case of success,
 602  * otherwise it returns pointer error using ERR_PTR macro.
 603  */
 604 const struct objagg_stats *objagg_stats_get(struct objagg *objagg)
 605 {
 606         struct objagg_stats *objagg_stats;
 607         struct objagg_obj *objagg_obj;
 608         size_t alloc_size;
 609         int i;
 610
 611         alloc_size = sizeof(*objagg_stats) +
 612                      sizeof(objagg_stats->stats_info[0]) * objagg->obj_count;
 613         objagg_stats = kzalloc(alloc_size, GFP_KERNEL);
 614         if (!objagg_stats)
 615                 return ERR_PTR(-ENOMEM);
 616
 617         i = 0;
 618         list_for_each_entry(objagg_obj, &objagg->obj_list, list) {
 619                 memcpy(&objagg_stats->stats_info[i].stats, &objagg_obj->stats,
 620                        sizeof(objagg_stats->stats_info[0].stats));
 621                 objagg_stats->stats_info[i].objagg_obj = objagg_obj;
 622                 objagg_stats->stats_info[i].is_root =
 623                                         objagg_obj_is_root(objagg_obj);
 624                 if (objagg_stats->stats_info[i].is_root)
 625                         objagg_stats->root_count++;
 626                 i++;
 627         }
 628         objagg_stats->stats_info_count = i;
 629
 630         sort(objagg_stats->stats_info, objagg_stats->stats_info_count,
 631              sizeof(struct objagg_obj_stats_info),
 632              objagg_stats_info_sort_cmp_func, NULL);
 633
 634         return objagg_stats;
 635 }
 636 EXPORT_SYMBOL(objagg_stats_get);
 637
 638 /**
 639  * objagg_stats_put - puts stats of the objagg instance
 640  * @objagg_stats:       objagg instance stats
 641  *
 642  * Note: all locking must be provided by the caller.
 643  */
 644 void objagg_stats_put(const struct objagg_stats *objagg_stats)
 645 {
 646         kfree(objagg_stats);
 647 }
 648 EXPORT_SYMBOL(objagg_stats_put);
 649
 650 static struct objagg_hints_node *
 651 objagg_hints_node_create(struct objagg_hints *objagg_hints,
 652                          struct objagg_obj *objagg_obj, size_t obj_size,
 653                          struct objagg_hints_node *parent_hnode)
 654 {
 655         unsigned int user_count = objagg_obj->stats.user_count;
 656         struct objagg_hints_node *hnode;
 657         int err;
 658
 659         hnode = kzalloc(sizeof(*hnode) + obj_size, GFP_KERNEL);
 660         if (!hnode)
 661                 return ERR_PTR(-ENOMEM);
 662         memcpy(hnode->obj, &objagg_obj->obj, obj_size);
 663         hnode->stats_info.stats.user_count = user_count;
 664         hnode->stats_info.stats.delta_user_count = user_count;
 665         if (parent_hnode) {
 666                 parent_hnode->stats_info.stats.delta_user_count += user_count;
 667         } else {
 668                 hnode->root_id = objagg_hints->root_count++;
 669                 hnode->stats_info.is_root = true;
 670         }
 671         hnode->stats_info.objagg_obj = objagg_obj;
 672
 673         err = rhashtable_insert_fast(&objagg_hints->node_ht, &hnode->ht_node,
 674                                      objagg_hints->ht_params);
 675         if (err)
 676                 goto err_ht_insert;
 677
 678         list_add(&hnode->list, &objagg_hints->node_list);
 679         hnode->parent = parent_hnode;
 680         objagg_hints->node_count++;
 681
 682         return hnode;
 683
 684 err_ht_insert:
 685         kfree(hnode);
 686         return ERR_PTR(err);
 687 }
 688
 689 static void objagg_hints_flush(struct objagg_hints *objagg_hints)
 690 {
 691         struct objagg_hints_node *hnode, *tmp;
 692
 693         list_for_each_entry_safe(hnode, tmp, &objagg_hints->node_list, list) {
 694                 list_del(&hnode->list);
 695                 rhashtable_remove_fast(&objagg_hints->node_ht, &hnode->ht_node,
 696                                        objagg_hints->ht_params);
 697                 kfree(hnode);
 698         }
 699 }
 700
 701 struct objagg_tmp_node {
 702         struct objagg_obj *objagg_obj;
 703         bool crossed_out;
 704 };
 705
 706 struct objagg_tmp_graph {
 707         struct objagg_tmp_node *nodes;
 708         unsigned long nodes_count;
 709         unsigned long *edges;
 710 };
 711
 712 static int objagg_tmp_graph_edge_index(struct objagg_tmp_graph *graph,
 713                                        int parent_index, int index)
 714 {
 715         return index * graph->nodes_count + parent_index;
 716 }
 717
 718 static void objagg_tmp_graph_edge_set(struct objagg_tmp_graph *graph,
 719                                       int parent_index, int index)
 720 {
 721         int edge_index = objagg_tmp_graph_edge_index(graph, index,
 722                                                      parent_index);
 723
 724         __set_bit(edge_index, graph->edges);
 725 }
 726
 727 static bool objagg_tmp_graph_is_edge(struct objagg_tmp_graph *graph,
 728                                      int parent_index, int index)
 729 {
 730         int edge_index = objagg_tmp_graph_edge_index(graph, index,
 731                                                      parent_index);
 732
 733         return test_bit(edge_index, graph->edges);
 734 }
 735
 736 static unsigned int objagg_tmp_graph_node_weight(struct objagg_tmp_graph *graph,
 737                                                  unsigned int index)
 738 {
 739         struct objagg_tmp_node *node = &graph->nodes[index];
 740         unsigned int weight = node->objagg_obj->stats.user_count;
 741         int j;
 742
 743         /* Node weight is sum of node users and all other nodes users
 744          * that this node can represent with delta.
 745          */
 746
 747         for (j = 0; j < graph->nodes_count; j++) {
 748                 if (!objagg_tmp_graph_is_edge(graph, index, j))
 749                         continue;
 750                 node = &graph->nodes[j];
 751                 if (node->crossed_out)
 752                         continue;
 753                 weight += node->objagg_obj->stats.user_count;
 754         }
 755         return weight;
 756 }
 757
 758 static int objagg_tmp_graph_node_max_weight(struct objagg_tmp_graph *graph)
 759 {
 760         struct objagg_tmp_node *node;
 761         unsigned int max_weight = 0;
 762         unsigned int weight;
 763         int max_index = -1;
 764         int i;
 765
 766         for (i = 0; i < graph->nodes_count; i++) {
 767                 node = &graph->nodes[i];
 768                 if (node->crossed_out)
 769                         continue;
 770                 weight = objagg_tmp_graph_node_weight(graph, i);
 771                 if (weight >= max_weight) {
 772                         max_weight = weight;
 773                         max_index = i;
 774                 }
 775         }
 776         return max_index;
 777 }
 778
 779 static struct objagg_tmp_graph *objagg_tmp_graph_create(struct objagg *objagg)
 780 {
 781         unsigned int nodes_count = objagg->obj_count;
 782         struct objagg_tmp_graph *graph;
 783         struct objagg_tmp_node *node;
 784         struct objagg_tmp_node *pnode;
 785         struct objagg_obj *objagg_obj;
 786         size_t alloc_size;
 787         int i, j;
 788
 789         graph = kzalloc(sizeof(*graph), GFP_KERNEL);
 790         if (!graph)
 791                 return NULL;
 792
 793         graph->nodes = kcalloc(nodes_count, sizeof(*graph->nodes), GFP_KERNEL);
 794         if (!graph->nodes)
 795                 goto err_nodes_alloc;
 796         graph->nodes_count = nodes_count;
 797
 798         alloc_size = BITS_TO_LONGS(nodes_count * nodes_count) *
 799                      sizeof(unsigned long);
 800         graph->edges = kzalloc(alloc_size, GFP_KERNEL);
 801         if (!graph->edges)
 802                 goto err_edges_alloc;
 803
 804         i = 0;
 805         list_for_each_entry(objagg_obj, &objagg->obj_list, list) {
 806                 node = &graph->nodes[i++];
 807                 node->objagg_obj = objagg_obj;
 808         }
 809
 810         /* Assemble a temporary graph. Insert edge X->Y in case Y can be
 811          * in delta of X.
 812          */
 813         for (i = 0; i < nodes_count; i++) {
 814                 for (j = 0; j < nodes_count; j++) {
 815                         if (i == j)
 816                                 continue;
 817                         pnode = &graph->nodes[i];
 818                         node = &graph->nodes[j];
 819                         if (objagg->ops->delta_check(objagg->priv,
 820                                                      pnode->objagg_obj->obj,
 821                                                      node->objagg_obj->obj)) {
 822                                 objagg_tmp_graph_edge_set(graph, i, j);
 823
 824                         }
 825                 }
 826         }
 827         return graph;
 828
 829 err_edges_alloc:
 830         kfree(graph->nodes);
 831 err_nodes_alloc:
 832         kfree(graph);
 833         return NULL;
 834 }
 835
 836 static void objagg_tmp_graph_destroy(struct objagg_tmp_graph *graph)
 837 {
 838         kfree(graph->edges);
 839         kfree(graph->nodes);
 840         kfree(graph);
 841 }
 842
 843 static int
 844 objagg_opt_simple_greedy_fillup_hints(struct objagg_hints *objagg_hints,
 845                                       struct objagg *objagg)
 846 {
 847         struct objagg_hints_node *hnode, *parent_hnode;
 848         struct objagg_tmp_graph *graph;
 849         struct objagg_tmp_node *node;
 850         int index;
 851         int j;
 852         int err;
 853
 854         graph = objagg_tmp_graph_create(objagg);
 855         if (!graph)
 856                 return -ENOMEM;
 857
 858         /* Find the nodes from the ones that can accommodate most users
 859          * and cross them out of the graph. Save them to the hint list.
 860          */
 861         while ((index = objagg_tmp_graph_node_max_weight(graph)) != -1) {
 862                 node = &graph->nodes[index];
 863                 node->crossed_out = true;
 864                 hnode = objagg_hints_node_create(objagg_hints,
 865                                                  node->objagg_obj,
 866                                                  objagg->ops->obj_size,
 867                                                  NULL);
 868                 if (IS_ERR(hnode)) {
 869                         err = PTR_ERR(hnode);
 870                         goto out;
 871                 }
 872                 parent_hnode = hnode;
 873                 for (j = 0; j < graph->nodes_count; j++) {
 874                         if (!objagg_tmp_graph_is_edge(graph, index, j))
 875                                 continue;
 876                         node = &graph->nodes[j];
 877                         if (node->crossed_out)
 878                                 continue;
 879                         node->crossed_out = true;
 880                         hnode = objagg_hints_node_create(objagg_hints,
 881                                                          node->objagg_obj,
 882                                                          objagg->ops->obj_size,
 883                                                          parent_hnode);
 884                         if (IS_ERR(hnode)) {
 885                                 err = PTR_ERR(hnode);
 886                                 goto out;
 887                         }
 888                 }
 889         }
 890
 891         err = 0;
 892 out:
 893         objagg_tmp_graph_destroy(graph);
 894         return err;
 895 }
 896
 897 struct objagg_opt_algo {
 898         int (*fillup_hints)(struct objagg_hints *objagg_hints,
 899                             struct objagg *objagg);
 900 };
 901
 902 static const struct objagg_opt_algo objagg_opt_simple_greedy = {
 903         .fillup_hints = objagg_opt_simple_greedy_fillup_hints,
 904 };
 905
 906
 907 static const struct objagg_opt_algo *objagg_opt_algos[] = {
 908         [OBJAGG_OPT_ALGO_SIMPLE_GREEDY] = &objagg_opt_simple_greedy,
 909 };
 910
 911 static int objagg_hints_obj_cmp(struct rhashtable_compare_arg *arg,
 912                                 const void *obj)
 913 {
 914         struct rhashtable *ht = arg->ht;
 915         struct objagg_hints *objagg_hints =
 916                         container_of(ht, struct objagg_hints, node_ht);
 917         const struct objagg_ops *ops = objagg_hints->ops;
 918         const char *ptr = obj;
 919
 920         ptr += ht->p.key_offset;
 921         return ops->hints_obj_cmp ? ops->hints_obj_cmp(ptr, arg->key) :
 922                                     memcmp(ptr, arg->key, ht->p.key_len);
 923 }
 924
 925 /**
 926  * objagg_hints_get - obtains hints instance
 927  * @objagg:             objagg instance
 928  * @opt_algo_type:      type of hints finding algorithm
 929  *
 930  * Note: all locking must be provided by the caller.
 931  *
 932  * According to the algo type, the existing objects of objagg instance
 933  * are going to be went-through to assemble an optimal tree. We call this
 934  * tree hints. These hints can be later on used for creation of
 935  * a new objagg instance. There, the future object creations are going
 936  * to be consulted with these hints in order to find out, where exactly
 937  * the new object should be put as a root or delta.
 938  *
 939  * Returns a pointer to hints instance in case of success,
 940  * otherwise it returns pointer error using ERR_PTR macro.
 941  */
 942 struct objagg_hints *objagg_hints_get(struct objagg *objagg,
 943                                       enum objagg_opt_algo_type opt_algo_type)
 944 {
 945         const struct objagg_opt_algo *algo = objagg_opt_algos[opt_algo_type];
 946         struct objagg_hints *objagg_hints;
 947         int err;
 948
 949         objagg_hints = kzalloc(sizeof(*objagg_hints), GFP_KERNEL);
 950         if (!objagg_hints)
 951                 return ERR_PTR(-ENOMEM);
 952
 953         objagg_hints->ops = objagg->ops;
 954         objagg_hints->refcount = 1;
 955
 956         INIT_LIST_HEAD(&objagg_hints->node_list);
 957
 958         objagg_hints->ht_params.key_len = objagg->ops->obj_size;
 959         objagg_hints->ht_params.key_offset =
 960                                 offsetof(struct objagg_hints_node, obj);
 961         objagg_hints->ht_params.head_offset =
 962                                 offsetof(struct objagg_hints_node, ht_node);
 963         objagg_hints->ht_params.obj_cmpfn = objagg_hints_obj_cmp;
 964
 965         err = rhashtable_init(&objagg_hints->node_ht, &objagg_hints->ht_params);
 966         if (err)
 967                 goto err_rhashtable_init;
 968
 969         err = algo->fillup_hints(objagg_hints, objagg);
 970         if (err)
 971                 goto err_fillup_hints;
 972
 973         if (WARN_ON(objagg_hints->node_count != objagg->obj_count)) {
 974                 err = -EINVAL;
 975                 goto err_node_count_check;
 976         }
 977
 978         return objagg_hints;
 979
 980 err_node_count_check:
 981 err_fillup_hints:
 982         objagg_hints_flush(objagg_hints);
 983         rhashtable_destroy(&objagg_hints->node_ht);
 984 err_rhashtable_init:
 985         kfree(objagg_hints);
 986         return ERR_PTR(err);
 987 }
 988 EXPORT_SYMBOL(objagg_hints_get);
 989
 990 /**
 991  * objagg_hints_put - puts hints instance
 992  * @objagg_hints:       objagg hints instance
 993  *
 994  * Note: all locking must be provided by the caller.
 995  */
 996 void objagg_hints_put(struct objagg_hints *objagg_hints)
 997 {
 998         if (--objagg_hints->refcount)
 999                 return;
1000         objagg_hints_flush(objagg_hints);
1001         rhashtable_destroy(&objagg_hints->node_ht);
1002         kfree(objagg_hints);
1003 }
1004 EXPORT_SYMBOL(objagg_hints_put);
1005
1006 /**
1007  * objagg_hints_stats_get - obtains stats of the hints instance
1008  * @objagg_hints:       hints instance
1009  *
1010  * Note: all locking must be provided by the caller.
1011  *
1012  * The returned structure contains statistics of all objects
1013  * currently in use, ordered by following rules:
1014  * 1) Root objects are always on lower indexes than the rest.
1015  * 2) Objects with higher delta user count are always on lower
1016  *    indexes.
1017  * 3) In case multiple objects have the same delta user count,
1018  *    the objects are ordered by user count.
1019  *
1020  * Returns a pointer to stats instance in case of success,
1021  * otherwise it returns pointer error using ERR_PTR macro.
1022  */
1023 const struct objagg_stats *
1024 objagg_hints_stats_get(struct objagg_hints *objagg_hints)
1025 {
1026         struct objagg_stats *objagg_stats;
1027         struct objagg_hints_node *hnode;
1028         int i;
1029
1030         objagg_stats = kzalloc(struct_size(objagg_stats, stats_info,
1031                                            objagg_hints->node_count),
1032                                GFP_KERNEL);
1033         if (!objagg_stats)
1034                 return ERR_PTR(-ENOMEM);
1035
1036         i = 0;
1037         list_for_each_entry(hnode, &objagg_hints->node_list, list) {
1038                 memcpy(&objagg_stats->stats_info[i], &hnode->stats_info,
1039                        sizeof(objagg_stats->stats_info[0]));
1040                 if (objagg_stats->stats_info[i].is_root)
1041                         objagg_stats->root_count++;
1042                 i++;
1043         }
1044         objagg_stats->stats_info_count = i;
1045
1046         sort(objagg_stats->stats_info, objagg_stats->stats_info_count,
1047              sizeof(struct objagg_obj_stats_info),
1048              objagg_stats_info_sort_cmp_func, NULL);
1049
1050         return objagg_stats;
1051 }
1052 EXPORT_SYMBOL(objagg_hints_stats_get);
1053
1054 MODULE_LICENSE("Dual BSD/GPL");
1055 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
1056 MODULE_DESCRIPTION("Object aggregation manager");