net/core/devlink.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/core/devlink.c - Network physical/parent device Netlink interface
   4  *
   5  * Heavily inspired by net/wireless/
   6  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   7  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   8  */
   9
  10 #include <linux/etherdevice.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/types.h>
  14 #include <linux/slab.h>
  15 #include <linux/gfp.h>
  16 #include <linux/device.h>
  17 #include <linux/list.h>
  18 #include <linux/netdevice.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/refcount.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/u64_stats_sync.h>
  23 #include <linux/timekeeping.h>
  24 #include <rdma/ib_verbs.h>
  25 #include <net/netlink.h>
  26 #include <net/genetlink.h>
  27 #include <net/rtnetlink.h>
  28 #include <net/net_namespace.h>
  29 #include <net/sock.h>
  30 #include <net/devlink.h>
  31 #define CREATE_TRACE_POINTS
  32 #include <trace/events/devlink.h>
  33
  34 #define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
  35         (__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
  36
  37 struct devlink_dev_stats {
  38         u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  39         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  40 };
  41
  42 struct devlink {
  43         u32 index;
  44         struct list_head port_list;
  45         struct list_head rate_list;
  46         struct list_head sb_list;
  47         struct list_head dpipe_table_list;
  48         struct list_head resource_list;
  49         struct list_head param_list;
  50         struct list_head region_list;
  51         struct list_head reporter_list;
  52         struct mutex reporters_lock; /* protects reporter_list */
  53         struct devlink_dpipe_headers *dpipe_headers;
  54         struct list_head trap_list;
  55         struct list_head trap_group_list;
  56         struct list_head trap_policer_list;
  57         struct list_head linecard_list;
  58         struct mutex linecards_lock; /* protects linecard_list */
  59         const struct devlink_ops *ops;
  60         u64 features;
  61         struct xarray snapshot_ids;
  62         struct devlink_dev_stats stats;
  63         struct device *dev;
  64         possible_net_t _net;
  65         /* Serializes access to devlink instance specific objects such as
  66          * port, sb, dpipe, resource, params, region, traps and more.
  67          */
  68         struct mutex lock;
  69         u8 reload_failed:1;
  70         refcount_t refcount;
  71         struct completion comp;
  72         char priv[] __aligned(NETDEV_ALIGN);
  73 };
  74
  75 struct devlink_linecard_ops;
  76 struct devlink_linecard_type;
  77
  78 struct devlink_linecard {
  79         struct list_head list;
  80         struct devlink *devlink;
  81         unsigned int index;
  82         refcount_t refcount;
  83         const struct devlink_linecard_ops *ops;
  84         void *priv;
  85         enum devlink_linecard_state state;
  86         struct mutex state_lock; /* Protects state */
  87         const char *type;
  88         struct devlink_linecard_type *types;
  89         unsigned int types_count;
  90 };
  91
  92 /**
  93  * struct devlink_resource - devlink resource
  94  * @name: name of the resource
  95  * @id: id, per devlink instance
  96  * @size: size of the resource
  97  * @size_new: updated size of the resource, reload is needed
  98  * @size_valid: valid in case the total size of the resource is valid
  99  *              including its children
 100  * @parent: parent resource
 101  * @size_params: size parameters
 102  * @list: parent list
 103  * @resource_list: list of child resources
 104  * @occ_get: occupancy getter callback
 105  * @occ_get_priv: occupancy getter callback priv
 106  */
 107 struct devlink_resource {
 108         const char *name;
 109         u64 id;
 110         u64 size;
 111         u64 size_new;
 112         bool size_valid;
 113         struct devlink_resource *parent;
 114         struct devlink_resource_size_params size_params;
 115         struct list_head list;
 116         struct list_head resource_list;
 117         devlink_resource_occ_get_t *occ_get;
 118         void *occ_get_priv;
 119 };
 120
 121 void *devlink_priv(struct devlink *devlink)
 122 {
 123         return &devlink->priv;
 124 }
 125 EXPORT_SYMBOL_GPL(devlink_priv);
 126
 127 struct devlink *priv_to_devlink(void *priv)
 128 {
 129         return container_of(priv, struct devlink, priv);
 130 }
 131 EXPORT_SYMBOL_GPL(priv_to_devlink);
 132
 133 struct device *devlink_to_dev(const struct devlink *devlink)
 134 {
 135         return devlink->dev;
 136 }
 137 EXPORT_SYMBOL_GPL(devlink_to_dev);
 138
 139 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
 140         {
 141                 .name = "destination mac",
 142                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
 143                 .bitwidth = 48,
 144         },
 145 };
 146
 147 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
 148         .name = "ethernet",
 149         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
 150         .fields = devlink_dpipe_fields_ethernet,
 151         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
 152         .global = true,
 153 };
 154 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
 155
 156 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
 157         {
 158                 .name = "destination ip",
 159                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
 160                 .bitwidth = 32,
 161         },
 162 };
 163
 164 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
 165         .name = "ipv4",
 166         .id = DEVLINK_DPIPE_HEADER_IPV4,
 167         .fields = devlink_dpipe_fields_ipv4,
 168         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
 169         .global = true,
 170 };
 171 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
 172
 173 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
 174         {
 175                 .name = "destination ip",
 176                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
 177                 .bitwidth = 128,
 178         },
 179 };
 180
 181 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
 182         .name = "ipv6",
 183         .id = DEVLINK_DPIPE_HEADER_IPV6,
 184         .fields = devlink_dpipe_fields_ipv6,
 185         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
 186         .global = true,
 187 };
 188 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
 189
 190 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
 191 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
 192 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
 193
 194 static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
 195         [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
 196         [DEVLINK_PORT_FN_ATTR_STATE] =
 197                 NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
 198                                  DEVLINK_PORT_FN_STATE_ACTIVE),
 199 };
 200
 201 static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
 202 #define DEVLINK_REGISTERED XA_MARK_1
 203
 204 /* devlink instances are open to the access from the user space after
 205  * devlink_register() call. Such logical barrier allows us to have certain
 206  * expectations related to locking.
 207  *
 208  * Before *_register() - we are in initialization stage and no parallel
 209  * access possible to the devlink instance. All drivers perform that phase
 210  * by implicitly holding device_lock.
 211  *
 212  * After *_register() - users and driver can access devlink instance at
 213  * the same time.
 214  */
 215 #define ASSERT_DEVLINK_REGISTERED(d)                                           \
 216         WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 217 #define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
 218         WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 219
 220 /* devlink_mutex
 221  *
 222  * An overall lock guarding every operation coming from userspace.
 223  * It also guards devlink devices list and it is taken when
 224  * driver registers/unregisters it.
 225  */
 226 static DEFINE_MUTEX(devlink_mutex);
 227
 228 struct net *devlink_net(const struct devlink *devlink)
 229 {
 230         return read_pnet(&devlink->_net);
 231 }
 232 EXPORT_SYMBOL_GPL(devlink_net);
 233
 234 void devlink_put(struct devlink *devlink)
 235 {
 236         if (refcount_dec_and_test(&devlink->refcount))
 237                 complete(&devlink->comp);
 238 }
 239
 240 struct devlink *__must_check devlink_try_get(struct devlink *devlink)
 241 {
 242         if (refcount_inc_not_zero(&devlink->refcount))
 243                 return devlink;
 244         return NULL;
 245 }
 246
 247 void devl_assert_locked(struct devlink *devlink)
 248 {
 249         lockdep_assert_held(&devlink->lock);
 250 }
 251 EXPORT_SYMBOL_GPL(devl_assert_locked);
 252
 253 #ifdef CONFIG_LOCKDEP
 254 /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
 255 bool devl_lock_is_held(struct devlink *devlink)
 256 {
 257         return lockdep_is_held(&devlink->lock);
 258 }
 259 EXPORT_SYMBOL_GPL(devl_lock_is_held);
 260 #endif
 261
 262 void devl_lock(struct devlink *devlink)
 263 {
 264         mutex_lock(&devlink->lock);
 265 }
 266 EXPORT_SYMBOL_GPL(devl_lock);
 267
 268 void devl_unlock(struct devlink *devlink)
 269 {
 270         mutex_unlock(&devlink->lock);
 271 }
 272 EXPORT_SYMBOL_GPL(devl_unlock);
 273
 274 static struct devlink *devlink_get_from_attrs(struct net *net,
 275                                               struct nlattr **attrs)
 276 {
 277         struct devlink *devlink;
 278         unsigned long index;
 279         bool found = false;
 280         char *busname;
 281         char *devname;
 282
 283         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 284                 return ERR_PTR(-EINVAL);
 285
 286         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 287         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 288
 289         lockdep_assert_held(&devlink_mutex);
 290
 291         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
 292                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 293                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 294                     net_eq(devlink_net(devlink), net)) {
 295                         found = true;
 296                         break;
 297                 }
 298         }
 299
 300         if (!found || !devlink_try_get(devlink))
 301                 devlink = ERR_PTR(-ENODEV);
 302
 303         return devlink;
 304 }
 305
 306 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 307                                                       unsigned int port_index)
 308 {
 309         struct devlink_port *devlink_port;
 310
 311         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 312                 if (devlink_port->index == port_index)
 313                         return devlink_port;
 314         }
 315         return NULL;
 316 }
 317
 318 static bool devlink_port_index_exists(struct devlink *devlink,
 319                                       unsigned int port_index)
 320 {
 321         return devlink_port_get_by_index(devlink, port_index);
 322 }
 323
 324 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 325                                                         struct nlattr **attrs)
 326 {
 327         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 328                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 329                 struct devlink_port *devlink_port;
 330
 331                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 332                 if (!devlink_port)
 333                         return ERR_PTR(-ENODEV);
 334                 return devlink_port;
 335         }
 336         return ERR_PTR(-EINVAL);
 337 }
 338
 339 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 340                                                        struct genl_info *info)
 341 {
 342         return devlink_port_get_from_attrs(devlink, info->attrs);
 343 }
 344
 345 static inline bool
 346 devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
 347 {
 348         return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
 349 }
 350
 351 static inline bool
 352 devlink_rate_is_node(struct devlink_rate *devlink_rate)
 353 {
 354         return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
 355 }
 356
 357 static struct devlink_rate *
 358 devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
 359 {
 360         struct devlink_rate *devlink_rate;
 361         struct devlink_port *devlink_port;
 362
 363         devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
 364         if (IS_ERR(devlink_port))
 365                 return ERR_CAST(devlink_port);
 366         devlink_rate = devlink_port->devlink_rate;
 367         return devlink_rate ?: ERR_PTR(-ENODEV);
 368 }
 369
 370 static struct devlink_rate *
 371 devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
 372 {
 373         static struct devlink_rate *devlink_rate;
 374
 375         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
 376                 if (devlink_rate_is_node(devlink_rate) &&
 377                     !strcmp(node_name, devlink_rate->name))
 378                         return devlink_rate;
 379         }
 380         return ERR_PTR(-ENODEV);
 381 }
 382
 383 static struct devlink_rate *
 384 devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 385 {
 386         const char *rate_node_name;
 387         size_t len;
 388
 389         if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 390                 return ERR_PTR(-EINVAL);
 391         rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
 392         len = strlen(rate_node_name);
 393         /* Name cannot be empty or decimal number */
 394         if (!len || strspn(rate_node_name, "0123456789") == len)
 395                 return ERR_PTR(-EINVAL);
 396
 397         return devlink_rate_node_get_by_name(devlink, rate_node_name);
 398 }
 399
 400 static struct devlink_rate *
 401 devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
 402 {
 403         return devlink_rate_node_get_from_attrs(devlink, info->attrs);
 404 }
 405
 406 static struct devlink_rate *
 407 devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
 408 {
 409         struct nlattr **attrs = info->attrs;
 410
 411         if (attrs[DEVLINK_ATTR_PORT_INDEX])
 412                 return devlink_rate_leaf_get_from_info(devlink, info);
 413         else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 414                 return devlink_rate_node_get_from_info(devlink, info);
 415         else
 416                 return ERR_PTR(-EINVAL);
 417 }
 418
 419 static struct devlink_linecard *
 420 devlink_linecard_get_by_index(struct devlink *devlink,
 421                               unsigned int linecard_index)
 422 {
 423         struct devlink_linecard *devlink_linecard;
 424
 425         list_for_each_entry(devlink_linecard, &devlink->linecard_list, list) {
 426                 if (devlink_linecard->index == linecard_index)
 427                         return devlink_linecard;
 428         }
 429         return NULL;
 430 }
 431
 432 static bool devlink_linecard_index_exists(struct devlink *devlink,
 433                                           unsigned int linecard_index)
 434 {
 435         return devlink_linecard_get_by_index(devlink, linecard_index);
 436 }
 437
 438 static struct devlink_linecard *
 439 devlink_linecard_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 440 {
 441         if (attrs[DEVLINK_ATTR_LINECARD_INDEX]) {
 442                 u32 linecard_index = nla_get_u32(attrs[DEVLINK_ATTR_LINECARD_INDEX]);
 443                 struct devlink_linecard *linecard;
 444
 445                 mutex_lock(&devlink->linecards_lock);
 446                 linecard = devlink_linecard_get_by_index(devlink, linecard_index);
 447                 if (linecard)
 448                         refcount_inc(&linecard->refcount);
 449                 mutex_unlock(&devlink->linecards_lock);
 450                 if (!linecard)
 451                         return ERR_PTR(-ENODEV);
 452                 return linecard;
 453         }
 454         return ERR_PTR(-EINVAL);
 455 }
 456
 457 static struct devlink_linecard *
 458 devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info)
 459 {
 460         return devlink_linecard_get_from_attrs(devlink, info->attrs);
 461 }
 462
 463 static void devlink_linecard_put(struct devlink_linecard *linecard)
 464 {
 465         if (refcount_dec_and_test(&linecard->refcount)) {
 466                 mutex_destroy(&linecard->state_lock);
 467                 kfree(linecard);
 468         }
 469 }
 470
 471 struct devlink_sb {
 472         struct list_head list;
 473         unsigned int index;
 474         u32 size;
 475         u16 ingress_pools_count;
 476         u16 egress_pools_count;
 477         u16 ingress_tc_count;
 478         u16 egress_tc_count;
 479 };
 480
 481 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 482 {
 483         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 484 }
 485
 486 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 487                                                   unsigned int sb_index)
 488 {
 489         struct devlink_sb *devlink_sb;
 490
 491         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 492                 if (devlink_sb->index == sb_index)
 493                         return devlink_sb;
 494         }
 495         return NULL;
 496 }
 497
 498 static bool devlink_sb_index_exists(struct devlink *devlink,
 499                                     unsigned int sb_index)
 500 {
 501         return devlink_sb_get_by_index(devlink, sb_index);
 502 }
 503
 504 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 505                                                     struct nlattr **attrs)
 506 {
 507         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 508                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 509                 struct devlink_sb *devlink_sb;
 510
 511                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 512                 if (!devlink_sb)
 513                         return ERR_PTR(-ENODEV);
 514                 return devlink_sb;
 515         }
 516         return ERR_PTR(-EINVAL);
 517 }
 518
 519 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 520                                                    struct genl_info *info)
 521 {
 522         return devlink_sb_get_from_attrs(devlink, info->attrs);
 523 }
 524
 525 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 526                                                 struct nlattr **attrs,
 527                                                 u16 *p_pool_index)
 528 {
 529         u16 val;
 530
 531         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 532                 return -EINVAL;
 533
 534         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 535         if (val >= devlink_sb_pool_count(devlink_sb))
 536                 return -EINVAL;
 537         *p_pool_index = val;
 538         return 0;
 539 }
 540
 541 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 542                                                struct genl_info *info,
 543                                                u16 *p_pool_index)
 544 {
 545         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 546                                                     p_pool_index);
 547 }
 548
 549 static int
 550 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 551                                     enum devlink_sb_pool_type *p_pool_type)
 552 {
 553         u8 val;
 554
 555         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 556                 return -EINVAL;
 557
 558         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 559         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 560             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 561                 return -EINVAL;
 562         *p_pool_type = val;
 563         return 0;
 564 }
 565
 566 static int
 567 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 568                                    enum devlink_sb_pool_type *p_pool_type)
 569 {
 570         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 571 }
 572
 573 static int
 574 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 575                                   enum devlink_sb_threshold_type *p_th_type)
 576 {
 577         u8 val;
 578
 579         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 580                 return -EINVAL;
 581
 582         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 583         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 584             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 585                 return -EINVAL;
 586         *p_th_type = val;
 587         return 0;
 588 }
 589
 590 static int
 591 devlink_sb_th_type_get_from_info(struct genl_info *info,
 592                                  enum devlink_sb_threshold_type *p_th_type)
 593 {
 594         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 595 }
 596
 597 static int
 598 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 599                                    struct nlattr **attrs,
 600                                    enum devlink_sb_pool_type pool_type,
 601                                    u16 *p_tc_index)
 602 {
 603         u16 val;
 604
 605         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 606                 return -EINVAL;
 607
 608         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 609         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 610             val >= devlink_sb->ingress_tc_count)
 611                 return -EINVAL;
 612         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 613             val >= devlink_sb->egress_tc_count)
 614                 return -EINVAL;
 615         *p_tc_index = val;
 616         return 0;
 617 }
 618
 619 static int
 620 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 621                                   struct genl_info *info,
 622                                   enum devlink_sb_pool_type pool_type,
 623                                   u16 *p_tc_index)
 624 {
 625         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 626                                                   pool_type, p_tc_index);
 627 }
 628
 629 struct devlink_region {
 630         struct devlink *devlink;
 631         struct devlink_port *port;
 632         struct list_head list;
 633         union {
 634                 const struct devlink_region_ops *ops;
 635                 const struct devlink_port_region_ops *port_ops;
 636         };
 637         struct list_head snapshot_list;
 638         u32 max_snapshots;
 639         u32 cur_snapshots;
 640         u64 size;
 641 };
 642
 643 struct devlink_snapshot {
 644         struct list_head list;
 645         struct devlink_region *region;
 646         u8 *data;
 647         u32 id;
 648 };
 649
 650 static struct devlink_region *
 651 devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
 652 {
 653         struct devlink_region *region;
 654
 655         list_for_each_entry(region, &devlink->region_list, list)
 656                 if (!strcmp(region->ops->name, region_name))
 657                         return region;
 658
 659         return NULL;
 660 }
 661
 662 static struct devlink_region *
 663 devlink_port_region_get_by_name(struct devlink_port *port,
 664                                 const char *region_name)
 665 {
 666         struct devlink_region *region;
 667
 668         list_for_each_entry(region, &port->region_list, list)
 669                 if (!strcmp(region->ops->name, region_name))
 670                         return region;
 671
 672         return NULL;
 673 }
 674
 675 static struct devlink_snapshot *
 676 devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
 677 {
 678         struct devlink_snapshot *snapshot;
 679
 680         list_for_each_entry(snapshot, &region->snapshot_list, list)
 681                 if (snapshot->id == id)
 682                         return snapshot;
 683
 684         return NULL;
 685 }
 686
 687 #define DEVLINK_NL_FLAG_NEED_PORT               BIT(0)
 688 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT    BIT(1)
 689 #define DEVLINK_NL_FLAG_NEED_RATE               BIT(2)
 690 #define DEVLINK_NL_FLAG_NEED_RATE_NODE          BIT(3)
 691 #define DEVLINK_NL_FLAG_NEED_LINECARD           BIT(4)
 692
 693 /* The per devlink instance lock is taken by default in the pre-doit
 694  * operation, yet several commands do not require this. The global
 695  * devlink lock is taken and protects from disruption by user-calls.
 696  */
 697 #define DEVLINK_NL_FLAG_NO_LOCK                 BIT(5)
 698
 699 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 700                                struct sk_buff *skb, struct genl_info *info)
 701 {
 702         struct devlink_linecard *linecard;
 703         struct devlink_port *devlink_port;
 704         struct devlink *devlink;
 705         int err;
 706
 707         mutex_lock(&devlink_mutex);
 708         devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
 709         if (IS_ERR(devlink)) {
 710                 mutex_unlock(&devlink_mutex);
 711                 return PTR_ERR(devlink);
 712         }
 713         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 714                 mutex_lock(&devlink->lock);
 715         info->user_ptr[0] = devlink;
 716         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 717                 devlink_port = devlink_port_get_from_info(devlink, info);
 718                 if (IS_ERR(devlink_port)) {
 719                         err = PTR_ERR(devlink_port);
 720                         goto unlock;
 721                 }
 722                 info->user_ptr[1] = devlink_port;
 723         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 724                 devlink_port = devlink_port_get_from_info(devlink, info);
 725                 if (!IS_ERR(devlink_port))
 726                         info->user_ptr[1] = devlink_port;
 727         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
 728                 struct devlink_rate *devlink_rate;
 729
 730                 devlink_rate = devlink_rate_get_from_info(devlink, info);
 731                 if (IS_ERR(devlink_rate)) {
 732                         err = PTR_ERR(devlink_rate);
 733                         goto unlock;
 734                 }
 735                 info->user_ptr[1] = devlink_rate;
 736         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
 737                 struct devlink_rate *rate_node;
 738
 739                 rate_node = devlink_rate_node_get_from_info(devlink, info);
 740                 if (IS_ERR(rate_node)) {
 741                         err = PTR_ERR(rate_node);
 742                         goto unlock;
 743                 }
 744                 info->user_ptr[1] = rate_node;
 745         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
 746                 linecard = devlink_linecard_get_from_info(devlink, info);
 747                 if (IS_ERR(linecard)) {
 748                         err = PTR_ERR(linecard);
 749                         goto unlock;
 750                 }
 751                 info->user_ptr[1] = linecard;
 752         }
 753         return 0;
 754
 755 unlock:
 756         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 757                 mutex_unlock(&devlink->lock);
 758         devlink_put(devlink);
 759         mutex_unlock(&devlink_mutex);
 760         return err;
 761 }
 762
 763 static void devlink_nl_post_doit(const struct genl_ops *ops,
 764                                  struct sk_buff *skb, struct genl_info *info)
 765 {
 766         struct devlink_linecard *linecard;
 767         struct devlink *devlink;
 768
 769         devlink = info->user_ptr[0];
 770         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
 771                 linecard = info->user_ptr[1];
 772                 devlink_linecard_put(linecard);
 773         }
 774         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 775                 mutex_unlock(&devlink->lock);
 776         devlink_put(devlink);
 777         mutex_unlock(&devlink_mutex);
 778 }
 779
 780 static struct genl_family devlink_nl_family;
 781
 782 enum devlink_multicast_groups {
 783         DEVLINK_MCGRP_CONFIG,
 784 };
 785
 786 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 787         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 788 };
 789
 790 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 791 {
 792         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 793                 return -EMSGSIZE;
 794         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 795                 return -EMSGSIZE;
 796         return 0;
 797 }
 798
 799 struct devlink_reload_combination {
 800         enum devlink_reload_action action;
 801         enum devlink_reload_limit limit;
 802 };
 803
 804 static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
 805         {
 806                 /* can't reinitialize driver with no down time */
 807                 .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 808                 .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
 809         },
 810 };
 811
 812 static bool
 813 devlink_reload_combination_is_invalid(enum devlink_reload_action action,
 814                                       enum devlink_reload_limit limit)
 815 {
 816         int i;
 817
 818         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
 819                 if (devlink_reload_invalid_combinations[i].action == action &&
 820                     devlink_reload_invalid_combinations[i].limit == limit)
 821                         return true;
 822         return false;
 823 }
 824
 825 static bool
 826 devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
 827 {
 828         return test_bit(action, &devlink->ops->reload_actions);
 829 }
 830
 831 static bool
 832 devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
 833 {
 834         return test_bit(limit, &devlink->ops->reload_limits);
 835 }
 836
 837 static int devlink_reload_stat_put(struct sk_buff *msg,
 838                                    enum devlink_reload_limit limit, u32 value)
 839 {
 840         struct nlattr *reload_stats_entry;
 841
 842         reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
 843         if (!reload_stats_entry)
 844                 return -EMSGSIZE;
 845
 846         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
 847             nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
 848                 goto nla_put_failure;
 849         nla_nest_end(msg, reload_stats_entry);
 850         return 0;
 851
 852 nla_put_failure:
 853         nla_nest_cancel(msg, reload_stats_entry);
 854         return -EMSGSIZE;
 855 }
 856
 857 static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
 858 {
 859         struct nlattr *reload_stats_attr, *act_info, *act_stats;
 860         int i, j, stat_idx;
 861         u32 value;
 862
 863         if (!is_remote)
 864                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
 865         else
 866                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
 867
 868         if (!reload_stats_attr)
 869                 return -EMSGSIZE;
 870
 871         for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
 872                 if ((!is_remote &&
 873                      !devlink_reload_action_is_supported(devlink, i)) ||
 874                     i == DEVLINK_RELOAD_ACTION_UNSPEC)
 875                         continue;
 876                 act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
 877                 if (!act_info)
 878                         goto nla_put_failure;
 879
 880                 if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
 881                         goto action_info_nest_cancel;
 882                 act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
 883                 if (!act_stats)
 884                         goto action_info_nest_cancel;
 885
 886                 for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
 887                         /* Remote stats are shown even if not locally supported.
 888                          * Stats of actions with unspecified limit are shown
 889                          * though drivers don't need to register unspecified
 890                          * limit.
 891                          */
 892                         if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
 893                              !devlink_reload_limit_is_supported(devlink, j)) ||
 894                             devlink_reload_combination_is_invalid(i, j))
 895                                 continue;
 896
 897                         stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
 898                         if (!is_remote)
 899                                 value = devlink->stats.reload_stats[stat_idx];
 900                         else
 901                                 value = devlink->stats.remote_reload_stats[stat_idx];
 902                         if (devlink_reload_stat_put(msg, j, value))
 903                                 goto action_stats_nest_cancel;
 904                 }
 905                 nla_nest_end(msg, act_stats);
 906                 nla_nest_end(msg, act_info);
 907         }
 908         nla_nest_end(msg, reload_stats_attr);
 909         return 0;
 910
 911 action_stats_nest_cancel:
 912         nla_nest_cancel(msg, act_stats);
 913 action_info_nest_cancel:
 914         nla_nest_cancel(msg, act_info);
 915 nla_put_failure:
 916         nla_nest_cancel(msg, reload_stats_attr);
 917         return -EMSGSIZE;
 918 }
 919
 920 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 921                            enum devlink_command cmd, u32 portid,
 922                            u32 seq, int flags)
 923 {
 924         struct nlattr *dev_stats;
 925         void *hdr;
 926
 927         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 928         if (!hdr)
 929                 return -EMSGSIZE;
 930
 931         if (devlink_nl_put_handle(msg, devlink))
 932                 goto nla_put_failure;
 933         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
 934                 goto nla_put_failure;
 935
 936         dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
 937         if (!dev_stats)
 938                 goto nla_put_failure;
 939
 940         if (devlink_reload_stats_put(msg, devlink, false))
 941                 goto dev_stats_nest_cancel;
 942         if (devlink_reload_stats_put(msg, devlink, true))
 943                 goto dev_stats_nest_cancel;
 944
 945         nla_nest_end(msg, dev_stats);
 946         genlmsg_end(msg, hdr);
 947         return 0;
 948
 949 dev_stats_nest_cancel:
 950         nla_nest_cancel(msg, dev_stats);
 951 nla_put_failure:
 952         genlmsg_cancel(msg, hdr);
 953         return -EMSGSIZE;
 954 }
 955
 956 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 957 {
 958         struct sk_buff *msg;
 959         int err;
 960
 961         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 962         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
 963
 964         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 965         if (!msg)
 966                 return;
 967
 968         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 969         if (err) {
 970                 nlmsg_free(msg);
 971                 return;
 972         }
 973
 974         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 975                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 976 }
 977
 978 static int devlink_nl_port_attrs_put(struct sk_buff *msg,
 979                                      struct devlink_port *devlink_port)
 980 {
 981         struct devlink_port_attrs *attrs = &devlink_port->attrs;
 982
 983         if (!devlink_port->attrs_set)
 984                 return 0;
 985         if (attrs->lanes) {
 986                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
 987                         return -EMSGSIZE;
 988         }
 989         if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
 990                 return -EMSGSIZE;
 991         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
 992                 return -EMSGSIZE;
 993         switch (devlink_port->attrs.flavour) {
 994         case DEVLINK_PORT_FLAVOUR_PCI_PF:
 995                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 996                                 attrs->pci_pf.controller) ||
 997                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
 998                         return -EMSGSIZE;
 999                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
1000                         return -EMSGSIZE;
1001                 break;
1002         case DEVLINK_PORT_FLAVOUR_PCI_VF:
1003                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
1004                                 attrs->pci_vf.controller) ||
1005                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
1006                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
1007                         return -EMSGSIZE;
1008                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
1009                         return -EMSGSIZE;
1010                 break;
1011         case DEVLINK_PORT_FLAVOUR_PCI_SF:
1012                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
1013                                 attrs->pci_sf.controller) ||
1014                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
1015                                 attrs->pci_sf.pf) ||
1016                     nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
1017                                 attrs->pci_sf.sf))
1018                         return -EMSGSIZE;
1019                 break;
1020         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
1021         case DEVLINK_PORT_FLAVOUR_CPU:
1022         case DEVLINK_PORT_FLAVOUR_DSA:
1023                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
1024                                 attrs->phys.port_number))
1025                         return -EMSGSIZE;
1026                 if (!attrs->split)
1027                         return 0;
1028                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
1029                                 attrs->phys.port_number))
1030                         return -EMSGSIZE;
1031                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
1032                                 attrs->phys.split_subport_number))
1033                         return -EMSGSIZE;
1034                 break;
1035         default:
1036                 break;
1037         }
1038         return 0;
1039 }
1040
1041 static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
1042                                         struct devlink_port *port,
1043                                         struct sk_buff *msg,
1044                                         struct netlink_ext_ack *extack,
1045                                         bool *msg_updated)
1046 {
1047         u8 hw_addr[MAX_ADDR_LEN];
1048         int hw_addr_len;
1049         int err;
1050
1051         if (!ops->port_function_hw_addr_get)
1052                 return 0;
1053
1054         err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
1055                                              extack);
1056         if (err) {
1057                 if (err == -EOPNOTSUPP)
1058                         return 0;
1059                 return err;
1060         }
1061         err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
1062         if (err)
1063                 return err;
1064         *msg_updated = true;
1065         return 0;
1066 }
1067
1068 static int devlink_nl_rate_fill(struct sk_buff *msg,
1069                                 struct devlink_rate *devlink_rate,
1070                                 enum devlink_command cmd, u32 portid, u32 seq,
1071                                 int flags, struct netlink_ext_ack *extack)
1072 {
1073         struct devlink *devlink = devlink_rate->devlink;
1074         void *hdr;
1075
1076         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1077         if (!hdr)
1078                 return -EMSGSIZE;
1079
1080         if (devlink_nl_put_handle(msg, devlink))
1081                 goto nla_put_failure;
1082
1083         if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
1084                 goto nla_put_failure;
1085
1086         if (devlink_rate_is_leaf(devlink_rate)) {
1087                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
1088                                 devlink_rate->devlink_port->index))
1089                         goto nla_put_failure;
1090         } else if (devlink_rate_is_node(devlink_rate)) {
1091                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
1092                                    devlink_rate->name))
1093                         goto nla_put_failure;
1094         }
1095
1096         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
1097                               devlink_rate->tx_share, DEVLINK_ATTR_PAD))
1098                 goto nla_put_failure;
1099
1100         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
1101                               devlink_rate->tx_max, DEVLINK_ATTR_PAD))
1102                 goto nla_put_failure;
1103
1104         if (devlink_rate->parent)
1105                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
1106                                    devlink_rate->parent->name))
1107                         goto nla_put_failure;
1108
1109         genlmsg_end(msg, hdr);
1110         return 0;
1111
1112 nla_put_failure:
1113         genlmsg_cancel(msg, hdr);
1114         return -EMSGSIZE;
1115 }
1116
1117 static bool
1118 devlink_port_fn_state_valid(enum devlink_port_fn_state state)
1119 {
1120         return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
1121                state == DEVLINK_PORT_FN_STATE_ACTIVE;
1122 }
1123
1124 static bool
1125 devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
1126 {
1127         return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
1128                opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
1129 }
1130
1131 static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
1132                                       struct devlink_port *port,
1133                                       struct sk_buff *msg,
1134                                       struct netlink_ext_ack *extack,
1135                                       bool *msg_updated)
1136 {
1137         enum devlink_port_fn_opstate opstate;
1138         enum devlink_port_fn_state state;
1139         int err;
1140
1141         if (!ops->port_fn_state_get)
1142                 return 0;
1143
1144         err = ops->port_fn_state_get(port, &state, &opstate, extack);
1145         if (err) {
1146                 if (err == -EOPNOTSUPP)
1147                         return 0;
1148                 return err;
1149         }
1150         if (!devlink_port_fn_state_valid(state)) {
1151                 WARN_ON_ONCE(1);
1152                 NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
1153                 return -EINVAL;
1154         }
1155         if (!devlink_port_fn_opstate_valid(opstate)) {
1156                 WARN_ON_ONCE(1);
1157                 NL_SET_ERR_MSG_MOD(extack,
1158                                    "Invalid operational state read from driver");
1159                 return -EINVAL;
1160         }
1161         if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
1162             nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
1163                 return -EMSGSIZE;
1164         *msg_updated = true;
1165         return 0;
1166 }
1167
1168 static int
1169 devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
1170                                    struct netlink_ext_ack *extack)
1171 {
1172         const struct devlink_ops *ops;
1173         struct nlattr *function_attr;
1174         bool msg_updated = false;
1175         int err;
1176
1177         function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
1178         if (!function_attr)
1179                 return -EMSGSIZE;
1180
1181         ops = port->devlink->ops;
1182         err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
1183                                            &msg_updated);
1184         if (err)
1185                 goto out;
1186         err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
1187 out:
1188         if (err || !msg_updated)
1189                 nla_nest_cancel(msg, function_attr);
1190         else
1191                 nla_nest_end(msg, function_attr);
1192         return err;
1193 }
1194
1195 static int devlink_nl_port_fill(struct sk_buff *msg,
1196                                 struct devlink_port *devlink_port,
1197                                 enum devlink_command cmd, u32 portid, u32 seq,
1198                                 int flags, struct netlink_ext_ack *extack)
1199 {
1200         struct devlink *devlink = devlink_port->devlink;
1201         void *hdr;
1202
1203         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1204         if (!hdr)
1205                 return -EMSGSIZE;
1206
1207         if (devlink_nl_put_handle(msg, devlink))
1208                 goto nla_put_failure;
1209         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1210                 goto nla_put_failure;
1211
1212         /* Hold rtnl lock while accessing port's netdev attributes. */
1213         rtnl_lock();
1214         spin_lock_bh(&devlink_port->type_lock);
1215         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
1216                 goto nla_put_failure_type_locked;
1217         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
1218             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
1219                         devlink_port->desired_type))
1220                 goto nla_put_failure_type_locked;
1221         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
1222                 struct net *net = devlink_net(devlink_port->devlink);
1223                 struct net_device *netdev = devlink_port->type_dev;
1224
1225                 if (netdev && net_eq(net, dev_net(netdev)) &&
1226                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
1227                                  netdev->ifindex) ||
1228                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
1229                                     netdev->name)))
1230                         goto nla_put_failure_type_locked;
1231         }
1232         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
1233                 struct ib_device *ibdev = devlink_port->type_dev;
1234
1235                 if (ibdev &&
1236                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
1237                                    ibdev->name))
1238                         goto nla_put_failure_type_locked;
1239         }
1240         spin_unlock_bh(&devlink_port->type_lock);
1241         rtnl_unlock();
1242         if (devlink_nl_port_attrs_put(msg, devlink_port))
1243                 goto nla_put_failure;
1244         if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
1245                 goto nla_put_failure;
1246         if (devlink_port->linecard &&
1247             nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX,
1248                         devlink_port->linecard->index))
1249                 goto nla_put_failure;
1250
1251         genlmsg_end(msg, hdr);
1252         return 0;
1253
1254 nla_put_failure_type_locked:
1255         spin_unlock_bh(&devlink_port->type_lock);
1256         rtnl_unlock();
1257 nla_put_failure:
1258         genlmsg_cancel(msg, hdr);
1259         return -EMSGSIZE;
1260 }
1261
1262 static void devlink_port_notify(struct devlink_port *devlink_port,
1263                                 enum devlink_command cmd)
1264 {
1265         struct devlink *devlink = devlink_port->devlink;
1266         struct sk_buff *msg;
1267         int err;
1268
1269         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
1270
1271         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1272                 return;
1273
1274         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1275         if (!msg)
1276                 return;
1277
1278         err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
1279         if (err) {
1280                 nlmsg_free(msg);
1281                 return;
1282         }
1283
1284         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1285                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1286 }
1287
1288 static void devlink_rate_notify(struct devlink_rate *devlink_rate,
1289                                 enum devlink_command cmd)
1290 {
1291         struct devlink *devlink = devlink_rate->devlink;
1292         struct sk_buff *msg;
1293         int err;
1294
1295         WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
1296
1297         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1298                 return;
1299
1300         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1301         if (!msg)
1302                 return;
1303
1304         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
1305         if (err) {
1306                 nlmsg_free(msg);
1307                 return;
1308         }
1309
1310         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1311                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1312 }
1313
1314 static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
1315                                           struct netlink_callback *cb)
1316 {
1317         struct devlink_rate *devlink_rate;
1318         struct devlink *devlink;
1319         int start = cb->args[0];
1320         unsigned long index;
1321         int idx = 0;
1322         int err = 0;
1323
1324         mutex_lock(&devlink_mutex);
1325         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1326                 if (!devlink_try_get(devlink))
1327                         continue;
1328
1329                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1330                         goto retry;
1331
1332                 mutex_lock(&devlink->lock);
1333                 list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
1334                         enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
1335                         u32 id = NETLINK_CB(cb->skb).portid;
1336
1337                         if (idx < start) {
1338                                 idx++;
1339                                 continue;
1340                         }
1341                         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
1342                                                    cb->nlh->nlmsg_seq,
1343                                                    NLM_F_MULTI, NULL);
1344                         if (err) {
1345                                 mutex_unlock(&devlink->lock);
1346                                 devlink_put(devlink);
1347                                 goto out;
1348                         }
1349                         idx++;
1350                 }
1351                 mutex_unlock(&devlink->lock);
1352 retry:
1353                 devlink_put(devlink);
1354         }
1355 out:
1356         mutex_unlock(&devlink_mutex);
1357         if (err != -EMSGSIZE)
1358                 return err;
1359
1360         cb->args[0] = idx;
1361         return msg->len;
1362 }
1363
1364 static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
1365                                         struct genl_info *info)
1366 {
1367         struct devlink_rate *devlink_rate = info->user_ptr[1];
1368         struct sk_buff *msg;
1369         int err;
1370
1371         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1372         if (!msg)
1373                 return -ENOMEM;
1374
1375         err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
1376                                    info->snd_portid, info->snd_seq, 0,
1377                                    info->extack);
1378         if (err) {
1379                 nlmsg_free(msg);
1380                 return err;
1381         }
1382
1383         return genlmsg_reply(msg, info);
1384 }
1385
1386 static bool
1387 devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
1388                             struct devlink_rate *parent)
1389 {
1390         while (parent) {
1391                 if (parent == devlink_rate)
1392                         return true;
1393                 parent = parent->parent;
1394         }
1395         return false;
1396 }
1397
1398 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
1399 {
1400         struct devlink *devlink = info->user_ptr[0];
1401         struct sk_buff *msg;
1402         int err;
1403
1404         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1405         if (!msg)
1406                 return -ENOMEM;
1407
1408         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1409                               info->snd_portid, info->snd_seq, 0);
1410         if (err) {
1411                 nlmsg_free(msg);
1412                 return err;
1413         }
1414
1415         return genlmsg_reply(msg, info);
1416 }
1417
1418 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
1419                                      struct netlink_callback *cb)
1420 {
1421         struct devlink *devlink;
1422         int start = cb->args[0];
1423         unsigned long index;
1424         int idx = 0;
1425         int err;
1426
1427         mutex_lock(&devlink_mutex);
1428         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1429                 if (!devlink_try_get(devlink))
1430                         continue;
1431
1432                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk))) {
1433                         devlink_put(devlink);
1434                         continue;
1435                 }
1436
1437                 if (idx < start) {
1438                         idx++;
1439                         devlink_put(devlink);
1440                         continue;
1441                 }
1442
1443                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1444                                       NETLINK_CB(cb->skb).portid,
1445                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
1446                 devlink_put(devlink);
1447                 if (err)
1448                         goto out;
1449                 idx++;
1450         }
1451 out:
1452         mutex_unlock(&devlink_mutex);
1453
1454         cb->args[0] = idx;
1455         return msg->len;
1456 }
1457
1458 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
1459                                         struct genl_info *info)
1460 {
1461         struct devlink_port *devlink_port = info->user_ptr[1];
1462         struct sk_buff *msg;
1463         int err;
1464
1465         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1466         if (!msg)
1467                 return -ENOMEM;
1468
1469         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
1470                                    info->snd_portid, info->snd_seq, 0,
1471                                    info->extack);
1472         if (err) {
1473                 nlmsg_free(msg);
1474                 return err;
1475         }
1476
1477         return genlmsg_reply(msg, info);
1478 }
1479
1480 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
1481                                           struct netlink_callback *cb)
1482 {
1483         struct devlink *devlink;
1484         struct devlink_port *devlink_port;
1485         int start = cb->args[0];
1486         unsigned long index;
1487         int idx = 0;
1488         int err;
1489
1490         mutex_lock(&devlink_mutex);
1491         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1492                 if (!devlink_try_get(devlink))
1493                         continue;
1494
1495                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1496                         goto retry;
1497
1498                 mutex_lock(&devlink->lock);
1499                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
1500                         if (idx < start) {
1501                                 idx++;
1502                                 continue;
1503                         }
1504                         err = devlink_nl_port_fill(msg, devlink_port,
1505                                                    DEVLINK_CMD_NEW,
1506                                                    NETLINK_CB(cb->skb).portid,
1507                                                    cb->nlh->nlmsg_seq,
1508                                                    NLM_F_MULTI, cb->extack);
1509                         if (err) {
1510                                 mutex_unlock(&devlink->lock);
1511                                 devlink_put(devlink);
1512                                 goto out;
1513                         }
1514                         idx++;
1515                 }
1516                 mutex_unlock(&devlink->lock);
1517 retry:
1518                 devlink_put(devlink);
1519         }
1520 out:
1521         mutex_unlock(&devlink_mutex);
1522
1523         cb->args[0] = idx;
1524         return msg->len;
1525 }
1526
1527 static int devlink_port_type_set(struct devlink_port *devlink_port,
1528                                  enum devlink_port_type port_type)
1529
1530 {
1531         int err;
1532
1533         if (!devlink_port->devlink->ops->port_type_set)
1534                 return -EOPNOTSUPP;
1535
1536         if (port_type == devlink_port->type)
1537                 return 0;
1538
1539         err = devlink_port->devlink->ops->port_type_set(devlink_port,
1540                                                         port_type);
1541         if (err)
1542                 return err;
1543
1544         devlink_port->desired_type = port_type;
1545         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
1546         return 0;
1547 }
1548
1549 static int devlink_port_function_hw_addr_set(struct devlink_port *port,
1550                                              const struct nlattr *attr,
1551                                              struct netlink_ext_ack *extack)
1552 {
1553         const struct devlink_ops *ops = port->devlink->ops;
1554         const u8 *hw_addr;
1555         int hw_addr_len;
1556
1557         hw_addr = nla_data(attr);
1558         hw_addr_len = nla_len(attr);
1559         if (hw_addr_len > MAX_ADDR_LEN) {
1560                 NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
1561                 return -EINVAL;
1562         }
1563         if (port->type == DEVLINK_PORT_TYPE_ETH) {
1564                 if (hw_addr_len != ETH_ALEN) {
1565                         NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
1566                         return -EINVAL;
1567                 }
1568                 if (!is_unicast_ether_addr(hw_addr)) {
1569                         NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
1570                         return -EINVAL;
1571                 }
1572         }
1573
1574         if (!ops->port_function_hw_addr_set) {
1575                 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
1576                 return -EOPNOTSUPP;
1577         }
1578
1579         return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
1580                                               extack);
1581 }
1582
1583 static int devlink_port_fn_state_set(struct devlink_port *port,
1584                                      const struct nlattr *attr,
1585                                      struct netlink_ext_ack *extack)
1586 {
1587         enum devlink_port_fn_state state;
1588         const struct devlink_ops *ops;
1589
1590         state = nla_get_u8(attr);
1591         ops = port->devlink->ops;
1592         if (!ops->port_fn_state_set) {
1593                 NL_SET_ERR_MSG_MOD(extack,
1594                                    "Function does not support state setting");
1595                 return -EOPNOTSUPP;
1596         }
1597         return ops->port_fn_state_set(port, state, extack);
1598 }
1599
1600 static int devlink_port_function_set(struct devlink_port *port,
1601                                      const struct nlattr *attr,
1602                                      struct netlink_ext_ack *extack)
1603 {
1604         struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
1605         int err;
1606
1607         err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
1608                                devlink_function_nl_policy, extack);
1609         if (err < 0) {
1610                 NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
1611                 return err;
1612         }
1613
1614         attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
1615         if (attr) {
1616                 err = devlink_port_function_hw_addr_set(port, attr, extack);
1617                 if (err)
1618                         return err;
1619         }
1620         /* Keep this as the last function attribute set, so that when
1621          * multiple port function attributes are set along with state,
1622          * Those can be applied first before activating the state.
1623          */
1624         attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
1625         if (attr)
1626                 err = devlink_port_fn_state_set(port, attr, extack);
1627
1628         if (!err)
1629                 devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
1630         return err;
1631 }
1632
1633 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
1634                                         struct genl_info *info)
1635 {
1636         struct devlink_port *devlink_port = info->user_ptr[1];
1637         int err;
1638
1639         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
1640                 enum devlink_port_type port_type;
1641
1642                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
1643                 err = devlink_port_type_set(devlink_port, port_type);
1644                 if (err)
1645                         return err;
1646         }
1647
1648         if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
1649                 struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
1650                 struct netlink_ext_ack *extack = info->extack;
1651
1652                 err = devlink_port_function_set(devlink_port, attr, extack);
1653                 if (err)
1654                         return err;
1655         }
1656
1657         return 0;
1658 }
1659
1660 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
1661                                           struct genl_info *info)
1662 {
1663         struct devlink_port *devlink_port = info->user_ptr[1];
1664         struct devlink *devlink = info->user_ptr[0];
1665         u32 count;
1666
1667         if (!info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
1668                 return -EINVAL;
1669         if (!devlink->ops->port_split)
1670                 return -EOPNOTSUPP;
1671
1672         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
1673
1674         if (!devlink_port->attrs.splittable) {
1675                 /* Split ports cannot be split. */
1676                 if (devlink_port->attrs.split)
1677                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
1678                 else
1679                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
1680                 return -EINVAL;
1681         }
1682
1683         if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
1684                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
1685                 return -EINVAL;
1686         }
1687
1688         return devlink->ops->port_split(devlink, devlink_port, count,
1689                                         info->extack);
1690 }
1691
1692 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
1693                                             struct genl_info *info)
1694 {
1695         struct devlink_port *devlink_port = info->user_ptr[1];
1696         struct devlink *devlink = info->user_ptr[0];
1697
1698         if (!devlink->ops->port_unsplit)
1699                 return -EOPNOTSUPP;
1700         return devlink->ops->port_unsplit(devlink, devlink_port, info->extack);
1701 }
1702
1703 static int devlink_port_new_notifiy(struct devlink *devlink,
1704                                     unsigned int port_index,
1705                                     struct genl_info *info)
1706 {
1707         struct devlink_port *devlink_port;
1708         struct sk_buff *msg;
1709         int err;
1710
1711         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1712         if (!msg)
1713                 return -ENOMEM;
1714
1715         mutex_lock(&devlink->lock);
1716         devlink_port = devlink_port_get_by_index(devlink, port_index);
1717         if (!devlink_port) {
1718                 err = -ENODEV;
1719                 goto out;
1720         }
1721
1722         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
1723                                    info->snd_portid, info->snd_seq, 0, NULL);
1724         if (err)
1725                 goto out;
1726
1727         err = genlmsg_reply(msg, info);
1728         mutex_unlock(&devlink->lock);
1729         return err;
1730
1731 out:
1732         mutex_unlock(&devlink->lock);
1733         nlmsg_free(msg);
1734         return err;
1735 }
1736
1737 static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
1738                                         struct genl_info *info)
1739 {
1740         struct netlink_ext_ack *extack = info->extack;
1741         struct devlink_port_new_attrs new_attrs = {};
1742         struct devlink *devlink = info->user_ptr[0];
1743         unsigned int new_port_index;
1744         int err;
1745
1746         if (!devlink->ops->port_new || !devlink->ops->port_del)
1747                 return -EOPNOTSUPP;
1748
1749         if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
1750             !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
1751                 NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
1752                 return -EINVAL;
1753         }
1754         new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
1755         new_attrs.pfnum =
1756                 nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
1757
1758         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1759                 /* Port index of the new port being created by driver. */
1760                 new_attrs.port_index =
1761                         nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1762                 new_attrs.port_index_valid = true;
1763         }
1764         if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
1765                 new_attrs.controller =
1766                         nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
1767                 new_attrs.controller_valid = true;
1768         }
1769         if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
1770             info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
1771                 new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
1772                 new_attrs.sfnum_valid = true;
1773         }
1774
1775         err = devlink->ops->port_new(devlink, &new_attrs, extack,
1776                                      &new_port_index);
1777         if (err)
1778                 return err;
1779
1780         err = devlink_port_new_notifiy(devlink, new_port_index, info);
1781         if (err && err != -ENODEV) {
1782                 /* Fail to send the response; destroy newly created port. */
1783                 devlink->ops->port_del(devlink, new_port_index, extack);
1784         }
1785         return err;
1786 }
1787
1788 static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
1789                                         struct genl_info *info)
1790 {
1791         struct netlink_ext_ack *extack = info->extack;
1792         struct devlink *devlink = info->user_ptr[0];
1793         unsigned int port_index;
1794
1795         if (!devlink->ops->port_del)
1796                 return -EOPNOTSUPP;
1797
1798         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1799                 NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
1800                 return -EINVAL;
1801         }
1802         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1803
1804         return devlink->ops->port_del(devlink, port_index, extack);
1805 }
1806
1807 static int
1808 devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
1809                                 struct genl_info *info,
1810                                 struct nlattr *nla_parent)
1811 {
1812         struct devlink *devlink = devlink_rate->devlink;
1813         const char *parent_name = nla_data(nla_parent);
1814         const struct devlink_ops *ops = devlink->ops;
1815         size_t len = strlen(parent_name);
1816         struct devlink_rate *parent;
1817         int err = -EOPNOTSUPP;
1818
1819         parent = devlink_rate->parent;
1820         if (parent && len) {
1821                 NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
1822                 return -EBUSY;
1823         } else if (parent && !len) {
1824                 if (devlink_rate_is_leaf(devlink_rate))
1825                         err = ops->rate_leaf_parent_set(devlink_rate, NULL,
1826                                                         devlink_rate->priv, NULL,
1827                                                         info->extack);
1828                 else if (devlink_rate_is_node(devlink_rate))
1829                         err = ops->rate_node_parent_set(devlink_rate, NULL,
1830                                                         devlink_rate->priv, NULL,
1831                                                         info->extack);
1832                 if (err)
1833                         return err;
1834
1835                 refcount_dec(&parent->refcnt);
1836                 devlink_rate->parent = NULL;
1837         } else if (!parent && len) {
1838                 parent = devlink_rate_node_get_by_name(devlink, parent_name);
1839                 if (IS_ERR(parent))
1840                         return -ENODEV;
1841
1842                 if (parent == devlink_rate) {
1843                         NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
1844                         return -EINVAL;
1845                 }
1846
1847                 if (devlink_rate_is_node(devlink_rate) &&
1848                     devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
1849                         NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
1850                         return -EEXIST;
1851                 }
1852
1853                 if (devlink_rate_is_leaf(devlink_rate))
1854                         err = ops->rate_leaf_parent_set(devlink_rate, parent,
1855                                                         devlink_rate->priv, parent->priv,
1856                                                         info->extack);
1857                 else if (devlink_rate_is_node(devlink_rate))
1858                         err = ops->rate_node_parent_set(devlink_rate, parent,
1859                                                         devlink_rate->priv, parent->priv,
1860                                                         info->extack);
1861                 if (err)
1862                         return err;
1863
1864                 refcount_inc(&parent->refcnt);
1865                 devlink_rate->parent = parent;
1866         }
1867
1868         return 0;
1869 }
1870
1871 static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
1872                                const struct devlink_ops *ops,
1873                                struct genl_info *info)
1874 {
1875         struct nlattr *nla_parent, **attrs = info->attrs;
1876         int err = -EOPNOTSUPP;
1877         u64 rate;
1878
1879         if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
1880                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
1881                 if (devlink_rate_is_leaf(devlink_rate))
1882                         err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
1883                                                           rate, info->extack);
1884                 else if (devlink_rate_is_node(devlink_rate))
1885                         err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
1886                                                           rate, info->extack);
1887                 if (err)
1888                         return err;
1889                 devlink_rate->tx_share = rate;
1890         }
1891
1892         if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
1893                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
1894                 if (devlink_rate_is_leaf(devlink_rate))
1895                         err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
1896                                                         rate, info->extack);
1897                 else if (devlink_rate_is_node(devlink_rate))
1898                         err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
1899                                                         rate, info->extack);
1900                 if (err)
1901                         return err;
1902                 devlink_rate->tx_max = rate;
1903         }
1904
1905         nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
1906         if (nla_parent) {
1907                 err = devlink_nl_rate_parent_node_set(devlink_rate, info,
1908                                                       nla_parent);
1909                 if (err)
1910                         return err;
1911         }
1912
1913         return 0;
1914 }
1915
1916 static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
1917                                            struct genl_info *info,
1918                                            enum devlink_rate_type type)
1919 {
1920         struct nlattr **attrs = info->attrs;
1921
1922         if (type == DEVLINK_RATE_TYPE_LEAF) {
1923                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
1924                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
1925                         return false;
1926                 }
1927                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
1928                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
1929                         return false;
1930                 }
1931                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1932                     !ops->rate_leaf_parent_set) {
1933                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
1934                         return false;
1935                 }
1936         } else if (type == DEVLINK_RATE_TYPE_NODE) {
1937                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
1938                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
1939                         return false;
1940                 }
1941                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
1942                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
1943                         return false;
1944                 }
1945                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1946                     !ops->rate_node_parent_set) {
1947                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
1948                         return false;
1949                 }
1950         } else {
1951                 WARN(1, "Unknown type of rate object");
1952                 return false;
1953         }
1954
1955         return true;
1956 }
1957
1958 static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
1959                                         struct genl_info *info)
1960 {
1961         struct devlink_rate *devlink_rate = info->user_ptr[1];
1962         struct devlink *devlink = devlink_rate->devlink;
1963         const struct devlink_ops *ops = devlink->ops;
1964         int err;
1965
1966         if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
1967                 return -EOPNOTSUPP;
1968
1969         err = devlink_nl_rate_set(devlink_rate, ops, info);
1970
1971         if (!err)
1972                 devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
1973         return err;
1974 }
1975
1976 static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
1977                                         struct genl_info *info)
1978 {
1979         struct devlink *devlink = info->user_ptr[0];
1980         struct devlink_rate *rate_node;
1981         const struct devlink_ops *ops;
1982         int err;
1983
1984         ops = devlink->ops;
1985         if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
1986                 NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
1987                 return -EOPNOTSUPP;
1988         }
1989
1990         if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
1991                 return -EOPNOTSUPP;
1992
1993         rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
1994         if (!IS_ERR(rate_node))
1995                 return -EEXIST;
1996         else if (rate_node == ERR_PTR(-EINVAL))
1997                 return -EINVAL;
1998
1999         rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
2000         if (!rate_node)
2001                 return -ENOMEM;
2002
2003         rate_node->devlink = devlink;
2004         rate_node->type = DEVLINK_RATE_TYPE_NODE;
2005         rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
2006         if (!rate_node->name) {
2007                 err = -ENOMEM;
2008                 goto err_strdup;
2009         }
2010
2011         err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
2012         if (err)
2013                 goto err_node_new;
2014
2015         err = devlink_nl_rate_set(rate_node, ops, info);
2016         if (err)
2017                 goto err_rate_set;
2018
2019         refcount_set(&rate_node->refcnt, 1);
2020         list_add(&rate_node->list, &devlink->rate_list);
2021         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
2022         return 0;
2023
2024 err_rate_set:
2025         ops->rate_node_del(rate_node, rate_node->priv, info->extack);
2026 err_node_new:
2027         kfree(rate_node->name);
2028 err_strdup:
2029         kfree(rate_node);
2030         return err;
2031 }
2032
2033 static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
2034                                         struct genl_info *info)
2035 {
2036         struct devlink_rate *rate_node = info->user_ptr[1];
2037         struct devlink *devlink = rate_node->devlink;
2038         const struct devlink_ops *ops = devlink->ops;
2039         int err;
2040
2041         if (refcount_read(&rate_node->refcnt) > 1) {
2042                 NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
2043                 return -EBUSY;
2044         }
2045
2046         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
2047         err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
2048         if (rate_node->parent)
2049                 refcount_dec(&rate_node->parent->refcnt);
2050         list_del(&rate_node->list);
2051         kfree(rate_node->name);
2052         kfree(rate_node);
2053         return err;
2054 }
2055
2056 struct devlink_linecard_type {
2057         const char *type;
2058         const void *priv;
2059 };
2060
2061 static int devlink_nl_linecard_fill(struct sk_buff *msg,
2062                                     struct devlink *devlink,
2063                                     struct devlink_linecard *linecard,
2064                                     enum devlink_command cmd, u32 portid,
2065                                     u32 seq, int flags,
2066                                     struct netlink_ext_ack *extack)
2067 {
2068         struct devlink_linecard_type *linecard_type;
2069         struct nlattr *attr;
2070         void *hdr;
2071         int i;
2072
2073         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2074         if (!hdr)
2075                 return -EMSGSIZE;
2076
2077         if (devlink_nl_put_handle(msg, devlink))
2078                 goto nla_put_failure;
2079         if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index))
2080                 goto nla_put_failure;
2081         if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state))
2082                 goto nla_put_failure;
2083         if (linecard->type &&
2084             nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, linecard->type))
2085                 goto nla_put_failure;
2086
2087         if (linecard->types_count) {
2088                 attr = nla_nest_start(msg,
2089                                       DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES);
2090                 if (!attr)
2091                         goto nla_put_failure;
2092                 for (i = 0; i < linecard->types_count; i++) {
2093                         linecard_type = &linecard->types[i];
2094                         if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE,
2095                                            linecard_type->type)) {
2096                                 nla_nest_cancel(msg, attr);
2097                                 goto nla_put_failure;
2098                         }
2099                 }
2100                 nla_nest_end(msg, attr);
2101         }
2102
2103         genlmsg_end(msg, hdr);
2104         return 0;
2105
2106 nla_put_failure:
2107         genlmsg_cancel(msg, hdr);
2108         return -EMSGSIZE;
2109 }
2110
2111 static void devlink_linecard_notify(struct devlink_linecard *linecard,
2112                                     enum devlink_command cmd)
2113 {
2114         struct devlink *devlink = linecard->devlink;
2115         struct sk_buff *msg;
2116         int err;
2117
2118         WARN_ON(cmd != DEVLINK_CMD_LINECARD_NEW &&
2119                 cmd != DEVLINK_CMD_LINECARD_DEL);
2120
2121         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
2122                 return;
2123
2124         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2125         if (!msg)
2126                 return;
2127
2128         err = devlink_nl_linecard_fill(msg, devlink, linecard, cmd, 0, 0, 0,
2129                                        NULL);
2130         if (err) {
2131                 nlmsg_free(msg);
2132                 return;
2133         }
2134
2135         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
2136                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
2137 }
2138
2139 static int devlink_nl_cmd_linecard_get_doit(struct sk_buff *skb,
2140                                             struct genl_info *info)
2141 {
2142         struct devlink_linecard *linecard = info->user_ptr[1];
2143         struct devlink *devlink = linecard->devlink;
2144         struct sk_buff *msg;
2145         int err;
2146
2147         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2148         if (!msg)
2149                 return -ENOMEM;
2150
2151         mutex_lock(&linecard->state_lock);
2152         err = devlink_nl_linecard_fill(msg, devlink, linecard,
2153                                        DEVLINK_CMD_LINECARD_NEW,
2154                                        info->snd_portid, info->snd_seq, 0,
2155                                        info->extack);
2156         mutex_unlock(&linecard->state_lock);
2157         if (err) {
2158                 nlmsg_free(msg);
2159                 return err;
2160         }
2161
2162         return genlmsg_reply(msg, info);
2163 }
2164
2165 static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg,
2166                                               struct netlink_callback *cb)
2167 {
2168         struct devlink_linecard *linecard;
2169         struct devlink *devlink;
2170         int start = cb->args[0];
2171         unsigned long index;
2172         int idx = 0;
2173         int err;
2174
2175         mutex_lock(&devlink_mutex);
2176         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2177                 if (!devlink_try_get(devlink))
2178                         continue;
2179
2180                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2181                         goto retry;
2182
2183                 mutex_lock(&devlink->linecards_lock);
2184                 list_for_each_entry(linecard, &devlink->linecard_list, list) {
2185                         if (idx < start) {
2186                                 idx++;
2187                                 continue;
2188                         }
2189                         mutex_lock(&linecard->state_lock);
2190                         err = devlink_nl_linecard_fill(msg, devlink, linecard,
2191                                                        DEVLINK_CMD_LINECARD_NEW,
2192                                                        NETLINK_CB(cb->skb).portid,
2193                                                        cb->nlh->nlmsg_seq,
2194                                                        NLM_F_MULTI,
2195                                                        cb->extack);
2196                         mutex_unlock(&linecard->state_lock);
2197                         if (err) {
2198                                 mutex_unlock(&devlink->linecards_lock);
2199                                 devlink_put(devlink);
2200                                 goto out;
2201                         }
2202                         idx++;
2203                 }
2204                 mutex_unlock(&devlink->linecards_lock);
2205 retry:
2206                 devlink_put(devlink);
2207         }
2208 out:
2209         mutex_unlock(&devlink_mutex);
2210
2211         cb->args[0] = idx;
2212         return msg->len;
2213 }
2214
2215 static struct devlink_linecard_type *
2216 devlink_linecard_type_lookup(struct devlink_linecard *linecard,
2217                              const char *type)
2218 {
2219         struct devlink_linecard_type *linecard_type;
2220         int i;
2221
2222         for (i = 0; i < linecard->types_count; i++) {
2223                 linecard_type = &linecard->types[i];
2224                 if (!strcmp(type, linecard_type->type))
2225                         return linecard_type;
2226         }
2227         return NULL;
2228 }
2229
2230 static int devlink_linecard_type_set(struct devlink_linecard *linecard,
2231                                      const char *type,
2232                                      struct netlink_ext_ack *extack)
2233 {
2234         const struct devlink_linecard_ops *ops = linecard->ops;
2235         struct devlink_linecard_type *linecard_type;
2236         int err;
2237
2238         mutex_lock(&linecard->state_lock);
2239         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
2240                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
2241                 err = -EBUSY;
2242                 goto out;
2243         }
2244         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
2245                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
2246                 err = -EBUSY;
2247                 goto out;
2248         }
2249
2250         linecard_type = devlink_linecard_type_lookup(linecard, type);
2251         if (!linecard_type) {
2252                 NL_SET_ERR_MSG_MOD(extack, "Unsupported line card type provided");
2253                 err = -EINVAL;
2254                 goto out;
2255         }
2256
2257         if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED &&
2258             linecard->state != DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
2259                 NL_SET_ERR_MSG_MOD(extack, "Line card already provisioned");
2260                 err = -EBUSY;
2261                 /* Check if the line card is provisioned in the same
2262                  * way the user asks. In case it is, make the operation
2263                  * to return success.
2264                  */
2265                 if (ops->same_provision &&
2266                     ops->same_provision(linecard, linecard->priv,
2267                                         linecard_type->type,
2268                                         linecard_type->priv))
2269                         err = 0;
2270                 goto out;
2271         }
2272
2273         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING;
2274         linecard->type = linecard_type->type;
2275         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2276         mutex_unlock(&linecard->state_lock);
2277         err = ops->provision(linecard, linecard->priv, linecard_type->type,
2278                              linecard_type->priv, extack);
2279         if (err) {
2280                 /* Provisioning failed. Assume the linecard is unprovisioned
2281                  * for future operations.
2282                  */
2283                 mutex_lock(&linecard->state_lock);
2284                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2285                 linecard->type = NULL;
2286                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2287                 mutex_unlock(&linecard->state_lock);
2288         }
2289         return err;
2290
2291 out:
2292         mutex_unlock(&linecard->state_lock);
2293         return err;
2294 }
2295
2296 static int devlink_linecard_type_unset(struct devlink_linecard *linecard,
2297                                        struct netlink_ext_ack *extack)
2298 {
2299         int err;
2300
2301         mutex_lock(&linecard->state_lock);
2302         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
2303                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
2304                 err = -EBUSY;
2305                 goto out;
2306         }
2307         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
2308                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
2309                 err = -EBUSY;
2310                 goto out;
2311         }
2312         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
2313                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2314                 linecard->type = NULL;
2315                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2316                 err = 0;
2317                 goto out;
2318         }
2319
2320         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) {
2321                 NL_SET_ERR_MSG_MOD(extack, "Line card is not provisioned");
2322                 err = 0;
2323                 goto out;
2324         }
2325         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING;
2326         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2327         mutex_unlock(&linecard->state_lock);
2328         err = linecard->ops->unprovision(linecard, linecard->priv,
2329                                          extack);
2330         if (err) {
2331                 /* Unprovisioning failed. Assume the linecard is unprovisioned
2332                  * for future operations.
2333                  */
2334                 mutex_lock(&linecard->state_lock);
2335                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2336                 linecard->type = NULL;
2337                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2338                 mutex_unlock(&linecard->state_lock);
2339         }
2340         return err;
2341
2342 out:
2343         mutex_unlock(&linecard->state_lock);
2344         return err;
2345 }
2346
2347 static int devlink_nl_cmd_linecard_set_doit(struct sk_buff *skb,
2348                                             struct genl_info *info)
2349 {
2350         struct devlink_linecard *linecard = info->user_ptr[1];
2351         struct netlink_ext_ack *extack = info->extack;
2352         int err;
2353
2354         if (info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) {
2355                 const char *type;
2356
2357                 type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]);
2358                 if (strcmp(type, "")) {
2359                         err = devlink_linecard_type_set(linecard, type, extack);
2360                         if (err)
2361                                 return err;
2362                 } else {
2363                         err = devlink_linecard_type_unset(linecard, extack);
2364                         if (err)
2365                                 return err;
2366                 }
2367         }
2368
2369         return 0;
2370 }
2371
2372 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
2373                               struct devlink_sb *devlink_sb,
2374                               enum devlink_command cmd, u32 portid,
2375                               u32 seq, int flags)
2376 {
2377         void *hdr;
2378
2379         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2380         if (!hdr)
2381                 return -EMSGSIZE;
2382
2383         if (devlink_nl_put_handle(msg, devlink))
2384                 goto nla_put_failure;
2385         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2386                 goto nla_put_failure;
2387         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
2388                 goto nla_put_failure;
2389         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
2390                         devlink_sb->ingress_pools_count))
2391                 goto nla_put_failure;
2392         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
2393                         devlink_sb->egress_pools_count))
2394                 goto nla_put_failure;
2395         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
2396                         devlink_sb->ingress_tc_count))
2397                 goto nla_put_failure;
2398         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
2399                         devlink_sb->egress_tc_count))
2400                 goto nla_put_failure;
2401
2402         genlmsg_end(msg, hdr);
2403         return 0;
2404
2405 nla_put_failure:
2406         genlmsg_cancel(msg, hdr);
2407         return -EMSGSIZE;
2408 }
2409
2410 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
2411                                       struct genl_info *info)
2412 {
2413         struct devlink *devlink = info->user_ptr[0];
2414         struct devlink_sb *devlink_sb;
2415         struct sk_buff *msg;
2416         int err;
2417
2418         devlink_sb = devlink_sb_get_from_info(devlink, info);
2419         if (IS_ERR(devlink_sb))
2420                 return PTR_ERR(devlink_sb);
2421
2422         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2423         if (!msg)
2424                 return -ENOMEM;
2425
2426         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2427                                  DEVLINK_CMD_SB_NEW,
2428                                  info->snd_portid, info->snd_seq, 0);
2429         if (err) {
2430                 nlmsg_free(msg);
2431                 return err;
2432         }
2433
2434         return genlmsg_reply(msg, info);
2435 }
2436
2437 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
2438                                         struct netlink_callback *cb)
2439 {
2440         struct devlink *devlink;
2441         struct devlink_sb *devlink_sb;
2442         int start = cb->args[0];
2443         unsigned long index;
2444         int idx = 0;
2445         int err;
2446
2447         mutex_lock(&devlink_mutex);
2448         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2449                 if (!devlink_try_get(devlink))
2450                         continue;
2451
2452                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2453                         goto retry;
2454
2455                 mutex_lock(&devlink->lock);
2456                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2457                         if (idx < start) {
2458                                 idx++;
2459                                 continue;
2460                         }
2461                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2462                                                  DEVLINK_CMD_SB_NEW,
2463                                                  NETLINK_CB(cb->skb).portid,
2464                                                  cb->nlh->nlmsg_seq,
2465                                                  NLM_F_MULTI);
2466                         if (err) {
2467                                 mutex_unlock(&devlink->lock);
2468                                 devlink_put(devlink);
2469                                 goto out;
2470                         }
2471                         idx++;
2472                 }
2473                 mutex_unlock(&devlink->lock);
2474 retry:
2475                 devlink_put(devlink);
2476         }
2477 out:
2478         mutex_unlock(&devlink_mutex);
2479
2480         cb->args[0] = idx;
2481         return msg->len;
2482 }
2483
2484 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
2485                                    struct devlink_sb *devlink_sb,
2486                                    u16 pool_index, enum devlink_command cmd,
2487                                    u32 portid, u32 seq, int flags)
2488 {
2489         struct devlink_sb_pool_info pool_info;
2490         void *hdr;
2491         int err;
2492
2493         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
2494                                         pool_index, &pool_info);
2495         if (err)
2496                 return err;
2497
2498         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2499         if (!hdr)
2500                 return -EMSGSIZE;
2501
2502         if (devlink_nl_put_handle(msg, devlink))
2503                 goto nla_put_failure;
2504         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2505                 goto nla_put_failure;
2506         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2507                 goto nla_put_failure;
2508         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
2509                 goto nla_put_failure;
2510         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
2511                 goto nla_put_failure;
2512         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
2513                        pool_info.threshold_type))
2514                 goto nla_put_failure;
2515         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
2516                         pool_info.cell_size))
2517                 goto nla_put_failure;
2518
2519         genlmsg_end(msg, hdr);
2520         return 0;
2521
2522 nla_put_failure:
2523         genlmsg_cancel(msg, hdr);
2524         return -EMSGSIZE;
2525 }
2526
2527 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
2528                                            struct genl_info *info)
2529 {
2530         struct devlink *devlink = info->user_ptr[0];
2531         struct devlink_sb *devlink_sb;
2532         struct sk_buff *msg;
2533         u16 pool_index;
2534         int err;
2535
2536         devlink_sb = devlink_sb_get_from_info(devlink, info);
2537         if (IS_ERR(devlink_sb))
2538                 return PTR_ERR(devlink_sb);
2539
2540         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2541                                                   &pool_index);
2542         if (err)
2543                 return err;
2544
2545         if (!devlink->ops->sb_pool_get)
2546                 return -EOPNOTSUPP;
2547
2548         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2549         if (!msg)
2550                 return -ENOMEM;
2551
2552         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
2553                                       DEVLINK_CMD_SB_POOL_NEW,
2554                                       info->snd_portid, info->snd_seq, 0);
2555         if (err) {
2556                 nlmsg_free(msg);
2557                 return err;
2558         }
2559
2560         return genlmsg_reply(msg, info);
2561 }
2562
2563 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2564                                 struct devlink *devlink,
2565                                 struct devlink_sb *devlink_sb,
2566                                 u32 portid, u32 seq)
2567 {
2568         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2569         u16 pool_index;
2570         int err;
2571
2572         for (pool_index = 0; pool_index < pool_count; pool_index++) {
2573                 if (*p_idx < start) {
2574                         (*p_idx)++;
2575                         continue;
2576                 }
2577                 err = devlink_nl_sb_pool_fill(msg, devlink,
2578                                               devlink_sb,
2579                                               pool_index,
2580                                               DEVLINK_CMD_SB_POOL_NEW,
2581                                               portid, seq, NLM_F_MULTI);
2582                 if (err)
2583                         return err;
2584                 (*p_idx)++;
2585         }
2586         return 0;
2587 }
2588
2589 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
2590                                              struct netlink_callback *cb)
2591 {
2592         struct devlink *devlink;
2593         struct devlink_sb *devlink_sb;
2594         int start = cb->args[0];
2595         unsigned long index;
2596         int idx = 0;
2597         int err = 0;
2598
2599         mutex_lock(&devlink_mutex);
2600         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2601                 if (!devlink_try_get(devlink))
2602                         continue;
2603
2604                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2605                     !devlink->ops->sb_pool_get)
2606                         goto retry;
2607
2608                 mutex_lock(&devlink->lock);
2609                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2610                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
2611                                                    devlink_sb,
2612                                                    NETLINK_CB(cb->skb).portid,
2613                                                    cb->nlh->nlmsg_seq);
2614                         if (err == -EOPNOTSUPP) {
2615                                 err = 0;
2616                         } else if (err) {
2617                                 mutex_unlock(&devlink->lock);
2618                                 devlink_put(devlink);
2619                                 goto out;
2620                         }
2621                 }
2622                 mutex_unlock(&devlink->lock);
2623 retry:
2624                 devlink_put(devlink);
2625         }
2626 out:
2627         mutex_unlock(&devlink_mutex);
2628
2629         if (err != -EMSGSIZE)
2630                 return err;
2631
2632         cb->args[0] = idx;
2633         return msg->len;
2634 }
2635
2636 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
2637                                u16 pool_index, u32 size,
2638                                enum devlink_sb_threshold_type threshold_type,
2639                                struct netlink_ext_ack *extack)
2640
2641 {
2642         const struct devlink_ops *ops = devlink->ops;
2643
2644         if (ops->sb_pool_set)
2645                 return ops->sb_pool_set(devlink, sb_index, pool_index,
2646                                         size, threshold_type, extack);
2647         return -EOPNOTSUPP;
2648 }
2649
2650 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
2651                                            struct genl_info *info)
2652 {
2653         struct devlink *devlink = info->user_ptr[0];
2654         enum devlink_sb_threshold_type threshold_type;
2655         struct devlink_sb *devlink_sb;
2656         u16 pool_index;
2657         u32 size;
2658         int err;
2659
2660         devlink_sb = devlink_sb_get_from_info(devlink, info);
2661         if (IS_ERR(devlink_sb))
2662                 return PTR_ERR(devlink_sb);
2663
2664         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2665                                                   &pool_index);
2666         if (err)
2667                 return err;
2668
2669         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
2670         if (err)
2671                 return err;
2672
2673         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
2674                 return -EINVAL;
2675
2676         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
2677         return devlink_sb_pool_set(devlink, devlink_sb->index,
2678                                    pool_index, size, threshold_type,
2679                                    info->extack);
2680 }
2681
2682 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
2683                                         struct devlink *devlink,
2684                                         struct devlink_port *devlink_port,
2685                                         struct devlink_sb *devlink_sb,
2686                                         u16 pool_index,
2687                                         enum devlink_command cmd,
2688                                         u32 portid, u32 seq, int flags)
2689 {
2690         const struct devlink_ops *ops = devlink->ops;
2691         u32 threshold;
2692         void *hdr;
2693         int err;
2694
2695         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
2696                                     pool_index, &threshold);
2697         if (err)
2698                 return err;
2699
2700         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2701         if (!hdr)
2702                 return -EMSGSIZE;
2703
2704         if (devlink_nl_put_handle(msg, devlink))
2705                 goto nla_put_failure;
2706         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2707                 goto nla_put_failure;
2708         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2709                 goto nla_put_failure;
2710         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2711                 goto nla_put_failure;
2712         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2713                 goto nla_put_failure;
2714
2715         if (ops->sb_occ_port_pool_get) {
2716                 u32 cur;
2717                 u32 max;
2718
2719                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
2720                                                 pool_index, &cur, &max);
2721                 if (err && err != -EOPNOTSUPP)
2722                         goto sb_occ_get_failure;
2723                 if (!err) {
2724                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2725                                 goto nla_put_failure;
2726                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2727                                 goto nla_put_failure;
2728                 }
2729         }
2730
2731         genlmsg_end(msg, hdr);
2732         return 0;
2733
2734 nla_put_failure:
2735         err = -EMSGSIZE;
2736 sb_occ_get_failure:
2737         genlmsg_cancel(msg, hdr);
2738         return err;
2739 }
2740
2741 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
2742                                                 struct genl_info *info)
2743 {
2744         struct devlink_port *devlink_port = info->user_ptr[1];
2745         struct devlink *devlink = devlink_port->devlink;
2746         struct devlink_sb *devlink_sb;
2747         struct sk_buff *msg;
2748         u16 pool_index;
2749         int err;
2750
2751         devlink_sb = devlink_sb_get_from_info(devlink, info);
2752         if (IS_ERR(devlink_sb))
2753                 return PTR_ERR(devlink_sb);
2754
2755         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2756                                                   &pool_index);
2757         if (err)
2758                 return err;
2759
2760         if (!devlink->ops->sb_port_pool_get)
2761                 return -EOPNOTSUPP;
2762
2763         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2764         if (!msg)
2765                 return -ENOMEM;
2766
2767         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
2768                                            devlink_sb, pool_index,
2769                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2770                                            info->snd_portid, info->snd_seq, 0);
2771         if (err) {
2772                 nlmsg_free(msg);
2773                 return err;
2774         }
2775
2776         return genlmsg_reply(msg, info);
2777 }
2778
2779 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2780                                      struct devlink *devlink,
2781                                      struct devlink_sb *devlink_sb,
2782                                      u32 portid, u32 seq)
2783 {
2784         struct devlink_port *devlink_port;
2785         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2786         u16 pool_index;
2787         int err;
2788
2789         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2790                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
2791                         if (*p_idx < start) {
2792                                 (*p_idx)++;
2793                                 continue;
2794                         }
2795                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
2796                                                            devlink_port,
2797                                                            devlink_sb,
2798                                                            pool_index,
2799                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2800                                                            portid, seq,
2801                                                            NLM_F_MULTI);
2802                         if (err)
2803                                 return err;
2804                         (*p_idx)++;
2805                 }
2806         }
2807         return 0;
2808 }
2809
2810 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
2811                                                   struct netlink_callback *cb)
2812 {
2813         struct devlink *devlink;
2814         struct devlink_sb *devlink_sb;
2815         int start = cb->args[0];
2816         unsigned long index;
2817         int idx = 0;
2818         int err = 0;
2819
2820         mutex_lock(&devlink_mutex);
2821         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2822                 if (!devlink_try_get(devlink))
2823                         continue;
2824
2825                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2826                     !devlink->ops->sb_port_pool_get)
2827                         goto retry;
2828
2829                 mutex_lock(&devlink->lock);
2830                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2831                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
2832                                                         devlink, devlink_sb,
2833                                                         NETLINK_CB(cb->skb).portid,
2834                                                         cb->nlh->nlmsg_seq);
2835                         if (err == -EOPNOTSUPP) {
2836                                 err = 0;
2837                         } else if (err) {
2838                                 mutex_unlock(&devlink->lock);
2839                                 devlink_put(devlink);
2840                                 goto out;
2841                         }
2842                 }
2843                 mutex_unlock(&devlink->lock);
2844 retry:
2845                 devlink_put(devlink);
2846         }
2847 out:
2848         mutex_unlock(&devlink_mutex);
2849
2850         if (err != -EMSGSIZE)
2851                 return err;
2852
2853         cb->args[0] = idx;
2854         return msg->len;
2855 }
2856
2857 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
2858                                     unsigned int sb_index, u16 pool_index,
2859                                     u32 threshold,
2860                                     struct netlink_ext_ack *extack)
2861
2862 {
2863         const struct devlink_ops *ops = devlink_port->devlink->ops;
2864
2865         if (ops->sb_port_pool_set)
2866                 return ops->sb_port_pool_set(devlink_port, sb_index,
2867                                              pool_index, threshold, extack);
2868         return -EOPNOTSUPP;
2869 }
2870
2871 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
2872                                                 struct genl_info *info)
2873 {
2874         struct devlink_port *devlink_port = info->user_ptr[1];
2875         struct devlink *devlink = info->user_ptr[0];
2876         struct devlink_sb *devlink_sb;
2877         u16 pool_index;
2878         u32 threshold;
2879         int err;
2880
2881         devlink_sb = devlink_sb_get_from_info(devlink, info);
2882         if (IS_ERR(devlink_sb))
2883                 return PTR_ERR(devlink_sb);
2884
2885         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2886                                                   &pool_index);
2887         if (err)
2888                 return err;
2889
2890         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2891                 return -EINVAL;
2892
2893         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2894         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
2895                                         pool_index, threshold, info->extack);
2896 }
2897
2898 static int
2899 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
2900                                 struct devlink_port *devlink_port,
2901                                 struct devlink_sb *devlink_sb, u16 tc_index,
2902                                 enum devlink_sb_pool_type pool_type,
2903                                 enum devlink_command cmd,
2904                                 u32 portid, u32 seq, int flags)
2905 {
2906         const struct devlink_ops *ops = devlink->ops;
2907         u16 pool_index;
2908         u32 threshold;
2909         void *hdr;
2910         int err;
2911
2912         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
2913                                        tc_index, pool_type,
2914                                        &pool_index, &threshold);
2915         if (err)
2916                 return err;
2917
2918         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2919         if (!hdr)
2920                 return -EMSGSIZE;
2921
2922         if (devlink_nl_put_handle(msg, devlink))
2923                 goto nla_put_failure;
2924         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2925                 goto nla_put_failure;
2926         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2927                 goto nla_put_failure;
2928         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
2929                 goto nla_put_failure;
2930         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
2931                 goto nla_put_failure;
2932         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2933                 goto nla_put_failure;
2934         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2935                 goto nla_put_failure;
2936
2937         if (ops->sb_occ_tc_port_bind_get) {
2938                 u32 cur;
2939                 u32 max;
2940
2941                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
2942                                                    devlink_sb->index,
2943                                                    tc_index, pool_type,
2944                                                    &cur, &max);
2945                 if (err && err != -EOPNOTSUPP)
2946                         return err;
2947                 if (!err) {
2948                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2949                                 goto nla_put_failure;
2950                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2951                                 goto nla_put_failure;
2952                 }
2953         }
2954
2955         genlmsg_end(msg, hdr);
2956         return 0;
2957
2958 nla_put_failure:
2959         genlmsg_cancel(msg, hdr);
2960         return -EMSGSIZE;
2961 }
2962
2963 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
2964                                                    struct genl_info *info)
2965 {
2966         struct devlink_port *devlink_port = info->user_ptr[1];
2967         struct devlink *devlink = devlink_port->devlink;
2968         struct devlink_sb *devlink_sb;
2969         struct sk_buff *msg;
2970         enum devlink_sb_pool_type pool_type;
2971         u16 tc_index;
2972         int err;
2973
2974         devlink_sb = devlink_sb_get_from_info(devlink, info);
2975         if (IS_ERR(devlink_sb))
2976                 return PTR_ERR(devlink_sb);
2977
2978         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2979         if (err)
2980                 return err;
2981
2982         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2983                                                 pool_type, &tc_index);
2984         if (err)
2985                 return err;
2986
2987         if (!devlink->ops->sb_tc_pool_bind_get)
2988                 return -EOPNOTSUPP;
2989
2990         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2991         if (!msg)
2992                 return -ENOMEM;
2993
2994         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
2995                                               devlink_sb, tc_index, pool_type,
2996                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2997                                               info->snd_portid,
2998                                               info->snd_seq, 0);
2999         if (err) {
3000                 nlmsg_free(msg);
3001                 return err;
3002         }
3003
3004         return genlmsg_reply(msg, info);
3005 }
3006
3007 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
3008                                         int start, int *p_idx,
3009                                         struct devlink *devlink,
3010                                         struct devlink_sb *devlink_sb,
3011                                         u32 portid, u32 seq)
3012 {
3013         struct devlink_port *devlink_port;
3014         u16 tc_index;
3015         int err;
3016
3017         list_for_each_entry(devlink_port, &devlink->port_list, list) {
3018                 for (tc_index = 0;
3019                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
3020                         if (*p_idx < start) {
3021                                 (*p_idx)++;
3022                                 continue;
3023                         }
3024                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
3025                                                               devlink_port,
3026                                                               devlink_sb,
3027                                                               tc_index,
3028                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
3029                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
3030                                                               portid, seq,
3031                                                               NLM_F_MULTI);
3032                         if (err)
3033                                 return err;
3034                         (*p_idx)++;
3035                 }
3036                 for (tc_index = 0;
3037                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
3038                         if (*p_idx < start) {
3039                                 (*p_idx)++;
3040                                 continue;
3041                         }
3042                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
3043                                                               devlink_port,
3044                                                               devlink_sb,
3045                                                               tc_index,
3046                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
3047                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
3048                                                               portid, seq,
3049                                                               NLM_F_MULTI);
3050                         if (err)
3051                                 return err;
3052                         (*p_idx)++;
3053                 }
3054         }
3055         return 0;
3056 }
3057
3058 static int
3059 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
3060                                           struct netlink_callback *cb)
3061 {
3062         struct devlink *devlink;
3063         struct devlink_sb *devlink_sb;
3064         int start = cb->args[0];
3065         unsigned long index;
3066         int idx = 0;
3067         int err = 0;
3068
3069         mutex_lock(&devlink_mutex);
3070         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
3071                 if (!devlink_try_get(devlink))
3072                         continue;
3073
3074                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
3075                     !devlink->ops->sb_tc_pool_bind_get)
3076                         goto retry;
3077
3078                 mutex_lock(&devlink->lock);
3079                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
3080                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
3081                                                            devlink,
3082                                                            devlink_sb,
3083                                                            NETLINK_CB(cb->skb).portid,
3084                                                            cb->nlh->nlmsg_seq);
3085                         if (err == -EOPNOTSUPP) {
3086                                 err = 0;
3087                         } else if (err) {
3088                                 mutex_unlock(&devlink->lock);
3089                                 devlink_put(devlink);
3090                                 goto out;
3091                         }
3092                 }
3093                 mutex_unlock(&devlink->lock);
3094 retry:
3095                 devlink_put(devlink);
3096         }
3097 out:
3098         mutex_unlock(&devlink_mutex);
3099
3100         if (err != -EMSGSIZE)
3101                 return err;
3102
3103         cb->args[0] = idx;
3104         return msg->len;
3105 }
3106
3107 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
3108                                        unsigned int sb_index, u16 tc_index,
3109                                        enum devlink_sb_pool_type pool_type,
3110                                        u16 pool_index, u32 threshold,
3111                                        struct netlink_ext_ack *extack)
3112
3113 {
3114         const struct devlink_ops *ops = devlink_port->devlink->ops;
3115
3116         if (ops->sb_tc_pool_bind_set)
3117                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
3118                                                 tc_index, pool_type,
3119                                                 pool_index, threshold, extack);
3120         return -EOPNOTSUPP;
3121 }
3122
3123 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
3124                                                    struct genl_info *info)
3125 {
3126         struct devlink_port *devlink_port = info->user_ptr[1];
3127         struct devlink *devlink = info->user_ptr[0];
3128         enum devlink_sb_pool_type pool_type;
3129         struct devlink_sb *devlink_sb;
3130         u16 tc_index;
3131         u16 pool_index;
3132         u32 threshold;
3133         int err;
3134
3135         devlink_sb = devlink_sb_get_from_info(devlink, info);
3136         if (IS_ERR(devlink_sb))
3137                 return PTR_ERR(devlink_sb);
3138
3139         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
3140         if (err)
3141                 return err;
3142
3143         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
3144                                                 pool_type, &tc_index);
3145         if (err)
3146                 return err;
3147
3148         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
3149                                                   &pool_index);
3150         if (err)
3151                 return err;
3152
3153         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
3154                 return -EINVAL;
3155
3156         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
3157         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
3158                                            tc_index, pool_type,
3159                                            pool_index, threshold, info->extack);
3160 }
3161
3162 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
3163                                                struct genl_info *info)
3164 {
3165         struct devlink *devlink = info->user_ptr[0];
3166         const struct devlink_ops *ops = devlink->ops;
3167         struct devlink_sb *devlink_sb;
3168
3169         devlink_sb = devlink_sb_get_from_info(devlink, info);
3170         if (IS_ERR(devlink_sb))
3171                 return PTR_ERR(devlink_sb);
3172
3173         if (ops->sb_occ_snapshot)
3174                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
3175         return -EOPNOTSUPP;
3176 }
3177
3178 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
3179                                                 struct genl_info *info)
3180 {
3181         struct devlink *devlink = info->user_ptr[0];
3182         const struct devlink_ops *ops = devlink->ops;
3183         struct devlink_sb *devlink_sb;
3184
3185         devlink_sb = devlink_sb_get_from_info(devlink, info);
3186         if (IS_ERR(devlink_sb))
3187                 return PTR_ERR(devlink_sb);
3188
3189         if (ops->sb_occ_max_clear)
3190                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
3191         return -EOPNOTSUPP;
3192 }
3193
3194 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
3195                                    enum devlink_command cmd, u32 portid,
3196                                    u32 seq, int flags)
3197 {
3198         const struct devlink_ops *ops = devlink->ops;
3199         enum devlink_eswitch_encap_mode encap_mode;
3200         u8 inline_mode;
3201         void *hdr;
3202         int err = 0;
3203         u16 mode;
3204
3205         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
3206         if (!hdr)
3207                 return -EMSGSIZE;
3208
3209         err = devlink_nl_put_handle(msg, devlink);
3210         if (err)
3211                 goto nla_put_failure;
3212
3213         if (ops->eswitch_mode_get) {
3214                 err = ops->eswitch_mode_get(devlink, &mode);
3215                 if (err)
3216                         goto nla_put_failure;
3217                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
3218                 if (err)
3219                         goto nla_put_failure;
3220         }
3221
3222         if (ops->eswitch_inline_mode_get) {
3223                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
3224                 if (err)
3225                         goto nla_put_failure;
3226                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
3227                                  inline_mode);
3228                 if (err)
3229                         goto nla_put_failure;
3230         }
3231
3232         if (ops->eswitch_encap_mode_get) {
3233                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
3234                 if (err)
3235                         goto nla_put_failure;
3236                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
3237                 if (err)
3238                         goto nla_put_failure;
3239         }
3240
3241         genlmsg_end(msg, hdr);
3242         return 0;
3243
3244 nla_put_failure:
3245         genlmsg_cancel(msg, hdr);
3246         return err;
3247 }
3248
3249 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
3250                                            struct genl_info *info)
3251 {
3252         struct devlink *devlink = info->user_ptr[0];
3253         struct sk_buff *msg;
3254         int err;
3255
3256         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3257         if (!msg)
3258                 return -ENOMEM;
3259
3260         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
3261                                       info->snd_portid, info->snd_seq, 0);
3262
3263         if (err) {
3264                 nlmsg_free(msg);
3265                 return err;
3266         }
3267
3268         return genlmsg_reply(msg, info);
3269 }
3270
3271 static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
3272                                     struct netlink_ext_ack *extack)
3273 {
3274         struct devlink_rate *devlink_rate;
3275
3276         list_for_each_entry(devlink_rate, &devlink->rate_list, list)
3277                 if (devlink_rate_is_node(devlink_rate)) {
3278                         NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
3279                         return -EBUSY;
3280                 }
3281         return 0;
3282 }
3283
3284 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
3285                                            struct genl_info *info)
3286 {
3287         struct devlink *devlink = info->user_ptr[0];
3288         const struct devlink_ops *ops = devlink->ops;
3289         enum devlink_eswitch_encap_mode encap_mode;
3290         u8 inline_mode;
3291         int err = 0;
3292         u16 mode;
3293
3294         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
3295                 if (!ops->eswitch_mode_set)
3296                         return -EOPNOTSUPP;
3297                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
3298                 err = devlink_rate_nodes_check(devlink, mode, info->extack);
3299                 if (err)
3300                         return err;
3301                 err = ops->eswitch_mode_set(devlink, mode, info->extack);
3302                 if (err)
3303                         return err;
3304         }
3305
3306         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
3307                 if (!ops->eswitch_inline_mode_set)
3308                         return -EOPNOTSUPP;
3309                 inline_mode = nla_get_u8(
3310                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
3311                 err = ops->eswitch_inline_mode_set(devlink, inline_mode,
3312                                                    info->extack);
3313                 if (err)
3314                         return err;
3315         }
3316
3317         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
3318                 if (!ops->eswitch_encap_mode_set)
3319                         return -EOPNOTSUPP;
3320                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
3321                 err = ops->eswitch_encap_mode_set(devlink, encap_mode,
3322                                                   info->extack);
3323                 if (err)
3324                         return err;
3325         }
3326
3327         return 0;
3328 }
3329
3330 int devlink_dpipe_match_put(struct sk_buff *skb,
3331                             struct devlink_dpipe_match *match)
3332 {
3333         struct devlink_dpipe_header *header = match->header;
3334         struct devlink_dpipe_field *field = &header->fields[match->field_id];
3335         struct nlattr *match_attr;
3336
3337         match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
3338         if (!match_attr)
3339                 return -EMSGSIZE;
3340
3341         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
3342             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
3343             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3344             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3345             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3346                 goto nla_put_failure;
3347
3348         nla_nest_end(skb, match_attr);
3349         return 0;
3350
3351 nla_put_failure:
3352         nla_nest_cancel(skb, match_attr);
3353         return -EMSGSIZE;
3354 }
3355 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
3356
3357 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
3358                                      struct sk_buff *skb)
3359 {
3360         struct nlattr *matches_attr;
3361
3362         matches_attr = nla_nest_start_noflag(skb,
3363                                              DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
3364         if (!matches_attr)
3365                 return -EMSGSIZE;
3366
3367         if (table->table_ops->matches_dump(table->priv, skb))
3368                 goto nla_put_failure;
3369
3370         nla_nest_end(skb, matches_attr);
3371         return 0;
3372
3373 nla_put_failure:
3374         nla_nest_cancel(skb, matches_attr);
3375         return -EMSGSIZE;
3376 }
3377
3378 int devlink_dpipe_action_put(struct sk_buff *skb,
3379                              struct devlink_dpipe_action *action)
3380 {
3381         struct devlink_dpipe_header *header = action->header;
3382         struct devlink_dpipe_field *field = &header->fields[action->field_id];
3383         struct nlattr *action_attr;
3384
3385         action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
3386         if (!action_attr)
3387                 return -EMSGSIZE;
3388
3389         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
3390             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
3391             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3392             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3393             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3394                 goto nla_put_failure;
3395
3396         nla_nest_end(skb, action_attr);
3397         return 0;
3398
3399 nla_put_failure:
3400         nla_nest_cancel(skb, action_attr);
3401         return -EMSGSIZE;
3402 }
3403 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
3404
3405 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
3406                                      struct sk_buff *skb)
3407 {
3408         struct nlattr *actions_attr;
3409
3410         actions_attr = nla_nest_start_noflag(skb,
3411                                              DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
3412         if (!actions_attr)
3413                 return -EMSGSIZE;
3414
3415         if (table->table_ops->actions_dump(table->priv, skb))
3416                 goto nla_put_failure;
3417
3418         nla_nest_end(skb, actions_attr);
3419         return 0;
3420
3421 nla_put_failure:
3422         nla_nest_cancel(skb, actions_attr);
3423         return -EMSGSIZE;
3424 }
3425
3426 static int devlink_dpipe_table_put(struct sk_buff *skb,
3427                                    struct devlink_dpipe_table *table)
3428 {
3429         struct nlattr *table_attr;
3430         u64 table_size;
3431
3432         table_size = table->table_ops->size_get(table->priv);
3433         table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
3434         if (!table_attr)
3435                 return -EMSGSIZE;
3436
3437         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
3438             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
3439                               DEVLINK_ATTR_PAD))
3440                 goto nla_put_failure;
3441         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
3442                        table->counters_enabled))
3443                 goto nla_put_failure;
3444
3445         if (table->resource_valid) {
3446                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
3447                                       table->resource_id, DEVLINK_ATTR_PAD) ||
3448                     nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
3449                                       table->resource_units, DEVLINK_ATTR_PAD))
3450                         goto nla_put_failure;
3451         }
3452         if (devlink_dpipe_matches_put(table, skb))
3453                 goto nla_put_failure;
3454
3455         if (devlink_dpipe_actions_put(table, skb))
3456                 goto nla_put_failure;
3457
3458         nla_nest_end(skb, table_attr);
3459         return 0;
3460
3461 nla_put_failure:
3462         nla_nest_cancel(skb, table_attr);
3463         return -EMSGSIZE;
3464 }
3465
3466 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
3467                                             struct genl_info *info)
3468 {
3469         int err;
3470
3471         if (*pskb) {
3472                 err = genlmsg_reply(*pskb, info);
3473                 if (err)
3474                         return err;
3475         }
3476         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
3477         if (!*pskb)
3478                 return -ENOMEM;
3479         return 0;
3480 }
3481
3482 static int devlink_dpipe_tables_fill(struct genl_info *info,
3483                                      enum devlink_command cmd, int flags,
3484                                      struct list_head *dpipe_tables,
3485                                      const char *table_name)
3486 {
3487         struct devlink *devlink = info->user_ptr[0];
3488         struct devlink_dpipe_table *table;
3489         struct nlattr *tables_attr;
3490         struct sk_buff *skb = NULL;
3491         struct nlmsghdr *nlh;
3492         bool incomplete;
3493         void *hdr;
3494         int i;
3495         int err;
3496
3497         table = list_first_entry(dpipe_tables,
3498                                  struct devlink_dpipe_table, list);
3499 start_again:
3500         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3501         if (err)
3502                 return err;
3503
3504         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3505                           &devlink_nl_family, NLM_F_MULTI, cmd);
3506         if (!hdr) {
3507                 nlmsg_free(skb);
3508                 return -EMSGSIZE;
3509         }
3510
3511         if (devlink_nl_put_handle(skb, devlink))
3512                 goto nla_put_failure;
3513         tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
3514         if (!tables_attr)
3515                 goto nla_put_failure;
3516
3517         i = 0;
3518         incomplete = false;
3519         list_for_each_entry_from(table, dpipe_tables, list) {
3520                 if (!table_name) {
3521                         err = devlink_dpipe_table_put(skb, table);
3522                         if (err) {
3523                                 if (!i)
3524                                         goto err_table_put;
3525                                 incomplete = true;
3526                                 break;
3527                         }
3528                 } else {
3529                         if (!strcmp(table->name, table_name)) {
3530                                 err = devlink_dpipe_table_put(skb, table);
3531                                 if (err)
3532                                         break;
3533                         }
3534                 }
3535                 i++;
3536         }
3537
3538         nla_nest_end(skb, tables_attr);
3539         genlmsg_end(skb, hdr);
3540         if (incomplete)
3541                 goto start_again;
3542
3543 send_done:
3544         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3545                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3546         if (!nlh) {
3547                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3548                 if (err)
3549                         return err;
3550                 goto send_done;
3551         }
3552
3553         return genlmsg_reply(skb, info);
3554
3555 nla_put_failure:
3556         err = -EMSGSIZE;
3557 err_table_put:
3558         nlmsg_free(skb);
3559         return err;
3560 }
3561
3562 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
3563                                           struct genl_info *info)
3564 {
3565         struct devlink *devlink = info->user_ptr[0];
3566         const char *table_name =  NULL;
3567
3568         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3569                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3570
3571         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
3572                                          &devlink->dpipe_table_list,
3573                                          table_name);
3574 }
3575
3576 static int devlink_dpipe_value_put(struct sk_buff *skb,
3577                                    struct devlink_dpipe_value *value)
3578 {
3579         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
3580                     value->value_size, value->value))
3581                 return -EMSGSIZE;
3582         if (value->mask)
3583                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
3584                             value->value_size, value->mask))
3585                         return -EMSGSIZE;
3586         if (value->mapping_valid)
3587                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
3588                                 value->mapping_value))
3589                         return -EMSGSIZE;
3590         return 0;
3591 }
3592
3593 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
3594                                           struct devlink_dpipe_value *value)
3595 {
3596         if (!value->action)
3597                 return -EINVAL;
3598         if (devlink_dpipe_action_put(skb, value->action))
3599                 return -EMSGSIZE;
3600         if (devlink_dpipe_value_put(skb, value))
3601                 return -EMSGSIZE;
3602         return 0;
3603 }
3604
3605 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
3606                                            struct devlink_dpipe_value *values,
3607                                            unsigned int values_count)
3608 {
3609         struct nlattr *action_attr;
3610         int i;
3611         int err;
3612
3613         for (i = 0; i < values_count; i++) {
3614                 action_attr = nla_nest_start_noflag(skb,
3615                                                     DEVLINK_ATTR_DPIPE_ACTION_VALUE);
3616                 if (!action_attr)
3617                         return -EMSGSIZE;
3618                 err = devlink_dpipe_action_value_put(skb, &values[i]);
3619                 if (err)
3620                         goto err_action_value_put;
3621                 nla_nest_end(skb, action_attr);
3622         }
3623         return 0;
3624
3625 err_action_value_put:
3626         nla_nest_cancel(skb, action_attr);
3627         return err;
3628 }
3629
3630 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
3631                                          struct devlink_dpipe_value *value)
3632 {
3633         if (!value->match)
3634                 return -EINVAL;
3635         if (devlink_dpipe_match_put(skb, value->match))
3636                 return -EMSGSIZE;
3637         if (devlink_dpipe_value_put(skb, value))
3638                 return -EMSGSIZE;
3639         return 0;
3640 }
3641
3642 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
3643                                           struct devlink_dpipe_value *values,
3644                                           unsigned int values_count)
3645 {
3646         struct nlattr *match_attr;
3647         int i;
3648         int err;
3649
3650         for (i = 0; i < values_count; i++) {
3651                 match_attr = nla_nest_start_noflag(skb,
3652                                                    DEVLINK_ATTR_DPIPE_MATCH_VALUE);
3653                 if (!match_attr)
3654                         return -EMSGSIZE;
3655                 err = devlink_dpipe_match_value_put(skb, &values[i]);
3656                 if (err)
3657                         goto err_match_value_put;
3658                 nla_nest_end(skb, match_attr);
3659         }
3660         return 0;
3661
3662 err_match_value_put:
3663         nla_nest_cancel(skb, match_attr);
3664         return err;
3665 }
3666
3667 static int devlink_dpipe_entry_put(struct sk_buff *skb,
3668                                    struct devlink_dpipe_entry *entry)
3669 {
3670         struct nlattr *entry_attr, *matches_attr, *actions_attr;
3671         int err;
3672
3673         entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
3674         if (!entry_attr)
3675                 return  -EMSGSIZE;
3676
3677         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
3678                               DEVLINK_ATTR_PAD))
3679                 goto nla_put_failure;
3680         if (entry->counter_valid)
3681                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
3682                                       entry->counter, DEVLINK_ATTR_PAD))
3683                         goto nla_put_failure;
3684
3685         matches_attr = nla_nest_start_noflag(skb,
3686                                              DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
3687         if (!matches_attr)
3688                 goto nla_put_failure;
3689
3690         err = devlink_dpipe_match_values_put(skb, entry->match_values,
3691                                              entry->match_values_count);
3692         if (err) {
3693                 nla_nest_cancel(skb, matches_attr);
3694                 goto err_match_values_put;
3695         }
3696         nla_nest_end(skb, matches_attr);
3697
3698         actions_attr = nla_nest_start_noflag(skb,
3699                                              DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
3700         if (!actions_attr)
3701                 goto nla_put_failure;
3702
3703         err = devlink_dpipe_action_values_put(skb, entry->action_values,
3704                                               entry->action_values_count);
3705         if (err) {
3706                 nla_nest_cancel(skb, actions_attr);
3707                 goto err_action_values_put;
3708         }
3709         nla_nest_end(skb, actions_attr);
3710
3711         nla_nest_end(skb, entry_attr);
3712         return 0;
3713
3714 nla_put_failure:
3715         err = -EMSGSIZE;
3716 err_match_values_put:
3717 err_action_values_put:
3718         nla_nest_cancel(skb, entry_attr);
3719         return err;
3720 }
3721
3722 static struct devlink_dpipe_table *
3723 devlink_dpipe_table_find(struct list_head *dpipe_tables,
3724                          const char *table_name, struct devlink *devlink)
3725 {
3726         struct devlink_dpipe_table *table;
3727         list_for_each_entry_rcu(table, dpipe_tables, list,
3728                                 lockdep_is_held(&devlink->lock)) {
3729                 if (!strcmp(table->name, table_name))
3730                         return table;
3731         }
3732         return NULL;
3733 }
3734
3735 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
3736 {
3737         struct devlink *devlink;
3738         int err;
3739
3740         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
3741                                                dump_ctx->info);
3742         if (err)
3743                 return err;
3744
3745         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
3746                                     dump_ctx->info->snd_portid,
3747                                     dump_ctx->info->snd_seq,
3748                                     &devlink_nl_family, NLM_F_MULTI,
3749                                     dump_ctx->cmd);
3750         if (!dump_ctx->hdr)
3751                 goto nla_put_failure;
3752
3753         devlink = dump_ctx->info->user_ptr[0];
3754         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
3755                 goto nla_put_failure;
3756         dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
3757                                                DEVLINK_ATTR_DPIPE_ENTRIES);
3758         if (!dump_ctx->nest)
3759                 goto nla_put_failure;
3760         return 0;
3761
3762 nla_put_failure:
3763         nlmsg_free(dump_ctx->skb);
3764         return -EMSGSIZE;
3765 }
3766 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
3767
3768 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
3769                                    struct devlink_dpipe_entry *entry)
3770 {
3771         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
3772 }
3773 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
3774
3775 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
3776 {
3777         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
3778         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
3779         return 0;
3780 }
3781 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
3782
3783 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
3784
3785 {
3786         unsigned int value_count, value_index;
3787         struct devlink_dpipe_value *value;
3788
3789         value = entry->action_values;
3790         value_count = entry->action_values_count;
3791         for (value_index = 0; value_index < value_count; value_index++) {
3792                 kfree(value[value_index].value);
3793                 kfree(value[value_index].mask);
3794         }
3795
3796         value = entry->match_values;
3797         value_count = entry->match_values_count;
3798         for (value_index = 0; value_index < value_count; value_index++) {
3799                 kfree(value[value_index].value);
3800                 kfree(value[value_index].mask);
3801         }
3802 }
3803 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
3804
3805 static int devlink_dpipe_entries_fill(struct genl_info *info,
3806                                       enum devlink_command cmd, int flags,
3807                                       struct devlink_dpipe_table *table)
3808 {
3809         struct devlink_dpipe_dump_ctx dump_ctx;
3810         struct nlmsghdr *nlh;
3811         int err;
3812
3813         dump_ctx.skb = NULL;
3814         dump_ctx.cmd = cmd;
3815         dump_ctx.info = info;
3816
3817         err = table->table_ops->entries_dump(table->priv,
3818                                              table->counters_enabled,
3819                                              &dump_ctx);
3820         if (err)
3821                 return err;
3822
3823 send_done:
3824         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
3825                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3826         if (!nlh) {
3827                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
3828                 if (err)
3829                         return err;
3830                 goto send_done;
3831         }
3832         return genlmsg_reply(dump_ctx.skb, info);
3833 }
3834
3835 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
3836                                             struct genl_info *info)
3837 {
3838         struct devlink *devlink = info->user_ptr[0];
3839         struct devlink_dpipe_table *table;
3840         const char *table_name;
3841
3842         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3843                 return -EINVAL;
3844
3845         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3846         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3847                                          table_name, devlink);
3848         if (!table)
3849                 return -EINVAL;
3850
3851         if (!table->table_ops->entries_dump)
3852                 return -EINVAL;
3853
3854         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
3855                                           0, table);
3856 }
3857
3858 static int devlink_dpipe_fields_put(struct sk_buff *skb,
3859                                     const struct devlink_dpipe_header *header)
3860 {
3861         struct devlink_dpipe_field *field;
3862         struct nlattr *field_attr;
3863         int i;
3864
3865         for (i = 0; i < header->fields_count; i++) {
3866                 field = &header->fields[i];
3867                 field_attr = nla_nest_start_noflag(skb,
3868                                                    DEVLINK_ATTR_DPIPE_FIELD);
3869                 if (!field_attr)
3870                         return -EMSGSIZE;
3871                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
3872                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3873                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
3874                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
3875                         goto nla_put_failure;
3876                 nla_nest_end(skb, field_attr);
3877         }
3878         return 0;
3879
3880 nla_put_failure:
3881         nla_nest_cancel(skb, field_attr);
3882         return -EMSGSIZE;
3883 }
3884
3885 static int devlink_dpipe_header_put(struct sk_buff *skb,
3886                                     struct devlink_dpipe_header *header)
3887 {
3888         struct nlattr *fields_attr, *header_attr;
3889         int err;
3890
3891         header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
3892         if (!header_attr)
3893                 return -EMSGSIZE;
3894
3895         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
3896             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3897             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3898                 goto nla_put_failure;
3899
3900         fields_attr = nla_nest_start_noflag(skb,
3901                                             DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
3902         if (!fields_attr)
3903                 goto nla_put_failure;
3904
3905         err = devlink_dpipe_fields_put(skb, header);
3906         if (err) {
3907                 nla_nest_cancel(skb, fields_attr);
3908                 goto nla_put_failure;
3909         }
3910         nla_nest_end(skb, fields_attr);
3911         nla_nest_end(skb, header_attr);
3912         return 0;
3913
3914 nla_put_failure:
3915         err = -EMSGSIZE;
3916         nla_nest_cancel(skb, header_attr);
3917         return err;
3918 }
3919
3920 static int devlink_dpipe_headers_fill(struct genl_info *info,
3921                                       enum devlink_command cmd, int flags,
3922                                       struct devlink_dpipe_headers *
3923                                       dpipe_headers)
3924 {
3925         struct devlink *devlink = info->user_ptr[0];
3926         struct nlattr *headers_attr;
3927         struct sk_buff *skb = NULL;
3928         struct nlmsghdr *nlh;
3929         void *hdr;
3930         int i, j;
3931         int err;
3932
3933         i = 0;
3934 start_again:
3935         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3936         if (err)
3937                 return err;
3938
3939         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3940                           &devlink_nl_family, NLM_F_MULTI, cmd);
3941         if (!hdr) {
3942                 nlmsg_free(skb);
3943                 return -EMSGSIZE;
3944         }
3945
3946         if (devlink_nl_put_handle(skb, devlink))
3947                 goto nla_put_failure;
3948         headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
3949         if (!headers_attr)
3950                 goto nla_put_failure;
3951
3952         j = 0;
3953         for (; i < dpipe_headers->headers_count; i++) {
3954                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
3955                 if (err) {
3956                         if (!j)
3957                                 goto err_table_put;
3958                         break;
3959                 }
3960                 j++;
3961         }
3962         nla_nest_end(skb, headers_attr);
3963         genlmsg_end(skb, hdr);
3964         if (i != dpipe_headers->headers_count)
3965                 goto start_again;
3966
3967 send_done:
3968         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3969                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3970         if (!nlh) {
3971                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3972                 if (err)
3973                         return err;
3974                 goto send_done;
3975         }
3976         return genlmsg_reply(skb, info);
3977
3978 nla_put_failure:
3979         err = -EMSGSIZE;
3980 err_table_put:
3981         nlmsg_free(skb);
3982         return err;
3983 }
3984
3985 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
3986                                             struct genl_info *info)
3987 {
3988         struct devlink *devlink = info->user_ptr[0];
3989
3990         if (!devlink->dpipe_headers)
3991                 return -EOPNOTSUPP;
3992         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
3993                                           0, devlink->dpipe_headers);
3994 }
3995
3996 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
3997                                             const char *table_name,
3998                                             bool enable)
3999 {
4000         struct devlink_dpipe_table *table;
4001
4002         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
4003                                          table_name, devlink);
4004         if (!table)
4005                 return -EINVAL;
4006
4007         if (table->counter_control_extern)
4008                 return -EOPNOTSUPP;
4009
4010         if (!(table->counters_enabled ^ enable))
4011                 return 0;
4012
4013         table->counters_enabled = enable;
4014         if (table->table_ops->counters_set_update)
4015                 table->table_ops->counters_set_update(table->priv, enable);
4016         return 0;
4017 }
4018
4019 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
4020                                                    struct genl_info *info)
4021 {
4022         struct devlink *devlink = info->user_ptr[0];
4023         const char *table_name;
4024         bool counters_enable;
4025
4026         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
4027             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
4028                 return -EINVAL;
4029
4030         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
4031         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
4032
4033         return devlink_dpipe_table_counters_set(devlink, table_name,
4034                                                 counters_enable);
4035 }
4036
4037 static struct devlink_resource *
4038 devlink_resource_find(struct devlink *devlink,
4039                       struct devlink_resource *resource, u64 resource_id)
4040 {
4041         struct list_head *resource_list;
4042
4043         if (resource)
4044                 resource_list = &resource->resource_list;
4045         else
4046                 resource_list = &devlink->resource_list;
4047
4048         list_for_each_entry(resource, resource_list, list) {
4049                 struct devlink_resource *child_resource;
4050
4051                 if (resource->id == resource_id)
4052                         return resource;
4053
4054                 child_resource = devlink_resource_find(devlink, resource,
4055                                                        resource_id);
4056                 if (child_resource)
4057                         return child_resource;
4058         }
4059         return NULL;
4060 }
4061
4062 static void
4063 devlink_resource_validate_children(struct devlink_resource *resource)
4064 {
4065         struct devlink_resource *child_resource;
4066         bool size_valid = true;
4067         u64 parts_size = 0;
4068
4069         if (list_empty(&resource->resource_list))
4070                 goto out;
4071
4072         list_for_each_entry(child_resource, &resource->resource_list, list)
4073                 parts_size += child_resource->size_new;
4074
4075         if (parts_size > resource->size_new)
4076                 size_valid = false;
4077 out:
4078         resource->size_valid = size_valid;
4079 }
4080
4081 static int
4082 devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
4083                                struct netlink_ext_ack *extack)
4084 {
4085         u64 reminder;
4086         int err = 0;
4087
4088         if (size > resource->size_params.size_max) {
4089                 NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
4090                 err = -EINVAL;
4091         }
4092
4093         if (size < resource->size_params.size_min) {
4094                 NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
4095                 err = -EINVAL;
4096         }
4097
4098         div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
4099         if (reminder) {
4100                 NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
4101                 err = -EINVAL;
4102         }
4103
4104         return err;
4105 }
4106
4107 static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
4108                                        struct genl_info *info)
4109 {
4110         struct devlink *devlink = info->user_ptr[0];
4111         struct devlink_resource *resource;
4112         u64 resource_id;
4113         u64 size;
4114         int err;
4115
4116         if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
4117             !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
4118                 return -EINVAL;
4119         resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
4120
4121         resource = devlink_resource_find(devlink, NULL, resource_id);
4122         if (!resource)
4123                 return -EINVAL;
4124
4125         size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
4126         err = devlink_resource_validate_size(resource, size, info->extack);
4127         if (err)
4128                 return err;
4129
4130         resource->size_new = size;
4131         devlink_resource_validate_children(resource);
4132         if (resource->parent)
4133                 devlink_resource_validate_children(resource->parent);
4134         return 0;
4135 }
4136
4137 static int
4138 devlink_resource_size_params_put(struct devlink_resource *resource,
4139                                  struct sk_buff *skb)
4140 {
4141         struct devlink_resource_size_params *size_params;
4142
4143         size_params = &resource->size_params;
4144         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
4145                               size_params->size_granularity, DEVLINK_ATTR_PAD) ||
4146             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
4147                               size_params->size_max, DEVLINK_ATTR_PAD) ||
4148             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
4149                               size_params->size_min, DEVLINK_ATTR_PAD) ||
4150             nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
4151                 return -EMSGSIZE;
4152         return 0;
4153 }
4154
4155 static int devlink_resource_occ_put(struct devlink_resource *resource,
4156                                     struct sk_buff *skb)
4157 {
4158         if (!resource->occ_get)
4159                 return 0;
4160         return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
4161                                  resource->occ_get(resource->occ_get_priv),
4162                                  DEVLINK_ATTR_PAD);
4163 }
4164
4165 static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
4166                                 struct devlink_resource *resource)
4167 {
4168         struct devlink_resource *child_resource;
4169         struct nlattr *child_resource_attr;
4170         struct nlattr *resource_attr;
4171
4172         resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
4173         if (!resource_attr)
4174                 return -EMSGSIZE;
4175
4176         if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
4177             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
4178                               DEVLINK_ATTR_PAD) ||
4179             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
4180                               DEVLINK_ATTR_PAD))
4181                 goto nla_put_failure;
4182         if (resource->size != resource->size_new)
4183                 nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
4184                                   resource->size_new, DEVLINK_ATTR_PAD);
4185         if (devlink_resource_occ_put(resource, skb))
4186                 goto nla_put_failure;
4187         if (devlink_resource_size_params_put(resource, skb))
4188                 goto nla_put_failure;
4189         if (list_empty(&resource->resource_list))
4190                 goto out;
4191
4192         if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
4193                        resource->size_valid))
4194                 goto nla_put_failure;
4195
4196         child_resource_attr = nla_nest_start_noflag(skb,
4197                                                     DEVLINK_ATTR_RESOURCE_LIST);
4198         if (!child_resource_attr)
4199                 goto nla_put_failure;
4200
4201         list_for_each_entry(child_resource, &resource->resource_list, list) {
4202                 if (devlink_resource_put(devlink, skb, child_resource))
4203                         goto resource_put_failure;
4204         }
4205
4206         nla_nest_end(skb, child_resource_attr);
4207 out:
4208         nla_nest_end(skb, resource_attr);
4209         return 0;
4210
4211 resource_put_failure:
4212         nla_nest_cancel(skb, child_resource_attr);
4213 nla_put_failure:
4214         nla_nest_cancel(skb, resource_attr);
4215         return -EMSGSIZE;
4216 }
4217
4218 static int devlink_resource_fill(struct genl_info *info,
4219                                  enum devlink_command cmd, int flags)
4220 {
4221         struct devlink *devlink = info->user_ptr[0];
4222         struct devlink_resource *resource;
4223         struct nlattr *resources_attr;
4224         struct sk_buff *skb = NULL;
4225         struct nlmsghdr *nlh;
4226         bool incomplete;
4227         void *hdr;
4228         int i;
4229         int err;
4230
4231         resource = list_first_entry(&devlink->resource_list,
4232                                     struct devlink_resource, list);
4233 start_again:
4234         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
4235         if (err)
4236                 return err;
4237
4238         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
4239                           &devlink_nl_family, NLM_F_MULTI, cmd);
4240         if (!hdr) {
4241                 nlmsg_free(skb);
4242                 return -EMSGSIZE;
4243         }
4244
4245         if (devlink_nl_put_handle(skb, devlink))
4246                 goto nla_put_failure;
4247
4248         resources_attr = nla_nest_start_noflag(skb,
4249                                                DEVLINK_ATTR_RESOURCE_LIST);
4250         if (!resources_attr)
4251                 goto nla_put_failure;
4252
4253         incomplete = false;
4254         i = 0;
4255         list_for_each_entry_from(resource, &devlink->resource_list, list) {
4256                 err = devlink_resource_put(devlink, skb, resource);
4257                 if (err) {
4258                         if (!i)
4259                                 goto err_resource_put;
4260                         incomplete = true;
4261                         break;
4262                 }
4263                 i++;
4264         }
4265         nla_nest_end(skb, resources_attr);
4266         genlmsg_end(skb, hdr);
4267         if (incomplete)
4268                 goto start_again;
4269 send_done:
4270         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
4271                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
4272         if (!nlh) {
4273                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
4274                 if (err)
4275                         return err;
4276                 goto send_done;
4277         }
4278         return genlmsg_reply(skb, info);
4279
4280 nla_put_failure:
4281         err = -EMSGSIZE;
4282 err_resource_put:
4283         nlmsg_free(skb);
4284         return err;
4285 }
4286
4287 static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
4288                                         struct genl_info *info)
4289 {
4290         struct devlink *devlink = info->user_ptr[0];
4291
4292         if (list_empty(&devlink->resource_list))
4293                 return -EOPNOTSUPP;
4294
4295         return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
4296 }
4297
4298 static int
4299 devlink_resources_validate(struct devlink *devlink,
4300                            struct devlink_resource *resource,
4301                            struct genl_info *info)
4302 {
4303         struct list_head *resource_list;
4304         int err = 0;
4305
4306         if (resource)
4307                 resource_list = &resource->resource_list;
4308         else
4309                 resource_list = &devlink->resource_list;
4310
4311         list_for_each_entry(resource, resource_list, list) {
4312                 if (!resource->size_valid)
4313                         return -EINVAL;
4314                 err = devlink_resources_validate(devlink, resource, info);
4315                 if (err)
4316                         return err;
4317         }
4318         return err;
4319 }
4320
4321 static struct net *devlink_netns_get(struct sk_buff *skb,
4322                                      struct genl_info *info)
4323 {
4324         struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
4325         struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
4326         struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
4327         struct net *net;
4328
4329         if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
4330                 NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
4331                 return ERR_PTR(-EINVAL);
4332         }
4333
4334         if (netns_pid_attr) {
4335                 net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
4336         } else if (netns_fd_attr) {
4337                 net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
4338         } else if (netns_id_attr) {
4339                 net = get_net_ns_by_id(sock_net(skb->sk),
4340                                        nla_get_u32(netns_id_attr));
4341                 if (!net)
4342                         net = ERR_PTR(-EINVAL);
4343         } else {
4344                 WARN_ON(1);
4345                 net = ERR_PTR(-EINVAL);
4346         }
4347         if (IS_ERR(net)) {
4348                 NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
4349                 return ERR_PTR(-EINVAL);
4350         }
4351         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
4352                 put_net(net);
4353                 return ERR_PTR(-EPERM);
4354         }
4355         return net;
4356 }
4357
4358 static void devlink_param_notify(struct devlink *devlink,
4359                                  unsigned int port_index,
4360                                  struct devlink_param_item *param_item,
4361                                  enum devlink_command cmd);
4362
4363 static void devlink_ns_change_notify(struct devlink *devlink,
4364                                      struct net *dest_net, struct net *curr_net,
4365                                      bool new)
4366 {
4367         struct devlink_param_item *param_item;
4368         enum devlink_command cmd;
4369
4370         /* Userspace needs to be notified about devlink objects
4371          * removed from original and entering new network namespace.
4372          * The rest of the devlink objects are re-created during
4373          * reload process so the notifications are generated separatelly.
4374          */
4375
4376         if (!dest_net || net_eq(dest_net, curr_net))
4377                 return;
4378
4379         if (new)
4380                 devlink_notify(devlink, DEVLINK_CMD_NEW);
4381
4382         cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
4383         list_for_each_entry(param_item, &devlink->param_list, list)
4384                 devlink_param_notify(devlink, 0, param_item, cmd);
4385
4386         if (!new)
4387                 devlink_notify(devlink, DEVLINK_CMD_DEL);
4388 }
4389
4390 static bool devlink_reload_supported(const struct devlink_ops *ops)
4391 {
4392         return ops->reload_down && ops->reload_up;
4393 }
4394
4395 static void devlink_reload_failed_set(struct devlink *devlink,
4396                                       bool reload_failed)
4397 {
4398         if (devlink->reload_failed == reload_failed)
4399                 return;
4400         devlink->reload_failed = reload_failed;
4401         devlink_notify(devlink, DEVLINK_CMD_NEW);
4402 }
4403
4404 bool devlink_is_reload_failed(const struct devlink *devlink)
4405 {
4406         return devlink->reload_failed;
4407 }
4408 EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
4409
4410 static void
4411 __devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
4412                               enum devlink_reload_limit limit, u32 actions_performed)
4413 {
4414         unsigned long actions = actions_performed;
4415         int stat_idx;
4416         int action;
4417
4418         for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
4419                 stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
4420                 reload_stats[stat_idx]++;
4421         }
4422         devlink_notify(devlink, DEVLINK_CMD_NEW);
4423 }
4424
4425 static void
4426 devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
4427                             u32 actions_performed)
4428 {
4429         __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
4430                                       actions_performed);
4431 }
4432
4433 /**
4434  *      devlink_remote_reload_actions_performed - Update devlink on reload actions
4435  *        performed which are not a direct result of devlink reload call.
4436  *
4437  *      This should be called by a driver after performing reload actions in case it was not
4438  *      a result of devlink reload call. For example fw_activate was performed as a result
4439  *      of devlink reload triggered fw_activate on another host.
4440  *      The motivation for this function is to keep data on reload actions performed on this
4441  *      function whether it was done due to direct devlink reload call or not.
4442  *
4443  *      @devlink: devlink
4444  *      @limit: reload limit
4445  *      @actions_performed: bitmask of actions performed
4446  */
4447 void devlink_remote_reload_actions_performed(struct devlink *devlink,
4448                                              enum devlink_reload_limit limit,
4449                                              u32 actions_performed)
4450 {
4451         if (WARN_ON(!actions_performed ||
4452                     actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
4453                     actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
4454                     limit > DEVLINK_RELOAD_LIMIT_MAX))
4455                 return;
4456
4457         __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
4458                                       actions_performed);
4459 }
4460 EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
4461
4462 static int devlink_reload(struct devlink *devlink, struct net *dest_net,
4463                           enum devlink_reload_action action, enum devlink_reload_limit limit,
4464                           u32 *actions_performed, struct netlink_ext_ack *extack)
4465 {
4466         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
4467         struct net *curr_net;
4468         int err;
4469
4470         memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
4471                sizeof(remote_reload_stats));
4472
4473         curr_net = devlink_net(devlink);
4474         devlink_ns_change_notify(devlink, dest_net, curr_net, false);
4475         err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
4476         if (err)
4477                 return err;
4478
4479         if (dest_net && !net_eq(dest_net, curr_net))
4480                 write_pnet(&devlink->_net, dest_net);
4481
4482         err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
4483         devlink_reload_failed_set(devlink, !!err);
4484         if (err)
4485                 return err;
4486
4487         devlink_ns_change_notify(devlink, dest_net, curr_net, true);
4488         WARN_ON(!(*actions_performed & BIT(action)));
4489         /* Catch driver on updating the remote action within devlink reload */
4490         WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
4491                        sizeof(remote_reload_stats)));
4492         devlink_reload_stats_update(devlink, limit, *actions_performed);
4493         return 0;
4494 }
4495
4496 static int
4497 devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
4498                                         enum devlink_command cmd, struct genl_info *info)
4499 {
4500         struct sk_buff *msg;
4501         void *hdr;
4502
4503         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4504         if (!msg)
4505                 return -ENOMEM;
4506
4507         hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
4508         if (!hdr)
4509                 goto free_msg;
4510
4511         if (devlink_nl_put_handle(msg, devlink))
4512                 goto nla_put_failure;
4513
4514         if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
4515                                actions_performed))
4516                 goto nla_put_failure;
4517         genlmsg_end(msg, hdr);
4518
4519         return genlmsg_reply(msg, info);
4520
4521 nla_put_failure:
4522         genlmsg_cancel(msg, hdr);
4523 free_msg:
4524         nlmsg_free(msg);
4525         return -EMSGSIZE;
4526 }
4527
4528 static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
4529 {
4530         struct devlink *devlink = info->user_ptr[0];
4531         enum devlink_reload_action action;
4532         enum devlink_reload_limit limit;
4533         struct net *dest_net = NULL;
4534         u32 actions_performed;
4535         int err;
4536
4537         if (!(devlink->features & DEVLINK_F_RELOAD))
4538                 return -EOPNOTSUPP;
4539
4540         err = devlink_resources_validate(devlink, NULL, info);
4541         if (err) {
4542                 NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
4543                 return err;
4544         }
4545
4546         if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
4547                 action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
4548         else
4549                 action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
4550
4551         if (!devlink_reload_action_is_supported(devlink, action)) {
4552                 NL_SET_ERR_MSG_MOD(info->extack,
4553                                    "Requested reload action is not supported by the driver");
4554                 return -EOPNOTSUPP;
4555         }
4556
4557         limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
4558         if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
4559                 struct nla_bitfield32 limits;
4560                 u32 limits_selected;
4561
4562                 limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
4563                 limits_selected = limits.value & limits.selector;
4564                 if (!limits_selected) {
4565                         NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
4566                         return -EINVAL;
4567                 }
4568                 for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
4569                         if (limits_selected & BIT(limit))
4570                                 break;
4571                 /* UAPI enables multiselection, but currently it is not used */
4572                 if (limits_selected != BIT(limit)) {
4573                         NL_SET_ERR_MSG_MOD(info->extack,
4574                                            "Multiselection of limit is not supported");
4575                         return -EOPNOTSUPP;
4576                 }
4577                 if (!devlink_reload_limit_is_supported(devlink, limit)) {
4578                         NL_SET_ERR_MSG_MOD(info->extack,
4579                                            "Requested limit is not supported by the driver");
4580                         return -EOPNOTSUPP;
4581                 }
4582                 if (devlink_reload_combination_is_invalid(action, limit)) {
4583                         NL_SET_ERR_MSG_MOD(info->extack,
4584                                            "Requested limit is invalid for this action");
4585                         return -EINVAL;
4586                 }
4587         }
4588         if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
4589             info->attrs[DEVLINK_ATTR_NETNS_FD] ||
4590             info->attrs[DEVLINK_ATTR_NETNS_ID]) {
4591                 dest_net = devlink_netns_get(skb, info);
4592                 if (IS_ERR(dest_net))
4593                         return PTR_ERR(dest_net);
4594         }
4595
4596         err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
4597
4598         if (dest_net)
4599                 put_net(dest_net);
4600
4601         if (err)
4602                 return err;
4603         /* For backward compatibility generate reply only if attributes used by user */
4604         if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
4605                 return 0;
4606
4607         return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
4608                                                        DEVLINK_CMD_RELOAD, info);
4609 }
4610
4611 static int devlink_nl_flash_update_fill(struct sk_buff *msg,
4612                                         struct devlink *devlink,
4613                                         enum devlink_command cmd,
4614                                         struct devlink_flash_notify *params)
4615 {
4616         void *hdr;
4617
4618         hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
4619         if (!hdr)
4620                 return -EMSGSIZE;
4621
4622         if (devlink_nl_put_handle(msg, devlink))
4623                 goto nla_put_failure;
4624
4625         if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
4626                 goto out;
4627
4628         if (params->status_msg &&
4629             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
4630                            params->status_msg))
4631                 goto nla_put_failure;
4632         if (params->component &&
4633             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
4634                            params->component))
4635                 goto nla_put_failure;
4636         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
4637                               params->done, DEVLINK_ATTR_PAD))
4638                 goto nla_put_failure;
4639         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
4640                               params->total, DEVLINK_ATTR_PAD))
4641                 goto nla_put_failure;
4642         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
4643                               params->timeout, DEVLINK_ATTR_PAD))
4644                 goto nla_put_failure;
4645
4646 out:
4647         genlmsg_end(msg, hdr);
4648         return 0;
4649
4650 nla_put_failure:
4651         genlmsg_cancel(msg, hdr);
4652         return -EMSGSIZE;
4653 }
4654
4655 static void __devlink_flash_update_notify(struct devlink *devlink,
4656                                           enum devlink_command cmd,
4657                                           struct devlink_flash_notify *params)
4658 {
4659         struct sk_buff *msg;
4660         int err;
4661
4662         WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
4663                 cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
4664                 cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
4665
4666         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
4667                 return;
4668
4669         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4670         if (!msg)
4671                 return;
4672
4673         err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
4674         if (err)
4675                 goto out_free_msg;
4676
4677         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4678                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4679         return;
4680
4681 out_free_msg:
4682         nlmsg_free(msg);
4683 }
4684
4685 static void devlink_flash_update_begin_notify(struct devlink *devlink)
4686 {
4687         struct devlink_flash_notify params = {};
4688
4689         __devlink_flash_update_notify(devlink,
4690                                       DEVLINK_CMD_FLASH_UPDATE,
4691                                       &params);
4692 }
4693
4694 static void devlink_flash_update_end_notify(struct devlink *devlink)
4695 {
4696         struct devlink_flash_notify params = {};
4697
4698         __devlink_flash_update_notify(devlink,
4699                                       DEVLINK_CMD_FLASH_UPDATE_END,
4700                                       &params);
4701 }
4702
4703 void devlink_flash_update_status_notify(struct devlink *devlink,
4704                                         const char *status_msg,
4705                                         const char *component,
4706                                         unsigned long done,
4707                                         unsigned long total)
4708 {
4709         struct devlink_flash_notify params = {
4710                 .status_msg = status_msg,
4711                 .component = component,
4712                 .done = done,
4713                 .total = total,
4714         };
4715
4716         __devlink_flash_update_notify(devlink,
4717                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4718                                       &params);
4719 }
4720 EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
4721
4722 void devlink_flash_update_timeout_notify(struct devlink *devlink,
4723                                          const char *status_msg,
4724                                          const char *component,
4725                                          unsigned long timeout)
4726 {
4727         struct devlink_flash_notify params = {
4728                 .status_msg = status_msg,
4729                 .component = component,
4730                 .timeout = timeout,
4731         };
4732
4733         __devlink_flash_update_notify(devlink,
4734                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4735                                       &params);
4736 }
4737 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
4738
4739 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
4740                                        struct genl_info *info)
4741 {
4742         struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name;
4743         struct devlink_flash_update_params params = {};
4744         struct devlink *devlink = info->user_ptr[0];
4745         const char *file_name;
4746         u32 supported_params;
4747         int ret;
4748
4749         if (!devlink->ops->flash_update)
4750                 return -EOPNOTSUPP;
4751
4752         if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
4753                 return -EINVAL;
4754
4755         supported_params = devlink->ops->supported_flash_update_params;
4756
4757         nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
4758         if (nla_component) {
4759                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
4760                         NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
4761                                             "component update is not supported by this device");
4762                         return -EOPNOTSUPP;
4763                 }
4764                 params.component = nla_data(nla_component);
4765         }
4766
4767         nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
4768         if (nla_overwrite_mask) {
4769                 struct nla_bitfield32 sections;
4770
4771                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
4772                         NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
4773                                             "overwrite settings are not supported by this device");
4774                         return -EOPNOTSUPP;
4775                 }
4776                 sections = nla_get_bitfield32(nla_overwrite_mask);
4777                 params.overwrite_mask = sections.value & sections.selector;
4778         }
4779
4780         nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
4781         file_name = nla_data(nla_file_name);
4782         ret = request_firmware(&params.fw, file_name, devlink->dev);
4783         if (ret) {
4784                 NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
4785                 return ret;
4786         }
4787
4788         devlink_flash_update_begin_notify(devlink);
4789         ret = devlink->ops->flash_update(devlink, &params, info->extack);
4790         devlink_flash_update_end_notify(devlink);
4791
4792         release_firmware(params.fw);
4793
4794         return ret;
4795 }
4796
4797 static const struct devlink_param devlink_param_generic[] = {
4798         {
4799                 .id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
4800                 .name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
4801                 .type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
4802         },
4803         {
4804                 .id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
4805                 .name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
4806                 .type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
4807         },
4808         {
4809                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
4810                 .name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
4811                 .type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
4812         },
4813         {
4814                 .id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
4815                 .name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
4816                 .type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
4817         },
4818         {
4819                 .id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
4820                 .name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
4821                 .type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
4822         },
4823         {
4824                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
4825                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
4826                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
4827         },
4828         {
4829                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
4830                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
4831                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
4832         },
4833         {
4834                 .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
4835                 .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
4836                 .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
4837         },
4838         {
4839                 .id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
4840                 .name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
4841                 .type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
4842         },
4843         {
4844                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
4845                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
4846                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
4847         },
4848         {
4849                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
4850                 .name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
4851                 .type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
4852         },
4853         {
4854                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
4855                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
4856                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
4857         },
4858         {
4859                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
4860                 .name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
4861                 .type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
4862         },
4863         {
4864                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
4865                 .name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
4866                 .type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
4867         },
4868         {
4869                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
4870                 .name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME,
4871                 .type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE,
4872         },
4873         {
4874                 .id = DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
4875                 .name = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_NAME,
4876                 .type = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_TYPE,
4877         },
4878         {
4879                 .id = DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
4880                 .name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
4881                 .type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
4882         },
4883 };
4884
4885 static int devlink_param_generic_verify(const struct devlink_param *param)
4886 {
4887         /* verify it match generic parameter by id and name */
4888         if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
4889                 return -EINVAL;
4890         if (strcmp(param->name, devlink_param_generic[param->id].name))
4891                 return -ENOENT;
4892
4893         WARN_ON(param->type != devlink_param_generic[param->id].type);
4894
4895         return 0;
4896 }
4897
4898 static int devlink_param_driver_verify(const struct devlink_param *param)
4899 {
4900         int i;
4901
4902         if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
4903                 return -EINVAL;
4904         /* verify no such name in generic params */
4905         for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
4906                 if (!strcmp(param->name, devlink_param_generic[i].name))
4907                         return -EEXIST;
4908
4909         return 0;
4910 }
4911
4912 static struct devlink_param_item *
4913 devlink_param_find_by_name(struct list_head *param_list,
4914                            const char *param_name)
4915 {
4916         struct devlink_param_item *param_item;
4917
4918         list_for_each_entry(param_item, param_list, list)
4919                 if (!strcmp(param_item->param->name, param_name))
4920                         return param_item;
4921         return NULL;
4922 }
4923
4924 static struct devlink_param_item *
4925 devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
4926 {
4927         struct devlink_param_item *param_item;
4928
4929         list_for_each_entry(param_item, param_list, list)
4930                 if (param_item->param->id == param_id)
4931                         return param_item;
4932         return NULL;
4933 }
4934
4935 static bool
4936 devlink_param_cmode_is_supported(const struct devlink_param *param,
4937                                  enum devlink_param_cmode cmode)
4938 {
4939         return test_bit(cmode, &param->supported_cmodes);
4940 }
4941
4942 static int devlink_param_get(struct devlink *devlink,
4943                              const struct devlink_param *param,
4944                              struct devlink_param_gset_ctx *ctx)
4945 {
4946         if (!param->get)
4947                 return -EOPNOTSUPP;
4948         return param->get(devlink, param->id, ctx);
4949 }
4950
4951 static int devlink_param_set(struct devlink *devlink,
4952                              const struct devlink_param *param,
4953                              struct devlink_param_gset_ctx *ctx)
4954 {
4955         if (!param->set)
4956                 return -EOPNOTSUPP;
4957         return param->set(devlink, param->id, ctx);
4958 }
4959
4960 static int
4961 devlink_param_type_to_nla_type(enum devlink_param_type param_type)
4962 {
4963         switch (param_type) {
4964         case DEVLINK_PARAM_TYPE_U8:
4965                 return NLA_U8;
4966         case DEVLINK_PARAM_TYPE_U16:
4967                 return NLA_U16;
4968         case DEVLINK_PARAM_TYPE_U32:
4969                 return NLA_U32;
4970         case DEVLINK_PARAM_TYPE_STRING:
4971                 return NLA_STRING;
4972         case DEVLINK_PARAM_TYPE_BOOL:
4973                 return NLA_FLAG;
4974         default:
4975                 return -EINVAL;
4976         }
4977 }
4978
4979 static int
4980 devlink_nl_param_value_fill_one(struct sk_buff *msg,
4981                                 enum devlink_param_type type,
4982                                 enum devlink_param_cmode cmode,
4983                                 union devlink_param_value val)
4984 {
4985         struct nlattr *param_value_attr;
4986
4987         param_value_attr = nla_nest_start_noflag(msg,
4988                                                  DEVLINK_ATTR_PARAM_VALUE);
4989         if (!param_value_attr)
4990                 goto nla_put_failure;
4991
4992         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
4993                 goto value_nest_cancel;
4994
4995         switch (type) {
4996         case DEVLINK_PARAM_TYPE_U8:
4997                 if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
4998                         goto value_nest_cancel;
4999                 break;
5000         case DEVLINK_PARAM_TYPE_U16:
5001                 if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
5002                         goto value_nest_cancel;
5003                 break;
5004         case DEVLINK_PARAM_TYPE_U32:
5005                 if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
5006                         goto value_nest_cancel;
5007                 break;
5008         case DEVLINK_PARAM_TYPE_STRING:
5009                 if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
5010                                    val.vstr))
5011                         goto value_nest_cancel;
5012                 break;
5013         case DEVLINK_PARAM_TYPE_BOOL:
5014                 if (val.vbool &&
5015                     nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
5016                         goto value_nest_cancel;
5017                 break;
5018         }
5019
5020         nla_nest_end(msg, param_value_attr);
5021         return 0;
5022
5023 value_nest_cancel:
5024         nla_nest_cancel(msg, param_value_attr);
5025 nla_put_failure:
5026         return -EMSGSIZE;
5027 }
5028
5029 static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
5030                                  unsigned int port_index,
5031                                  struct devlink_param_item *param_item,
5032                                  enum devlink_command cmd,
5033                                  u32 portid, u32 seq, int flags)
5034 {
5035         union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
5036         bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
5037         const struct devlink_param *param = param_item->param;
5038         struct devlink_param_gset_ctx ctx;
5039         struct nlattr *param_values_list;
5040         struct nlattr *param_attr;
5041         int nla_type;
5042         void *hdr;
5043         int err;
5044         int i;
5045
5046         /* Get value from driver part to driverinit configuration mode */
5047         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
5048                 if (!devlink_param_cmode_is_supported(param, i))
5049                         continue;
5050                 if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
5051                         if (!param_item->driverinit_value_valid)
5052                                 return -EOPNOTSUPP;
5053                         param_value[i] = param_item->driverinit_value;
5054                 } else {
5055                         ctx.cmode = i;
5056                         err = devlink_param_get(devlink, param, &ctx);
5057                         if (err)
5058                                 return err;
5059                         param_value[i] = ctx.val;
5060                 }
5061                 param_value_set[i] = true;
5062         }
5063
5064         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5065         if (!hdr)
5066                 return -EMSGSIZE;
5067
5068         if (devlink_nl_put_handle(msg, devlink))
5069                 goto genlmsg_cancel;
5070
5071         if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
5072             cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
5073             cmd == DEVLINK_CMD_PORT_PARAM_DEL)
5074                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
5075                         goto genlmsg_cancel;
5076
5077         param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
5078         if (!param_attr)
5079                 goto genlmsg_cancel;
5080         if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
5081                 goto param_nest_cancel;
5082         if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
5083                 goto param_nest_cancel;
5084
5085         nla_type = devlink_param_type_to_nla_type(param->type);
5086         if (nla_type < 0)
5087                 goto param_nest_cancel;
5088         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
5089                 goto param_nest_cancel;
5090
5091         param_values_list = nla_nest_start_noflag(msg,
5092                                                   DEVLINK_ATTR_PARAM_VALUES_LIST);
5093         if (!param_values_list)
5094                 goto param_nest_cancel;
5095
5096         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
5097                 if (!param_value_set[i])
5098                         continue;
5099                 err = devlink_nl_param_value_fill_one(msg, param->type,
5100                                                       i, param_value[i]);
5101                 if (err)
5102                         goto values_list_nest_cancel;
5103         }
5104
5105         nla_nest_end(msg, param_values_list);
5106         nla_nest_end(msg, param_attr);
5107         genlmsg_end(msg, hdr);
5108         return 0;
5109
5110 values_list_nest_cancel:
5111         nla_nest_end(msg, param_values_list);
5112 param_nest_cancel:
5113         nla_nest_cancel(msg, param_attr);
5114 genlmsg_cancel:
5115         genlmsg_cancel(msg, hdr);
5116         return -EMSGSIZE;
5117 }
5118
5119 static void devlink_param_notify(struct devlink *devlink,
5120                                  unsigned int port_index,
5121                                  struct devlink_param_item *param_item,
5122                                  enum devlink_command cmd)
5123 {
5124         struct sk_buff *msg;
5125         int err;
5126
5127         WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
5128                 cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
5129                 cmd != DEVLINK_CMD_PORT_PARAM_DEL);
5130         ASSERT_DEVLINK_REGISTERED(devlink);
5131
5132         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5133         if (!msg)
5134                 return;
5135         err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
5136                                     0, 0, 0);
5137         if (err) {
5138                 nlmsg_free(msg);
5139                 return;
5140         }
5141
5142         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
5143                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5144 }
5145
5146 static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
5147                                            struct netlink_callback *cb)
5148 {
5149         struct devlink_param_item *param_item;
5150         struct devlink *devlink;
5151         int start = cb->args[0];
5152         unsigned long index;
5153         int idx = 0;
5154         int err = 0;
5155
5156         mutex_lock(&devlink_mutex);
5157         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5158                 if (!devlink_try_get(devlink))
5159                         continue;
5160
5161                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5162                         goto retry;
5163
5164                 mutex_lock(&devlink->lock);
5165                 list_for_each_entry(param_item, &devlink->param_list, list) {
5166                         if (idx < start) {
5167                                 idx++;
5168                                 continue;
5169                         }
5170                         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
5171                                                     DEVLINK_CMD_PARAM_GET,
5172                                                     NETLINK_CB(cb->skb).portid,
5173                                                     cb->nlh->nlmsg_seq,
5174                                                     NLM_F_MULTI);
5175                         if (err == -EOPNOTSUPP) {
5176                                 err = 0;
5177                         } else if (err) {
5178                                 mutex_unlock(&devlink->lock);
5179                                 devlink_put(devlink);
5180                                 goto out;
5181                         }
5182                         idx++;
5183                 }
5184                 mutex_unlock(&devlink->lock);
5185 retry:
5186                 devlink_put(devlink);
5187         }
5188 out:
5189         mutex_unlock(&devlink_mutex);
5190
5191         if (err != -EMSGSIZE)
5192                 return err;
5193
5194         cb->args[0] = idx;
5195         return msg->len;
5196 }
5197
5198 static int
5199 devlink_param_type_get_from_info(struct genl_info *info,
5200                                  enum devlink_param_type *param_type)
5201 {
5202         if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
5203                 return -EINVAL;
5204
5205         switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
5206         case NLA_U8:
5207                 *param_type = DEVLINK_PARAM_TYPE_U8;
5208                 break;
5209         case NLA_U16:
5210                 *param_type = DEVLINK_PARAM_TYPE_U16;
5211                 break;
5212         case NLA_U32:
5213                 *param_type = DEVLINK_PARAM_TYPE_U32;
5214                 break;
5215         case NLA_STRING:
5216                 *param_type = DEVLINK_PARAM_TYPE_STRING;
5217                 break;
5218         case NLA_FLAG:
5219                 *param_type = DEVLINK_PARAM_TYPE_BOOL;
5220                 break;
5221         default:
5222                 return -EINVAL;
5223         }
5224
5225         return 0;
5226 }
5227
5228 static int
5229 devlink_param_value_get_from_info(const struct devlink_param *param,
5230                                   struct genl_info *info,
5231                                   union devlink_param_value *value)
5232 {
5233         struct nlattr *param_data;
5234         int len;
5235
5236         param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
5237
5238         if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
5239                 return -EINVAL;
5240
5241         switch (param->type) {
5242         case DEVLINK_PARAM_TYPE_U8:
5243                 if (nla_len(param_data) != sizeof(u8))
5244                         return -EINVAL;
5245                 value->vu8 = nla_get_u8(param_data);
5246                 break;
5247         case DEVLINK_PARAM_TYPE_U16:
5248                 if (nla_len(param_data) != sizeof(u16))
5249                         return -EINVAL;
5250                 value->vu16 = nla_get_u16(param_data);
5251                 break;
5252         case DEVLINK_PARAM_TYPE_U32:
5253                 if (nla_len(param_data) != sizeof(u32))
5254                         return -EINVAL;
5255                 value->vu32 = nla_get_u32(param_data);
5256                 break;
5257         case DEVLINK_PARAM_TYPE_STRING:
5258                 len = strnlen(nla_data(param_data), nla_len(param_data));
5259                 if (len == nla_len(param_data) ||
5260                     len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
5261                         return -EINVAL;
5262                 strcpy(value->vstr, nla_data(param_data));
5263                 break;
5264         case DEVLINK_PARAM_TYPE_BOOL:
5265                 if (param_data && nla_len(param_data))
5266                         return -EINVAL;
5267                 value->vbool = nla_get_flag(param_data);
5268                 break;
5269         }
5270         return 0;
5271 }
5272
5273 static struct devlink_param_item *
5274 devlink_param_get_from_info(struct list_head *param_list,
5275                             struct genl_info *info)
5276 {
5277         char *param_name;
5278
5279         if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
5280                 return NULL;
5281
5282         param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
5283         return devlink_param_find_by_name(param_list, param_name);
5284 }
5285
5286 static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
5287                                          struct genl_info *info)
5288 {
5289         struct devlink *devlink = info->user_ptr[0];
5290         struct devlink_param_item *param_item;
5291         struct sk_buff *msg;
5292         int err;
5293
5294         param_item = devlink_param_get_from_info(&devlink->param_list, info);
5295         if (!param_item)
5296                 return -EINVAL;
5297
5298         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5299         if (!msg)
5300                 return -ENOMEM;
5301
5302         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
5303                                     DEVLINK_CMD_PARAM_GET,
5304                                     info->snd_portid, info->snd_seq, 0);
5305         if (err) {
5306                 nlmsg_free(msg);
5307                 return err;
5308         }
5309
5310         return genlmsg_reply(msg, info);
5311 }
5312
5313 static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
5314                                            unsigned int port_index,
5315                                            struct list_head *param_list,
5316                                            struct genl_info *info,
5317                                            enum devlink_command cmd)
5318 {
5319         enum devlink_param_type param_type;
5320         struct devlink_param_gset_ctx ctx;
5321         enum devlink_param_cmode cmode;
5322         struct devlink_param_item *param_item;
5323         const struct devlink_param *param;
5324         union devlink_param_value value;
5325         int err = 0;
5326
5327         param_item = devlink_param_get_from_info(param_list, info);
5328         if (!param_item)
5329                 return -EINVAL;
5330         param = param_item->param;
5331         err = devlink_param_type_get_from_info(info, &param_type);
5332         if (err)
5333                 return err;
5334         if (param_type != param->type)
5335                 return -EINVAL;
5336         err = devlink_param_value_get_from_info(param, info, &value);
5337         if (err)
5338                 return err;
5339         if (param->validate) {
5340                 err = param->validate(devlink, param->id, value, info->extack);
5341                 if (err)
5342                         return err;
5343         }
5344
5345         if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
5346                 return -EINVAL;
5347         cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
5348         if (!devlink_param_cmode_is_supported(param, cmode))
5349                 return -EOPNOTSUPP;
5350
5351         if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
5352                 if (param->type == DEVLINK_PARAM_TYPE_STRING)
5353                         strcpy(param_item->driverinit_value.vstr, value.vstr);
5354                 else
5355                         param_item->driverinit_value = value;
5356                 param_item->driverinit_value_valid = true;
5357         } else {
5358                 if (!param->set)
5359                         return -EOPNOTSUPP;
5360                 ctx.val = value;
5361                 ctx.cmode = cmode;
5362                 err = devlink_param_set(devlink, param, &ctx);
5363                 if (err)
5364                         return err;
5365         }
5366
5367         devlink_param_notify(devlink, port_index, param_item, cmd);
5368         return 0;
5369 }
5370
5371 static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
5372                                          struct genl_info *info)
5373 {
5374         struct devlink *devlink = info->user_ptr[0];
5375
5376         return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
5377                                                info, DEVLINK_CMD_PARAM_NEW);
5378 }
5379
5380 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
5381                                                 struct netlink_callback *cb)
5382 {
5383         struct devlink_param_item *param_item;
5384         struct devlink_port *devlink_port;
5385         struct devlink *devlink;
5386         int start = cb->args[0];
5387         unsigned long index;
5388         int idx = 0;
5389         int err = 0;
5390
5391         mutex_lock(&devlink_mutex);
5392         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5393                 if (!devlink_try_get(devlink))
5394                         continue;
5395
5396                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5397                         goto retry;
5398
5399                 mutex_lock(&devlink->lock);
5400                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
5401                         list_for_each_entry(param_item,
5402                                             &devlink_port->param_list, list) {
5403                                 if (idx < start) {
5404                                         idx++;
5405                                         continue;
5406                                 }
5407                                 err = devlink_nl_param_fill(msg,
5408                                                 devlink_port->devlink,
5409                                                 devlink_port->index, param_item,
5410                                                 DEVLINK_CMD_PORT_PARAM_GET,
5411                                                 NETLINK_CB(cb->skb).portid,
5412                                                 cb->nlh->nlmsg_seq,
5413                                                 NLM_F_MULTI);
5414                                 if (err == -EOPNOTSUPP) {
5415                                         err = 0;
5416                                 } else if (err) {
5417                                         mutex_unlock(&devlink->lock);
5418                                         devlink_put(devlink);
5419                                         goto out;
5420                                 }
5421                                 idx++;
5422                         }
5423                 }
5424                 mutex_unlock(&devlink->lock);
5425 retry:
5426                 devlink_put(devlink);
5427         }
5428 out:
5429         mutex_unlock(&devlink_mutex);
5430
5431         if (err != -EMSGSIZE)
5432                 return err;
5433
5434         cb->args[0] = idx;
5435         return msg->len;
5436 }
5437
5438 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
5439                                               struct genl_info *info)
5440 {
5441         struct devlink_port *devlink_port = info->user_ptr[1];
5442         struct devlink_param_item *param_item;
5443         struct sk_buff *msg;
5444         int err;
5445
5446         param_item = devlink_param_get_from_info(&devlink_port->param_list,
5447                                                  info);
5448         if (!param_item)
5449                 return -EINVAL;
5450
5451         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5452         if (!msg)
5453                 return -ENOMEM;
5454
5455         err = devlink_nl_param_fill(msg, devlink_port->devlink,
5456                                     devlink_port->index, param_item,
5457                                     DEVLINK_CMD_PORT_PARAM_GET,
5458                                     info->snd_portid, info->snd_seq, 0);
5459         if (err) {
5460                 nlmsg_free(msg);
5461                 return err;
5462         }
5463
5464         return genlmsg_reply(msg, info);
5465 }
5466
5467 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
5468                                               struct genl_info *info)
5469 {
5470         struct devlink_port *devlink_port = info->user_ptr[1];
5471
5472         return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
5473                                                devlink_port->index,
5474                                                &devlink_port->param_list, info,
5475                                                DEVLINK_CMD_PORT_PARAM_NEW);
5476 }
5477
5478 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
5479                                              struct devlink *devlink,
5480                                              struct devlink_snapshot *snapshot)
5481 {
5482         struct nlattr *snap_attr;
5483         int err;
5484
5485         snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
5486         if (!snap_attr)
5487                 return -EINVAL;
5488
5489         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
5490         if (err)
5491                 goto nla_put_failure;
5492
5493         nla_nest_end(msg, snap_attr);
5494         return 0;
5495
5496 nla_put_failure:
5497         nla_nest_cancel(msg, snap_attr);
5498         return err;
5499 }
5500
5501 static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
5502                                               struct devlink *devlink,
5503                                               struct devlink_region *region)
5504 {
5505         struct devlink_snapshot *snapshot;
5506         struct nlattr *snapshots_attr;
5507         int err;
5508
5509         snapshots_attr = nla_nest_start_noflag(msg,
5510                                                DEVLINK_ATTR_REGION_SNAPSHOTS);
5511         if (!snapshots_attr)
5512                 return -EINVAL;
5513
5514         list_for_each_entry(snapshot, &region->snapshot_list, list) {
5515                 err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
5516                 if (err)
5517                         goto nla_put_failure;
5518         }
5519
5520         nla_nest_end(msg, snapshots_attr);
5521         return 0;
5522
5523 nla_put_failure:
5524         nla_nest_cancel(msg, snapshots_attr);
5525         return err;
5526 }
5527
5528 static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
5529                                   enum devlink_command cmd, u32 portid,
5530                                   u32 seq, int flags,
5531                                   struct devlink_region *region)
5532 {
5533         void *hdr;
5534         int err;
5535
5536         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5537         if (!hdr)
5538                 return -EMSGSIZE;
5539
5540         err = devlink_nl_put_handle(msg, devlink);
5541         if (err)
5542                 goto nla_put_failure;
5543
5544         if (region->port) {
5545                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5546                                   region->port->index);
5547                 if (err)
5548                         goto nla_put_failure;
5549         }
5550
5551         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
5552         if (err)
5553                 goto nla_put_failure;
5554
5555         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5556                                 region->size,
5557                                 DEVLINK_ATTR_PAD);
5558         if (err)
5559                 goto nla_put_failure;
5560
5561         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
5562                           region->max_snapshots);
5563         if (err)
5564                 goto nla_put_failure;
5565
5566         err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
5567         if (err)
5568                 goto nla_put_failure;
5569
5570         genlmsg_end(msg, hdr);
5571         return 0;
5572
5573 nla_put_failure:
5574         genlmsg_cancel(msg, hdr);
5575         return err;
5576 }
5577
5578 static struct sk_buff *
5579 devlink_nl_region_notify_build(struct devlink_region *region,
5580                                struct devlink_snapshot *snapshot,
5581                                enum devlink_command cmd, u32 portid, u32 seq)
5582 {
5583         struct devlink *devlink = region->devlink;
5584         struct sk_buff *msg;
5585         void *hdr;
5586         int err;
5587
5588
5589         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5590         if (!msg)
5591                 return ERR_PTR(-ENOMEM);
5592
5593         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
5594         if (!hdr) {
5595                 err = -EMSGSIZE;
5596                 goto out_free_msg;
5597         }
5598
5599         err = devlink_nl_put_handle(msg, devlink);
5600         if (err)
5601                 goto out_cancel_msg;
5602
5603         if (region->port) {
5604                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5605                                   region->port->index);
5606                 if (err)
5607                         goto out_cancel_msg;
5608         }
5609
5610         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
5611                              region->ops->name);
5612         if (err)
5613                 goto out_cancel_msg;
5614
5615         if (snapshot) {
5616                 err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
5617                                   snapshot->id);
5618                 if (err)
5619                         goto out_cancel_msg;
5620         } else {
5621                 err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5622                                         region->size, DEVLINK_ATTR_PAD);
5623                 if (err)
5624                         goto out_cancel_msg;
5625         }
5626         genlmsg_end(msg, hdr);
5627
5628         return msg;
5629
5630 out_cancel_msg:
5631         genlmsg_cancel(msg, hdr);
5632 out_free_msg:
5633         nlmsg_free(msg);
5634         return ERR_PTR(err);
5635 }
5636
5637 static void devlink_nl_region_notify(struct devlink_region *region,
5638                                      struct devlink_snapshot *snapshot,
5639                                      enum devlink_command cmd)
5640 {
5641         struct devlink *devlink = region->devlink;
5642         struct sk_buff *msg;
5643
5644         WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
5645         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
5646                 return;
5647
5648         msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
5649         if (IS_ERR(msg))
5650                 return;
5651
5652         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
5653                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5654 }
5655
5656 /**
5657  * __devlink_snapshot_id_increment - Increment number of snapshots using an id
5658  *      @devlink: devlink instance
5659  *      @id: the snapshot id
5660  *
5661  *      Track when a new snapshot begins using an id. Load the count for the
5662  *      given id from the snapshot xarray, increment it, and store it back.
5663  *
5664  *      Called when a new snapshot is created with the given id.
5665  *
5666  *      The id *must* have been previously allocated by
5667  *      devlink_region_snapshot_id_get().
5668  *
5669  *      Returns 0 on success, or an error on failure.
5670  */
5671 static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
5672 {
5673         unsigned long count;
5674         void *p;
5675
5676         lockdep_assert_held(&devlink->lock);
5677
5678         p = xa_load(&devlink->snapshot_ids, id);
5679         if (WARN_ON(!p))
5680                 return -EINVAL;
5681
5682         if (WARN_ON(!xa_is_value(p)))
5683                 return -EINVAL;
5684
5685         count = xa_to_value(p);
5686         count++;
5687
5688         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5689                                GFP_KERNEL));
5690 }
5691
5692 /**
5693  * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
5694  *      @devlink: devlink instance
5695  *      @id: the snapshot id
5696  *
5697  *      Track when a snapshot is deleted and stops using an id. Load the count
5698  *      for the given id from the snapshot xarray, decrement it, and store it
5699  *      back.
5700  *
5701  *      If the count reaches zero, erase this id from the xarray, freeing it
5702  *      up for future re-use by devlink_region_snapshot_id_get().
5703  *
5704  *      Called when a snapshot using the given id is deleted, and when the
5705  *      initial allocator of the id is finished using it.
5706  */
5707 static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
5708 {
5709         unsigned long count;
5710         void *p;
5711
5712         lockdep_assert_held(&devlink->lock);
5713
5714         p = xa_load(&devlink->snapshot_ids, id);
5715         if (WARN_ON(!p))
5716                 return;
5717
5718         if (WARN_ON(!xa_is_value(p)))
5719                 return;
5720
5721         count = xa_to_value(p);
5722
5723         if (count > 1) {
5724                 count--;
5725                 xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5726                          GFP_KERNEL);
5727         } else {
5728                 /* If this was the last user, we can erase this id */
5729                 xa_erase(&devlink->snapshot_ids, id);
5730         }
5731 }
5732
5733 /**
5734  *      __devlink_snapshot_id_insert - Insert a specific snapshot ID
5735  *      @devlink: devlink instance
5736  *      @id: the snapshot id
5737  *
5738  *      Mark the given snapshot id as used by inserting a zero value into the
5739  *      snapshot xarray.
5740  *
5741  *      This must be called while holding the devlink instance lock. Unlike
5742  *      devlink_snapshot_id_get, the initial reference count is zero, not one.
5743  *      It is expected that the id will immediately be used before
5744  *      releasing the devlink instance lock.
5745  *
5746  *      Returns zero on success, or an error code if the snapshot id could not
5747  *      be inserted.
5748  */
5749 static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
5750 {
5751         lockdep_assert_held(&devlink->lock);
5752
5753         if (xa_load(&devlink->snapshot_ids, id))
5754                 return -EEXIST;
5755
5756         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
5757                                GFP_KERNEL));
5758 }
5759
5760 /**
5761  *      __devlink_region_snapshot_id_get - get snapshot ID
5762  *      @devlink: devlink instance
5763  *      @id: storage to return snapshot id
5764  *
5765  *      Allocates a new snapshot id. Returns zero on success, or a negative
5766  *      error on failure. Must be called while holding the devlink instance
5767  *      lock.
5768  *
5769  *      Snapshot IDs are tracked using an xarray which stores the number of
5770  *      users of the snapshot id.
5771  *
5772  *      Note that the caller of this function counts as a 'user', in order to
5773  *      avoid race conditions. The caller must release its hold on the
5774  *      snapshot by using devlink_region_snapshot_id_put.
5775  */
5776 static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
5777 {
5778         lockdep_assert_held(&devlink->lock);
5779
5780         return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
5781                         xa_limit_32b, GFP_KERNEL);
5782 }
5783
5784 /**
5785  *      __devlink_region_snapshot_create - create a new snapshot
5786  *      This will add a new snapshot of a region. The snapshot
5787  *      will be stored on the region struct and can be accessed
5788  *      from devlink. This is useful for future analyses of snapshots.
5789  *      Multiple snapshots can be created on a region.
5790  *      The @snapshot_id should be obtained using the getter function.
5791  *
5792  *      Must be called only while holding the devlink instance lock.
5793  *
5794  *      @region: devlink region of the snapshot
5795  *      @data: snapshot data
5796  *      @snapshot_id: snapshot id to be created
5797  */
5798 static int
5799 __devlink_region_snapshot_create(struct devlink_region *region,
5800                                  u8 *data, u32 snapshot_id)
5801 {
5802         struct devlink *devlink = region->devlink;
5803         struct devlink_snapshot *snapshot;
5804         int err;
5805
5806         lockdep_assert_held(&devlink->lock);
5807
5808         /* check if region can hold one more snapshot */
5809         if (region->cur_snapshots == region->max_snapshots)
5810                 return -ENOSPC;
5811
5812         if (devlink_region_snapshot_get_by_id(region, snapshot_id))
5813                 return -EEXIST;
5814
5815         snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
5816         if (!snapshot)
5817                 return -ENOMEM;
5818
5819         err = __devlink_snapshot_id_increment(devlink, snapshot_id);
5820         if (err)
5821                 goto err_snapshot_id_increment;
5822
5823         snapshot->id = snapshot_id;
5824         snapshot->region = region;
5825         snapshot->data = data;
5826
5827         list_add_tail(&snapshot->list, &region->snapshot_list);
5828
5829         region->cur_snapshots++;
5830
5831         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
5832         return 0;
5833
5834 err_snapshot_id_increment:
5835         kfree(snapshot);
5836         return err;
5837 }
5838
5839 static void devlink_region_snapshot_del(struct devlink_region *region,
5840                                         struct devlink_snapshot *snapshot)
5841 {
5842         struct devlink *devlink = region->devlink;
5843
5844         lockdep_assert_held(&devlink->lock);
5845
5846         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
5847         region->cur_snapshots--;
5848         list_del(&snapshot->list);
5849         region->ops->destructor(snapshot->data);
5850         __devlink_snapshot_id_decrement(devlink, snapshot->id);
5851         kfree(snapshot);
5852 }
5853
5854 static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
5855                                           struct genl_info *info)
5856 {
5857         struct devlink *devlink = info->user_ptr[0];
5858         struct devlink_port *port = NULL;
5859         struct devlink_region *region;
5860         const char *region_name;
5861         struct sk_buff *msg;
5862         unsigned int index;
5863         int err;
5864
5865         if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
5866                 return -EINVAL;
5867
5868         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5869                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5870
5871                 port = devlink_port_get_by_index(devlink, index);
5872                 if (!port)
5873                         return -ENODEV;
5874         }
5875
5876         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5877         if (port)
5878                 region = devlink_port_region_get_by_name(port, region_name);
5879         else
5880                 region = devlink_region_get_by_name(devlink, region_name);
5881
5882         if (!region)
5883                 return -EINVAL;
5884
5885         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5886         if (!msg)
5887                 return -ENOMEM;
5888
5889         err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
5890                                      info->snd_portid, info->snd_seq, 0,
5891                                      region);
5892         if (err) {
5893                 nlmsg_free(msg);
5894                 return err;
5895         }
5896
5897         return genlmsg_reply(msg, info);
5898 }
5899
5900 static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
5901                                                  struct netlink_callback *cb,
5902                                                  struct devlink_port *port,
5903                                                  int *idx,
5904                                                  int start)
5905 {
5906         struct devlink_region *region;
5907         int err = 0;
5908
5909         list_for_each_entry(region, &port->region_list, list) {
5910                 if (*idx < start) {
5911                         (*idx)++;
5912                         continue;
5913                 }
5914                 err = devlink_nl_region_fill(msg, port->devlink,
5915                                              DEVLINK_CMD_REGION_GET,
5916                                              NETLINK_CB(cb->skb).portid,
5917                                              cb->nlh->nlmsg_seq,
5918                                              NLM_F_MULTI, region);
5919                 if (err)
5920                         goto out;
5921                 (*idx)++;
5922         }
5923
5924 out:
5925         return err;
5926 }
5927
5928 static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
5929                                                     struct netlink_callback *cb,
5930                                                     struct devlink *devlink,
5931                                                     int *idx,
5932                                                     int start)
5933 {
5934         struct devlink_region *region;
5935         struct devlink_port *port;
5936         int err = 0;
5937
5938         mutex_lock(&devlink->lock);
5939         list_for_each_entry(region, &devlink->region_list, list) {
5940                 if (*idx < start) {
5941                         (*idx)++;
5942                         continue;
5943                 }
5944                 err = devlink_nl_region_fill(msg, devlink,
5945                                              DEVLINK_CMD_REGION_GET,
5946                                              NETLINK_CB(cb->skb).portid,
5947                                              cb->nlh->nlmsg_seq,
5948                                              NLM_F_MULTI, region);
5949                 if (err)
5950                         goto out;
5951                 (*idx)++;
5952         }
5953
5954         list_for_each_entry(port, &devlink->port_list, list) {
5955                 err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
5956                                                             start);
5957                 if (err)
5958                         goto out;
5959         }
5960
5961 out:
5962         mutex_unlock(&devlink->lock);
5963         return err;
5964 }
5965
5966 static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
5967                                             struct netlink_callback *cb)
5968 {
5969         struct devlink *devlink;
5970         int start = cb->args[0];
5971         unsigned long index;
5972         int idx = 0;
5973         int err = 0;
5974
5975         mutex_lock(&devlink_mutex);
5976         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5977                 if (!devlink_try_get(devlink))
5978                         continue;
5979
5980                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5981                         goto retry;
5982
5983                 err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
5984                                                                &idx, start);
5985 retry:
5986                 devlink_put(devlink);
5987                 if (err)
5988                         goto out;
5989         }
5990 out:
5991         mutex_unlock(&devlink_mutex);
5992         cb->args[0] = idx;
5993         return msg->len;
5994 }
5995
5996 static int devlink_nl_cmd_region_del(struct sk_buff *skb,
5997                                      struct genl_info *info)
5998 {
5999         struct devlink *devlink = info->user_ptr[0];
6000         struct devlink_snapshot *snapshot;
6001         struct devlink_port *port = NULL;
6002         struct devlink_region *region;
6003         const char *region_name;
6004         unsigned int index;
6005         u32 snapshot_id;
6006
6007         if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
6008             !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
6009                 return -EINVAL;
6010
6011         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
6012         snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
6013
6014         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6015                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6016
6017                 port = devlink_port_get_by_index(devlink, index);
6018                 if (!port)
6019                         return -ENODEV;
6020         }
6021
6022         if (port)
6023                 region = devlink_port_region_get_by_name(port, region_name);
6024         else
6025                 region = devlink_region_get_by_name(devlink, region_name);
6026
6027         if (!region)
6028                 return -EINVAL;
6029
6030         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
6031         if (!snapshot)
6032                 return -EINVAL;
6033
6034         devlink_region_snapshot_del(region, snapshot);
6035         return 0;
6036 }
6037
6038 static int
6039 devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
6040 {
6041         struct devlink *devlink = info->user_ptr[0];
6042         struct devlink_snapshot *snapshot;
6043         struct devlink_port *port = NULL;
6044         struct nlattr *snapshot_id_attr;
6045         struct devlink_region *region;
6046         const char *region_name;
6047         unsigned int index;
6048         u32 snapshot_id;
6049         u8 *data;
6050         int err;
6051
6052         if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
6053                 NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
6054                 return -EINVAL;
6055         }
6056
6057         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
6058
6059         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6060                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6061
6062                 port = devlink_port_get_by_index(devlink, index);
6063                 if (!port)
6064                         return -ENODEV;
6065         }
6066
6067         if (port)
6068                 region = devlink_port_region_get_by_name(port, region_name);
6069         else
6070                 region = devlink_region_get_by_name(devlink, region_name);
6071
6072         if (!region) {
6073                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
6074                 return -EINVAL;
6075         }
6076
6077         if (!region->ops->snapshot) {
6078                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
6079                 return -EOPNOTSUPP;
6080         }
6081
6082         if (region->cur_snapshots == region->max_snapshots) {
6083                 NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
6084                 return -ENOSPC;
6085         }
6086
6087         snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
6088         if (snapshot_id_attr) {
6089                 snapshot_id = nla_get_u32(snapshot_id_attr);
6090
6091                 if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
6092                         NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
6093                         return -EEXIST;
6094                 }
6095
6096                 err = __devlink_snapshot_id_insert(devlink, snapshot_id);
6097                 if (err)
6098                         return err;
6099         } else {
6100                 err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
6101                 if (err) {
6102                         NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
6103                         return err;
6104                 }
6105         }
6106
6107         if (port)
6108                 err = region->port_ops->snapshot(port, region->port_ops,
6109                                                  info->extack, &data);
6110         else
6111                 err = region->ops->snapshot(devlink, region->ops,
6112                                             info->extack, &data);
6113         if (err)
6114                 goto err_snapshot_capture;
6115
6116         err = __devlink_region_snapshot_create(region, data, snapshot_id);
6117         if (err)
6118                 goto err_snapshot_create;
6119
6120         if (!snapshot_id_attr) {
6121                 struct sk_buff *msg;
6122
6123                 snapshot = devlink_region_snapshot_get_by_id(region,
6124                                                              snapshot_id);
6125                 if (WARN_ON(!snapshot))
6126                         return -EINVAL;
6127
6128                 msg = devlink_nl_region_notify_build(region, snapshot,
6129                                                      DEVLINK_CMD_REGION_NEW,
6130                                                      info->snd_portid,
6131                                                      info->snd_seq);
6132                 err = PTR_ERR_OR_ZERO(msg);
6133                 if (err)
6134                         goto err_notify;
6135
6136                 err = genlmsg_reply(msg, info);
6137                 if (err)
6138                         goto err_notify;
6139         }
6140
6141         return 0;
6142
6143 err_snapshot_create:
6144         region->ops->destructor(data);
6145 err_snapshot_capture:
6146         __devlink_snapshot_id_decrement(devlink, snapshot_id);
6147         return err;
6148
6149 err_notify:
6150         devlink_region_snapshot_del(region, snapshot);
6151         return err;
6152 }
6153
6154 static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
6155                                                  struct devlink *devlink,
6156                                                  u8 *chunk, u32 chunk_size,
6157                                                  u64 addr)
6158 {
6159         struct nlattr *chunk_attr;
6160         int err;
6161
6162         chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
6163         if (!chunk_attr)
6164                 return -EINVAL;
6165
6166         err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
6167         if (err)
6168                 goto nla_put_failure;
6169
6170         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
6171                                 DEVLINK_ATTR_PAD);
6172         if (err)
6173                 goto nla_put_failure;
6174
6175         nla_nest_end(msg, chunk_attr);
6176         return 0;
6177
6178 nla_put_failure:
6179         nla_nest_cancel(msg, chunk_attr);
6180         return err;
6181 }
6182
6183 #define DEVLINK_REGION_READ_CHUNK_SIZE 256
6184
6185 static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
6186                                                 struct devlink *devlink,
6187                                                 struct devlink_region *region,
6188                                                 struct nlattr **attrs,
6189                                                 u64 start_offset,
6190                                                 u64 end_offset,
6191                                                 u64 *new_offset)
6192 {
6193         struct devlink_snapshot *snapshot;
6194         u64 curr_offset = start_offset;
6195         u32 snapshot_id;
6196         int err = 0;
6197
6198         *new_offset = start_offset;
6199
6200         snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
6201         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
6202         if (!snapshot)
6203                 return -EINVAL;
6204
6205         while (curr_offset < end_offset) {
6206                 u32 data_size;
6207                 u8 *data;
6208
6209                 if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
6210                         data_size = end_offset - curr_offset;
6211                 else
6212                         data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
6213
6214                 data = &snapshot->data[curr_offset];
6215                 err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
6216                                                             data, data_size,
6217                                                             curr_offset);
6218                 if (err)
6219                         break;
6220
6221                 curr_offset += data_size;
6222         }
6223         *new_offset = curr_offset;
6224
6225         return err;
6226 }
6227
6228 static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
6229                                              struct netlink_callback *cb)
6230 {
6231         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
6232         u64 ret_offset, start_offset, end_offset = U64_MAX;
6233         struct nlattr **attrs = info->attrs;
6234         struct devlink_port *port = NULL;
6235         struct devlink_region *region;
6236         struct nlattr *chunks_attr;
6237         const char *region_name;
6238         struct devlink *devlink;
6239         unsigned int index;
6240         void *hdr;
6241         int err;
6242
6243         start_offset = *((u64 *)&cb->args[0]);
6244
6245         mutex_lock(&devlink_mutex);
6246         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
6247         if (IS_ERR(devlink)) {
6248                 err = PTR_ERR(devlink);
6249                 goto out_dev;
6250         }
6251
6252         mutex_lock(&devlink->lock);
6253
6254         if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
6255             !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
6256                 err = -EINVAL;
6257                 goto out_unlock;
6258         }
6259
6260         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6261                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6262
6263                 port = devlink_port_get_by_index(devlink, index);
6264                 if (!port) {
6265                         err = -ENODEV;
6266                         goto out_unlock;
6267                 }
6268         }
6269
6270         region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
6271
6272         if (port)
6273                 region = devlink_port_region_get_by_name(port, region_name);
6274         else
6275                 region = devlink_region_get_by_name(devlink, region_name);
6276
6277         if (!region) {
6278                 err = -EINVAL;
6279                 goto out_unlock;
6280         }
6281
6282         if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
6283             attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
6284                 if (!start_offset)
6285                         start_offset =
6286                                 nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
6287
6288                 end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
6289                 end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
6290         }
6291
6292         if (end_offset > region->size)
6293                 end_offset = region->size;
6294
6295         /* return 0 if there is no further data to read */
6296         if (start_offset == end_offset) {
6297                 err = 0;
6298                 goto out_unlock;
6299         }
6300
6301         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
6302                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
6303                           DEVLINK_CMD_REGION_READ);
6304         if (!hdr) {
6305                 err = -EMSGSIZE;
6306                 goto out_unlock;
6307         }
6308
6309         err = devlink_nl_put_handle(skb, devlink);
6310         if (err)
6311                 goto nla_put_failure;
6312
6313         if (region->port) {
6314                 err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
6315                                   region->port->index);
6316                 if (err)
6317                         goto nla_put_failure;
6318         }
6319
6320         err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
6321         if (err)
6322                 goto nla_put_failure;
6323
6324         chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
6325         if (!chunks_attr) {
6326                 err = -EMSGSIZE;
6327                 goto nla_put_failure;
6328         }
6329
6330         err = devlink_nl_region_read_snapshot_fill(skb, devlink,
6331                                                    region, attrs,
6332                                                    start_offset,
6333                                                    end_offset, &ret_offset);
6334
6335         if (err && err != -EMSGSIZE)
6336                 goto nla_put_failure;
6337
6338         /* Check if there was any progress done to prevent infinite loop */
6339         if (ret_offset == start_offset) {
6340                 err = -EINVAL;
6341                 goto nla_put_failure;
6342         }
6343
6344         *((u64 *)&cb->args[0]) = ret_offset;
6345
6346         nla_nest_end(skb, chunks_attr);
6347         genlmsg_end(skb, hdr);
6348         mutex_unlock(&devlink->lock);
6349         devlink_put(devlink);
6350         mutex_unlock(&devlink_mutex);
6351
6352         return skb->len;
6353
6354 nla_put_failure:
6355         genlmsg_cancel(skb, hdr);
6356 out_unlock:
6357         mutex_unlock(&devlink->lock);
6358         devlink_put(devlink);
6359 out_dev:
6360         mutex_unlock(&devlink_mutex);
6361         return err;
6362 }
6363
6364 struct devlink_info_req {
6365         struct sk_buff *msg;
6366 };
6367
6368 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
6369 {
6370         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
6371 }
6372 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
6373
6374 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
6375 {
6376         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
6377 }
6378 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
6379
6380 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
6381                                          const char *bsn)
6382 {
6383         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
6384                               bsn);
6385 }
6386 EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
6387
6388 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
6389                                     const char *version_name,
6390                                     const char *version_value)
6391 {
6392         struct nlattr *nest;
6393         int err;
6394
6395         nest = nla_nest_start_noflag(req->msg, attr);
6396         if (!nest)
6397                 return -EMSGSIZE;
6398
6399         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
6400                              version_name);
6401         if (err)
6402                 goto nla_put_failure;
6403
6404         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
6405                              version_value);
6406         if (err)
6407                 goto nla_put_failure;
6408
6409         nla_nest_end(req->msg, nest);
6410
6411         return 0;
6412
6413 nla_put_failure:
6414         nla_nest_cancel(req->msg, nest);
6415         return err;
6416 }
6417
6418 int devlink_info_version_fixed_put(struct devlink_info_req *req,
6419                                    const char *version_name,
6420                                    const char *version_value)
6421 {
6422         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
6423                                         version_name, version_value);
6424 }
6425 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
6426
6427 int devlink_info_version_stored_put(struct devlink_info_req *req,
6428                                     const char *version_name,
6429                                     const char *version_value)
6430 {
6431         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
6432                                         version_name, version_value);
6433 }
6434 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
6435
6436 int devlink_info_version_running_put(struct devlink_info_req *req,
6437                                      const char *version_name,
6438                                      const char *version_value)
6439 {
6440         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
6441                                         version_name, version_value);
6442 }
6443 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
6444
6445 static int
6446 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
6447                      enum devlink_command cmd, u32 portid,
6448                      u32 seq, int flags, struct netlink_ext_ack *extack)
6449 {
6450         struct devlink_info_req req;
6451         void *hdr;
6452         int err;
6453
6454         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6455         if (!hdr)
6456                 return -EMSGSIZE;
6457
6458         err = -EMSGSIZE;
6459         if (devlink_nl_put_handle(msg, devlink))
6460                 goto err_cancel_msg;
6461
6462         req.msg = msg;
6463         err = devlink->ops->info_get(devlink, &req, extack);
6464         if (err)
6465                 goto err_cancel_msg;
6466
6467         genlmsg_end(msg, hdr);
6468         return 0;
6469
6470 err_cancel_msg:
6471         genlmsg_cancel(msg, hdr);
6472         return err;
6473 }
6474
6475 static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
6476                                         struct genl_info *info)
6477 {
6478         struct devlink *devlink = info->user_ptr[0];
6479         struct sk_buff *msg;
6480         int err;
6481
6482         if (!devlink->ops->info_get)
6483                 return -EOPNOTSUPP;
6484
6485         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6486         if (!msg)
6487                 return -ENOMEM;
6488
6489         err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6490                                    info->snd_portid, info->snd_seq, 0,
6491                                    info->extack);
6492         if (err) {
6493                 nlmsg_free(msg);
6494                 return err;
6495         }
6496
6497         return genlmsg_reply(msg, info);
6498 }
6499
6500 static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
6501                                           struct netlink_callback *cb)
6502 {
6503         struct devlink *devlink;
6504         int start = cb->args[0];
6505         unsigned long index;
6506         int idx = 0;
6507         int err = 0;
6508
6509         mutex_lock(&devlink_mutex);
6510         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
6511                 if (!devlink_try_get(devlink))
6512                         continue;
6513
6514                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6515                         goto retry;
6516
6517                 if (idx < start || !devlink->ops->info_get)
6518                         goto inc;
6519
6520                 mutex_lock(&devlink->lock);
6521                 err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6522                                            NETLINK_CB(cb->skb).portid,
6523                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
6524                                            cb->extack);
6525                 mutex_unlock(&devlink->lock);
6526                 if (err == -EOPNOTSUPP)
6527                         err = 0;
6528                 else if (err) {
6529                         devlink_put(devlink);
6530                         break;
6531                 }
6532 inc:
6533                 idx++;
6534 retry:
6535                 devlink_put(devlink);
6536         }
6537         mutex_unlock(&devlink_mutex);
6538
6539         if (err != -EMSGSIZE)
6540                 return err;
6541
6542         cb->args[0] = idx;
6543         return msg->len;
6544 }
6545
6546 struct devlink_fmsg_item {
6547         struct list_head list;
6548         int attrtype;
6549         u8 nla_type;
6550         u16 len;
6551         int value[];
6552 };
6553
6554 struct devlink_fmsg {
6555         struct list_head item_list;
6556         bool putting_binary; /* This flag forces enclosing of binary data
6557                               * in an array brackets. It forces using
6558                               * of designated API:
6559                               * devlink_fmsg_binary_pair_nest_start()
6560                               * devlink_fmsg_binary_pair_nest_end()
6561                               */
6562 };
6563
6564 static struct devlink_fmsg *devlink_fmsg_alloc(void)
6565 {
6566         struct devlink_fmsg *fmsg;
6567
6568         fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
6569         if (!fmsg)
6570                 return NULL;
6571
6572         INIT_LIST_HEAD(&fmsg->item_list);
6573
6574         return fmsg;
6575 }
6576
6577 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
6578 {
6579         struct devlink_fmsg_item *item, *tmp;
6580
6581         list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
6582                 list_del(&item->list);
6583                 kfree(item);
6584         }
6585         kfree(fmsg);
6586 }
6587
6588 static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
6589                                     int attrtype)
6590 {
6591         struct devlink_fmsg_item *item;
6592
6593         item = kzalloc(sizeof(*item), GFP_KERNEL);
6594         if (!item)
6595                 return -ENOMEM;
6596
6597         item->attrtype = attrtype;
6598         list_add_tail(&item->list, &fmsg->item_list);
6599
6600         return 0;
6601 }
6602
6603 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
6604 {
6605         if (fmsg->putting_binary)
6606                 return -EINVAL;
6607
6608         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
6609 }
6610 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
6611
6612 static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
6613 {
6614         if (fmsg->putting_binary)
6615                 return -EINVAL;
6616
6617         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
6618 }
6619
6620 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
6621 {
6622         if (fmsg->putting_binary)
6623                 return -EINVAL;
6624
6625         return devlink_fmsg_nest_end(fmsg);
6626 }
6627 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
6628
6629 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
6630
6631 static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
6632 {
6633         struct devlink_fmsg_item *item;
6634
6635         if (fmsg->putting_binary)
6636                 return -EINVAL;
6637
6638         if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
6639                 return -EMSGSIZE;
6640
6641         item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
6642         if (!item)
6643                 return -ENOMEM;
6644
6645         item->nla_type = NLA_NUL_STRING;
6646         item->len = strlen(name) + 1;
6647         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
6648         memcpy(&item->value, name, item->len);
6649         list_add_tail(&item->list, &fmsg->item_list);
6650
6651         return 0;
6652 }
6653
6654 int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
6655 {
6656         int err;
6657
6658         if (fmsg->putting_binary)
6659                 return -EINVAL;
6660
6661         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
6662         if (err)
6663                 return err;
6664
6665         err = devlink_fmsg_put_name(fmsg, name);
6666         if (err)
6667                 return err;
6668
6669         return 0;
6670 }
6671 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
6672
6673 int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
6674 {
6675         if (fmsg->putting_binary)
6676                 return -EINVAL;
6677
6678         return devlink_fmsg_nest_end(fmsg);
6679 }
6680 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
6681
6682 int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
6683                                      const char *name)
6684 {
6685         int err;
6686
6687         if (fmsg->putting_binary)
6688                 return -EINVAL;
6689
6690         err = devlink_fmsg_pair_nest_start(fmsg, name);
6691         if (err)
6692                 return err;
6693
6694         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
6695         if (err)
6696                 return err;
6697
6698         return 0;
6699 }
6700 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
6701
6702 int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
6703 {
6704         int err;
6705
6706         if (fmsg->putting_binary)
6707                 return -EINVAL;
6708
6709         err = devlink_fmsg_nest_end(fmsg);
6710         if (err)
6711                 return err;
6712
6713         err = devlink_fmsg_nest_end(fmsg);
6714         if (err)
6715                 return err;
6716
6717         return 0;
6718 }
6719 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
6720
6721 int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
6722                                         const char *name)
6723 {
6724         int err;
6725
6726         err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
6727         if (err)
6728                 return err;
6729
6730         fmsg->putting_binary = true;
6731         return err;
6732 }
6733 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
6734
6735 int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
6736 {
6737         if (!fmsg->putting_binary)
6738                 return -EINVAL;
6739
6740         fmsg->putting_binary = false;
6741         return devlink_fmsg_arr_pair_nest_end(fmsg);
6742 }
6743 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
6744
6745 static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
6746                                   const void *value, u16 value_len,
6747                                   u8 value_nla_type)
6748 {
6749         struct devlink_fmsg_item *item;
6750
6751         if (value_len > DEVLINK_FMSG_MAX_SIZE)
6752                 return -EMSGSIZE;
6753
6754         item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
6755         if (!item)
6756                 return -ENOMEM;
6757
6758         item->nla_type = value_nla_type;
6759         item->len = value_len;
6760         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6761         memcpy(&item->value, value, item->len);
6762         list_add_tail(&item->list, &fmsg->item_list);
6763
6764         return 0;
6765 }
6766
6767 static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
6768 {
6769         if (fmsg->putting_binary)
6770                 return -EINVAL;
6771
6772         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
6773 }
6774
6775 static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
6776 {
6777         if (fmsg->putting_binary)
6778                 return -EINVAL;
6779
6780         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
6781 }
6782
6783 int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
6784 {
6785         if (fmsg->putting_binary)
6786                 return -EINVAL;
6787
6788         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
6789 }
6790 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
6791
6792 static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
6793 {
6794         if (fmsg->putting_binary)
6795                 return -EINVAL;
6796
6797         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
6798 }
6799
6800 int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
6801 {
6802         if (fmsg->putting_binary)
6803                 return -EINVAL;
6804
6805         return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
6806                                       NLA_NUL_STRING);
6807 }
6808 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
6809
6810 int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
6811                             u16 value_len)
6812 {
6813         if (!fmsg->putting_binary)
6814                 return -EINVAL;
6815
6816         return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
6817 }
6818 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
6819
6820 int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
6821                                bool value)
6822 {
6823         int err;
6824
6825         err = devlink_fmsg_pair_nest_start(fmsg, name);
6826         if (err)
6827                 return err;
6828
6829         err = devlink_fmsg_bool_put(fmsg, value);
6830         if (err)
6831                 return err;
6832
6833         err = devlink_fmsg_pair_nest_end(fmsg);
6834         if (err)
6835                 return err;
6836
6837         return 0;
6838 }
6839 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
6840
6841 int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
6842                              u8 value)
6843 {
6844         int err;
6845
6846         err = devlink_fmsg_pair_nest_start(fmsg, name);
6847         if (err)
6848                 return err;
6849
6850         err = devlink_fmsg_u8_put(fmsg, value);
6851         if (err)
6852                 return err;
6853
6854         err = devlink_fmsg_pair_nest_end(fmsg);
6855         if (err)
6856                 return err;
6857
6858         return 0;
6859 }
6860 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
6861
6862 int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
6863                               u32 value)
6864 {
6865         int err;
6866
6867         err = devlink_fmsg_pair_nest_start(fmsg, name);
6868         if (err)
6869                 return err;
6870
6871         err = devlink_fmsg_u32_put(fmsg, value);
6872         if (err)
6873                 return err;
6874
6875         err = devlink_fmsg_pair_nest_end(fmsg);
6876         if (err)
6877                 return err;
6878
6879         return 0;
6880 }
6881 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
6882
6883 int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
6884                               u64 value)
6885 {
6886         int err;
6887
6888         err = devlink_fmsg_pair_nest_start(fmsg, name);
6889         if (err)
6890                 return err;
6891
6892         err = devlink_fmsg_u64_put(fmsg, value);
6893         if (err)
6894                 return err;
6895
6896         err = devlink_fmsg_pair_nest_end(fmsg);
6897         if (err)
6898                 return err;
6899
6900         return 0;
6901 }
6902 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
6903
6904 int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
6905                                  const char *value)
6906 {
6907         int err;
6908
6909         err = devlink_fmsg_pair_nest_start(fmsg, name);
6910         if (err)
6911                 return err;
6912
6913         err = devlink_fmsg_string_put(fmsg, value);
6914         if (err)
6915                 return err;
6916
6917         err = devlink_fmsg_pair_nest_end(fmsg);
6918         if (err)
6919                 return err;
6920
6921         return 0;
6922 }
6923 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
6924
6925 int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
6926                                  const void *value, u32 value_len)
6927 {
6928         u32 data_size;
6929         int end_err;
6930         u32 offset;
6931         int err;
6932
6933         err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
6934         if (err)
6935                 return err;
6936
6937         for (offset = 0; offset < value_len; offset += data_size) {
6938                 data_size = value_len - offset;
6939                 if (data_size > DEVLINK_FMSG_MAX_SIZE)
6940                         data_size = DEVLINK_FMSG_MAX_SIZE;
6941                 err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
6942                 if (err)
6943                         break;
6944                 /* Exit from loop with a break (instead of
6945                  * return) to make sure putting_binary is turned off in
6946                  * devlink_fmsg_binary_pair_nest_end
6947                  */
6948         }
6949
6950         end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
6951         if (end_err)
6952                 err = end_err;
6953
6954         return err;
6955 }
6956 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
6957
6958 static int
6959 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6960 {
6961         switch (msg->nla_type) {
6962         case NLA_FLAG:
6963         case NLA_U8:
6964         case NLA_U32:
6965         case NLA_U64:
6966         case NLA_NUL_STRING:
6967         case NLA_BINARY:
6968                 return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
6969                                   msg->nla_type);
6970         default:
6971                 return -EINVAL;
6972         }
6973 }
6974
6975 static int
6976 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6977 {
6978         int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6979         u8 tmp;
6980
6981         switch (msg->nla_type) {
6982         case NLA_FLAG:
6983                 /* Always provide flag data, regardless of its value */
6984                 tmp = *(bool *) msg->value;
6985
6986                 return nla_put_u8(skb, attrtype, tmp);
6987         case NLA_U8:
6988                 return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
6989         case NLA_U32:
6990                 return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
6991         case NLA_U64:
6992                 return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
6993                                          DEVLINK_ATTR_PAD);
6994         case NLA_NUL_STRING:
6995                 return nla_put_string(skb, attrtype, (char *) &msg->value);
6996         case NLA_BINARY:
6997                 return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
6998         default:
6999                 return -EINVAL;
7000         }
7001 }
7002
7003 static int
7004 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
7005                          int *start)
7006 {
7007         struct devlink_fmsg_item *item;
7008         struct nlattr *fmsg_nlattr;
7009         int i = 0;
7010         int err;
7011
7012         fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
7013         if (!fmsg_nlattr)
7014                 return -EMSGSIZE;
7015
7016         list_for_each_entry(item, &fmsg->item_list, list) {
7017                 if (i < *start) {
7018                         i++;
7019                         continue;
7020                 }
7021
7022                 switch (item->attrtype) {
7023                 case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
7024                 case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
7025                 case DEVLINK_ATTR_FMSG_ARR_NEST_START:
7026                 case DEVLINK_ATTR_FMSG_NEST_END:
7027                         err = nla_put_flag(skb, item->attrtype);
7028                         break;
7029                 case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
7030                         err = devlink_fmsg_item_fill_type(item, skb);
7031                         if (err)
7032                                 break;
7033                         err = devlink_fmsg_item_fill_data(item, skb);
7034                         break;
7035                 case DEVLINK_ATTR_FMSG_OBJ_NAME:
7036                         err = nla_put_string(skb, item->attrtype,
7037                                              (char *) &item->value);
7038                         break;
7039                 default:
7040                         err = -EINVAL;
7041                         break;
7042                 }
7043                 if (!err)
7044                         *start = ++i;
7045                 else
7046                         break;
7047         }
7048
7049         nla_nest_end(skb, fmsg_nlattr);
7050         return err;
7051 }
7052
7053 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
7054                             struct genl_info *info,
7055                             enum devlink_command cmd, int flags)
7056 {
7057         struct nlmsghdr *nlh;
7058         struct sk_buff *skb;
7059         bool last = false;
7060         int index = 0;
7061         void *hdr;
7062         int err;
7063
7064         while (!last) {
7065                 int tmp_index = index;
7066
7067                 skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
7068                 if (!skb)
7069                         return -ENOMEM;
7070
7071                 hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
7072                                   &devlink_nl_family, flags | NLM_F_MULTI, cmd);
7073                 if (!hdr) {
7074                         err = -EMSGSIZE;
7075                         goto nla_put_failure;
7076                 }
7077
7078                 err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
7079                 if (!err)
7080                         last = true;
7081                 else if (err != -EMSGSIZE || tmp_index == index)
7082                         goto nla_put_failure;
7083
7084                 genlmsg_end(skb, hdr);
7085                 err = genlmsg_reply(skb, info);
7086                 if (err)
7087                         return err;
7088         }
7089
7090         skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
7091         if (!skb)
7092                 return -ENOMEM;
7093         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
7094                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
7095         if (!nlh) {
7096                 err = -EMSGSIZE;
7097                 goto nla_put_failure;
7098         }
7099
7100         return genlmsg_reply(skb, info);
7101
7102 nla_put_failure:
7103         nlmsg_free(skb);
7104         return err;
7105 }
7106
7107 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
7108                                struct netlink_callback *cb,
7109                                enum devlink_command cmd)
7110 {
7111         int index = cb->args[0];
7112         int tmp_index = index;
7113         void *hdr;
7114         int err;
7115
7116         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7117                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
7118         if (!hdr) {
7119                 err = -EMSGSIZE;
7120                 goto nla_put_failure;
7121         }
7122
7123         err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
7124         if ((err && err != -EMSGSIZE) || tmp_index == index)
7125                 goto nla_put_failure;
7126
7127         cb->args[0] = index;
7128         genlmsg_end(skb, hdr);
7129         return skb->len;
7130
7131 nla_put_failure:
7132         genlmsg_cancel(skb, hdr);
7133         return err;
7134 }
7135
7136 struct devlink_health_reporter {
7137         struct list_head list;
7138         void *priv;
7139         const struct devlink_health_reporter_ops *ops;
7140         struct devlink *devlink;
7141         struct devlink_port *devlink_port;
7142         struct devlink_fmsg *dump_fmsg;
7143         struct mutex dump_lock; /* lock parallel read/write from dump buffers */
7144         u64 graceful_period;
7145         bool auto_recover;
7146         bool auto_dump;
7147         u8 health_state;
7148         u64 dump_ts;
7149         u64 dump_real_ts;
7150         u64 error_count;
7151         u64 recovery_count;
7152         u64 last_recovery_ts;
7153         refcount_t refcount;
7154 };
7155
7156 void *
7157 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
7158 {
7159         return reporter->priv;
7160 }
7161 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
7162
7163 static struct devlink_health_reporter *
7164 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
7165                                        struct mutex *list_lock,
7166                                        const char *reporter_name)
7167 {
7168         struct devlink_health_reporter *reporter;
7169
7170         lockdep_assert_held(list_lock);
7171         list_for_each_entry(reporter, reporter_list, list)
7172                 if (!strcmp(reporter->ops->name, reporter_name))
7173                         return reporter;
7174         return NULL;
7175 }
7176
7177 static struct devlink_health_reporter *
7178 devlink_health_reporter_find_by_name(struct devlink *devlink,
7179                                      const char *reporter_name)
7180 {
7181         return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
7182                                                       &devlink->reporters_lock,
7183                                                       reporter_name);
7184 }
7185
7186 static struct devlink_health_reporter *
7187 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
7188                                           const char *reporter_name)
7189 {
7190         return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
7191                                                       &devlink_port->reporters_lock,
7192                                                       reporter_name);
7193 }
7194
7195 static struct devlink_health_reporter *
7196 __devlink_health_reporter_create(struct devlink *devlink,
7197                                  const struct devlink_health_reporter_ops *ops,
7198                                  u64 graceful_period, void *priv)
7199 {
7200         struct devlink_health_reporter *reporter;
7201
7202         if (WARN_ON(graceful_period && !ops->recover))
7203                 return ERR_PTR(-EINVAL);
7204
7205         reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
7206         if (!reporter)
7207                 return ERR_PTR(-ENOMEM);
7208
7209         reporter->priv = priv;
7210         reporter->ops = ops;
7211         reporter->devlink = devlink;
7212         reporter->graceful_period = graceful_period;
7213         reporter->auto_recover = !!ops->recover;
7214         reporter->auto_dump = !!ops->dump;
7215         mutex_init(&reporter->dump_lock);
7216         refcount_set(&reporter->refcount, 1);
7217         return reporter;
7218 }
7219
7220 /**
7221  *      devlink_port_health_reporter_create - create devlink health reporter for
7222  *                                            specified port instance
7223  *
7224  *      @port: devlink_port which should contain the new reporter
7225  *      @ops: ops
7226  *      @graceful_period: to avoid recovery loops, in msecs
7227  *      @priv: priv
7228  */
7229 struct devlink_health_reporter *
7230 devlink_port_health_reporter_create(struct devlink_port *port,
7231                                     const struct devlink_health_reporter_ops *ops,
7232                                     u64 graceful_period, void *priv)
7233 {
7234         struct devlink_health_reporter *reporter;
7235
7236         mutex_lock(&port->reporters_lock);
7237         if (__devlink_health_reporter_find_by_name(&port->reporter_list,
7238                                                    &port->reporters_lock, ops->name)) {
7239                 reporter = ERR_PTR(-EEXIST);
7240                 goto unlock;
7241         }
7242
7243         reporter = __devlink_health_reporter_create(port->devlink, ops,
7244                                                     graceful_period, priv);
7245         if (IS_ERR(reporter))
7246                 goto unlock;
7247
7248         reporter->devlink_port = port;
7249         list_add_tail(&reporter->list, &port->reporter_list);
7250 unlock:
7251         mutex_unlock(&port->reporters_lock);
7252         return reporter;
7253 }
7254 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
7255
7256 /**
7257  *      devlink_health_reporter_create - create devlink health reporter
7258  *
7259  *      @devlink: devlink
7260  *      @ops: ops
7261  *      @graceful_period: to avoid recovery loops, in msecs
7262  *      @priv: priv
7263  */
7264 struct devlink_health_reporter *
7265 devlink_health_reporter_create(struct devlink *devlink,
7266                                const struct devlink_health_reporter_ops *ops,
7267                                u64 graceful_period, void *priv)
7268 {
7269         struct devlink_health_reporter *reporter;
7270
7271         mutex_lock(&devlink->reporters_lock);
7272         if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
7273                 reporter = ERR_PTR(-EEXIST);
7274                 goto unlock;
7275         }
7276
7277         reporter = __devlink_health_reporter_create(devlink, ops,
7278                                                     graceful_period, priv);
7279         if (IS_ERR(reporter))
7280                 goto unlock;
7281
7282         list_add_tail(&reporter->list, &devlink->reporter_list);
7283 unlock:
7284         mutex_unlock(&devlink->reporters_lock);
7285         return reporter;
7286 }
7287 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
7288
7289 static void
7290 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
7291 {
7292         mutex_destroy(&reporter->dump_lock);
7293         if (reporter->dump_fmsg)
7294                 devlink_fmsg_free(reporter->dump_fmsg);
7295         kfree(reporter);
7296 }
7297
7298 static void
7299 devlink_health_reporter_put(struct devlink_health_reporter *reporter)
7300 {
7301         if (refcount_dec_and_test(&reporter->refcount))
7302                 devlink_health_reporter_free(reporter);
7303 }
7304
7305 static void
7306 __devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
7307 {
7308         list_del(&reporter->list);
7309         devlink_health_reporter_put(reporter);
7310 }
7311
7312 /**
7313  *      devlink_health_reporter_destroy - destroy devlink health reporter
7314  *
7315  *      @reporter: devlink health reporter to destroy
7316  */
7317 void
7318 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
7319 {
7320         struct mutex *lock = &reporter->devlink->reporters_lock;
7321
7322         mutex_lock(lock);
7323         __devlink_health_reporter_destroy(reporter);
7324         mutex_unlock(lock);
7325 }
7326 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
7327
7328 /**
7329  *      devlink_port_health_reporter_destroy - destroy devlink port health reporter
7330  *
7331  *      @reporter: devlink health reporter to destroy
7332  */
7333 void
7334 devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
7335 {
7336         struct mutex *lock = &reporter->devlink_port->reporters_lock;
7337
7338         mutex_lock(lock);
7339         __devlink_health_reporter_destroy(reporter);
7340         mutex_unlock(lock);
7341 }
7342 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
7343
7344 static int
7345 devlink_nl_health_reporter_fill(struct sk_buff *msg,
7346                                 struct devlink_health_reporter *reporter,
7347                                 enum devlink_command cmd, u32 portid,
7348                                 u32 seq, int flags)
7349 {
7350         struct devlink *devlink = reporter->devlink;
7351         struct nlattr *reporter_attr;
7352         void *hdr;
7353
7354         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7355         if (!hdr)
7356                 return -EMSGSIZE;
7357
7358         if (devlink_nl_put_handle(msg, devlink))
7359                 goto genlmsg_cancel;
7360
7361         if (reporter->devlink_port) {
7362                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
7363                         goto genlmsg_cancel;
7364         }
7365         reporter_attr = nla_nest_start_noflag(msg,
7366                                               DEVLINK_ATTR_HEALTH_REPORTER);
7367         if (!reporter_attr)
7368                 goto genlmsg_cancel;
7369         if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
7370                            reporter->ops->name))
7371                 goto reporter_nest_cancel;
7372         if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
7373                        reporter->health_state))
7374                 goto reporter_nest_cancel;
7375         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
7376                               reporter->error_count, DEVLINK_ATTR_PAD))
7377                 goto reporter_nest_cancel;
7378         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
7379                               reporter->recovery_count, DEVLINK_ATTR_PAD))
7380                 goto reporter_nest_cancel;
7381         if (reporter->ops->recover &&
7382             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
7383                               reporter->graceful_period,
7384                               DEVLINK_ATTR_PAD))
7385                 goto reporter_nest_cancel;
7386         if (reporter->ops->recover &&
7387             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
7388                        reporter->auto_recover))
7389                 goto reporter_nest_cancel;
7390         if (reporter->dump_fmsg &&
7391             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
7392                               jiffies_to_msecs(reporter->dump_ts),
7393                               DEVLINK_ATTR_PAD))
7394                 goto reporter_nest_cancel;
7395         if (reporter->dump_fmsg &&
7396             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
7397                               reporter->dump_real_ts, DEVLINK_ATTR_PAD))
7398                 goto reporter_nest_cancel;
7399         if (reporter->ops->dump &&
7400             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
7401                        reporter->auto_dump))
7402                 goto reporter_nest_cancel;
7403
7404         nla_nest_end(msg, reporter_attr);
7405         genlmsg_end(msg, hdr);
7406         return 0;
7407
7408 reporter_nest_cancel:
7409         nla_nest_end(msg, reporter_attr);
7410 genlmsg_cancel:
7411         genlmsg_cancel(msg, hdr);
7412         return -EMSGSIZE;
7413 }
7414
7415 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
7416                                    enum devlink_command cmd)
7417 {
7418         struct devlink *devlink = reporter->devlink;
7419         struct sk_buff *msg;
7420         int err;
7421
7422         WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7423         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
7424
7425         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7426         if (!msg)
7427                 return;
7428
7429         err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
7430         if (err) {
7431                 nlmsg_free(msg);
7432                 return;
7433         }
7434
7435         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
7436                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
7437 }
7438
7439 void
7440 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
7441 {
7442         reporter->recovery_count++;
7443         reporter->last_recovery_ts = jiffies;
7444 }
7445 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
7446
7447 static int
7448 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
7449                                 void *priv_ctx, struct netlink_ext_ack *extack)
7450 {
7451         int err;
7452
7453         if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
7454                 return 0;
7455
7456         if (!reporter->ops->recover)
7457                 return -EOPNOTSUPP;
7458
7459         err = reporter->ops->recover(reporter, priv_ctx, extack);
7460         if (err)
7461                 return err;
7462
7463         devlink_health_reporter_recovery_done(reporter);
7464         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
7465         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7466
7467         return 0;
7468 }
7469
7470 static void
7471 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
7472 {
7473         if (!reporter->dump_fmsg)
7474                 return;
7475         devlink_fmsg_free(reporter->dump_fmsg);
7476         reporter->dump_fmsg = NULL;
7477 }
7478
7479 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
7480                                   void *priv_ctx,
7481                                   struct netlink_ext_ack *extack)
7482 {
7483         int err;
7484
7485         if (!reporter->ops->dump)
7486                 return 0;
7487
7488         if (reporter->dump_fmsg)
7489                 return 0;
7490
7491         reporter->dump_fmsg = devlink_fmsg_alloc();
7492         if (!reporter->dump_fmsg) {
7493                 err = -ENOMEM;
7494                 return err;
7495         }
7496
7497         err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
7498         if (err)
7499                 goto dump_err;
7500
7501         err = reporter->ops->dump(reporter, reporter->dump_fmsg,
7502                                   priv_ctx, extack);
7503         if (err)
7504                 goto dump_err;
7505
7506         err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
7507         if (err)
7508                 goto dump_err;
7509
7510         reporter->dump_ts = jiffies;
7511         reporter->dump_real_ts = ktime_get_real_ns();
7512
7513         return 0;
7514
7515 dump_err:
7516         devlink_health_dump_clear(reporter);
7517         return err;
7518 }
7519
7520 int devlink_health_report(struct devlink_health_reporter *reporter,
7521                           const char *msg, void *priv_ctx)
7522 {
7523         enum devlink_health_reporter_state prev_health_state;
7524         struct devlink *devlink = reporter->devlink;
7525         unsigned long recover_ts_threshold;
7526
7527         /* write a log message of the current error */
7528         WARN_ON(!msg);
7529         trace_devlink_health_report(devlink, reporter->ops->name, msg);
7530         reporter->error_count++;
7531         prev_health_state = reporter->health_state;
7532         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7533         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7534
7535         /* abort if the previous error wasn't recovered */
7536         recover_ts_threshold = reporter->last_recovery_ts +
7537                                msecs_to_jiffies(reporter->graceful_period);
7538         if (reporter->auto_recover &&
7539             (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
7540              (reporter->last_recovery_ts && reporter->recovery_count &&
7541               time_is_after_jiffies(recover_ts_threshold)))) {
7542                 trace_devlink_health_recover_aborted(devlink,
7543                                                      reporter->ops->name,
7544                                                      reporter->health_state,
7545                                                      jiffies -
7546                                                      reporter->last_recovery_ts);
7547                 return -ECANCELED;
7548         }
7549
7550         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7551
7552         if (reporter->auto_dump) {
7553                 mutex_lock(&reporter->dump_lock);
7554                 /* store current dump of current error, for later analysis */
7555                 devlink_health_do_dump(reporter, priv_ctx, NULL);
7556                 mutex_unlock(&reporter->dump_lock);
7557         }
7558
7559         if (reporter->auto_recover)
7560                 return devlink_health_reporter_recover(reporter,
7561                                                        priv_ctx, NULL);
7562
7563         return 0;
7564 }
7565 EXPORT_SYMBOL_GPL(devlink_health_report);
7566
7567 static struct devlink_health_reporter *
7568 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
7569                                        struct nlattr **attrs)
7570 {
7571         struct devlink_health_reporter *reporter;
7572         struct devlink_port *devlink_port;
7573         char *reporter_name;
7574
7575         if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
7576                 return NULL;
7577
7578         reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
7579         devlink_port = devlink_port_get_from_attrs(devlink, attrs);
7580         if (IS_ERR(devlink_port)) {
7581                 mutex_lock(&devlink->reporters_lock);
7582                 reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
7583                 if (reporter)
7584                         refcount_inc(&reporter->refcount);
7585                 mutex_unlock(&devlink->reporters_lock);
7586         } else {
7587                 mutex_lock(&devlink_port->reporters_lock);
7588                 reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
7589                 if (reporter)
7590                         refcount_inc(&reporter->refcount);
7591                 mutex_unlock(&devlink_port->reporters_lock);
7592         }
7593
7594         return reporter;
7595 }
7596
7597 static struct devlink_health_reporter *
7598 devlink_health_reporter_get_from_info(struct devlink *devlink,
7599                                       struct genl_info *info)
7600 {
7601         return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
7602 }
7603
7604 static struct devlink_health_reporter *
7605 devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
7606 {
7607         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
7608         struct devlink_health_reporter *reporter;
7609         struct nlattr **attrs = info->attrs;
7610         struct devlink *devlink;
7611
7612         mutex_lock(&devlink_mutex);
7613         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
7614         if (IS_ERR(devlink))
7615                 goto unlock;
7616
7617         reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
7618         devlink_put(devlink);
7619         mutex_unlock(&devlink_mutex);
7620         return reporter;
7621 unlock:
7622         mutex_unlock(&devlink_mutex);
7623         return NULL;
7624 }
7625
7626 void
7627 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
7628                                      enum devlink_health_reporter_state state)
7629 {
7630         if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
7631                     state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
7632                 return;
7633
7634         if (reporter->health_state == state)
7635                 return;
7636
7637         reporter->health_state = state;
7638         trace_devlink_health_reporter_state_update(reporter->devlink,
7639                                                    reporter->ops->name, state);
7640         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7641 }
7642 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
7643
7644 static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
7645                                                    struct genl_info *info)
7646 {
7647         struct devlink *devlink = info->user_ptr[0];
7648         struct devlink_health_reporter *reporter;
7649         struct sk_buff *msg;
7650         int err;
7651
7652         reporter = devlink_health_reporter_get_from_info(devlink, info);
7653         if (!reporter)
7654                 return -EINVAL;
7655
7656         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7657         if (!msg) {
7658                 err = -ENOMEM;
7659                 goto out;
7660         }
7661
7662         err = devlink_nl_health_reporter_fill(msg, reporter,
7663                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
7664                                               info->snd_portid, info->snd_seq,
7665                                               0);
7666         if (err) {
7667                 nlmsg_free(msg);
7668                 goto out;
7669         }
7670
7671         err = genlmsg_reply(msg, info);
7672 out:
7673         devlink_health_reporter_put(reporter);
7674         return err;
7675 }
7676
7677 static int
7678 devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
7679                                           struct netlink_callback *cb)
7680 {
7681         struct devlink_health_reporter *reporter;
7682         struct devlink_port *port;
7683         struct devlink *devlink;
7684         int start = cb->args[0];
7685         unsigned long index;
7686         int idx = 0;
7687         int err;
7688
7689         mutex_lock(&devlink_mutex);
7690         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7691                 if (!devlink_try_get(devlink))
7692                         continue;
7693
7694                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7695                         goto retry_rep;
7696
7697                 mutex_lock(&devlink->reporters_lock);
7698                 list_for_each_entry(reporter, &devlink->reporter_list,
7699                                     list) {
7700                         if (idx < start) {
7701                                 idx++;
7702                                 continue;
7703                         }
7704                         err = devlink_nl_health_reporter_fill(
7705                                 msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
7706                                 NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7707                                 NLM_F_MULTI);
7708                         if (err) {
7709                                 mutex_unlock(&devlink->reporters_lock);
7710                                 devlink_put(devlink);
7711                                 goto out;
7712                         }
7713                         idx++;
7714                 }
7715                 mutex_unlock(&devlink->reporters_lock);
7716 retry_rep:
7717                 devlink_put(devlink);
7718         }
7719
7720         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7721                 if (!devlink_try_get(devlink))
7722                         continue;
7723
7724                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7725                         goto retry_port;
7726
7727                 mutex_lock(&devlink->lock);
7728                 list_for_each_entry(port, &devlink->port_list, list) {
7729                         mutex_lock(&port->reporters_lock);
7730                         list_for_each_entry(reporter, &port->reporter_list, list) {
7731                                 if (idx < start) {
7732                                         idx++;
7733                                         continue;
7734                                 }
7735                                 err = devlink_nl_health_reporter_fill(
7736                                         msg, reporter,
7737                                         DEVLINK_CMD_HEALTH_REPORTER_GET,
7738                                         NETLINK_CB(cb->skb).portid,
7739                                         cb->nlh->nlmsg_seq, NLM_F_MULTI);
7740                                 if (err) {
7741                                         mutex_unlock(&port->reporters_lock);
7742                                         mutex_unlock(&devlink->lock);
7743                                         devlink_put(devlink);
7744                                         goto out;
7745                                 }
7746                                 idx++;
7747                         }
7748                         mutex_unlock(&port->reporters_lock);
7749                 }
7750                 mutex_unlock(&devlink->lock);
7751 retry_port:
7752                 devlink_put(devlink);
7753         }
7754 out:
7755         mutex_unlock(&devlink_mutex);
7756
7757         cb->args[0] = idx;
7758         return msg->len;
7759 }
7760
7761 static int
7762 devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
7763                                         struct genl_info *info)
7764 {
7765         struct devlink *devlink = info->user_ptr[0];
7766         struct devlink_health_reporter *reporter;
7767         int err;
7768
7769         reporter = devlink_health_reporter_get_from_info(devlink, info);
7770         if (!reporter)
7771                 return -EINVAL;
7772
7773         if (!reporter->ops->recover &&
7774             (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
7775              info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
7776                 err = -EOPNOTSUPP;
7777                 goto out;
7778         }
7779         if (!reporter->ops->dump &&
7780             info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
7781                 err = -EOPNOTSUPP;
7782                 goto out;
7783         }
7784
7785         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
7786                 reporter->graceful_period =
7787                         nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
7788
7789         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
7790                 reporter->auto_recover =
7791                         nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
7792
7793         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
7794                 reporter->auto_dump =
7795                 nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
7796
7797         devlink_health_reporter_put(reporter);
7798         return 0;
7799 out:
7800         devlink_health_reporter_put(reporter);
7801         return err;
7802 }
7803
7804 static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
7805                                                        struct genl_info *info)
7806 {
7807         struct devlink *devlink = info->user_ptr[0];
7808         struct devlink_health_reporter *reporter;
7809         int err;
7810
7811         reporter = devlink_health_reporter_get_from_info(devlink, info);
7812         if (!reporter)
7813                 return -EINVAL;
7814
7815         err = devlink_health_reporter_recover(reporter, NULL, info->extack);
7816
7817         devlink_health_reporter_put(reporter);
7818         return err;
7819 }
7820
7821 static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
7822                                                         struct genl_info *info)
7823 {
7824         struct devlink *devlink = info->user_ptr[0];
7825         struct devlink_health_reporter *reporter;
7826         struct devlink_fmsg *fmsg;
7827         int err;
7828
7829         reporter = devlink_health_reporter_get_from_info(devlink, info);
7830         if (!reporter)
7831                 return -EINVAL;
7832
7833         if (!reporter->ops->diagnose) {
7834                 devlink_health_reporter_put(reporter);
7835                 return -EOPNOTSUPP;
7836         }
7837
7838         fmsg = devlink_fmsg_alloc();
7839         if (!fmsg) {
7840                 devlink_health_reporter_put(reporter);
7841                 return -ENOMEM;
7842         }
7843
7844         err = devlink_fmsg_obj_nest_start(fmsg);
7845         if (err)
7846                 goto out;
7847
7848         err = reporter->ops->diagnose(reporter, fmsg, info->extack);
7849         if (err)
7850                 goto out;
7851
7852         err = devlink_fmsg_obj_nest_end(fmsg);
7853         if (err)
7854                 goto out;
7855
7856         err = devlink_fmsg_snd(fmsg, info,
7857                                DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
7858
7859 out:
7860         devlink_fmsg_free(fmsg);
7861         devlink_health_reporter_put(reporter);
7862         return err;
7863 }
7864
7865 static int
7866 devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
7867                                                struct netlink_callback *cb)
7868 {
7869         struct devlink_health_reporter *reporter;
7870         u64 start = cb->args[0];
7871         int err;
7872
7873         reporter = devlink_health_reporter_get_from_cb(cb);
7874         if (!reporter)
7875                 return -EINVAL;
7876
7877         if (!reporter->ops->dump) {
7878                 err = -EOPNOTSUPP;
7879                 goto out;
7880         }
7881         mutex_lock(&reporter->dump_lock);
7882         if (!start) {
7883                 err = devlink_health_do_dump(reporter, NULL, cb->extack);
7884                 if (err)
7885                         goto unlock;
7886                 cb->args[1] = reporter->dump_ts;
7887         }
7888         if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
7889                 NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
7890                 err = -EAGAIN;
7891                 goto unlock;
7892         }
7893
7894         err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
7895                                   DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
7896 unlock:
7897         mutex_unlock(&reporter->dump_lock);
7898 out:
7899         devlink_health_reporter_put(reporter);
7900         return err;
7901 }
7902
7903 static int
7904 devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
7905                                                struct genl_info *info)
7906 {
7907         struct devlink *devlink = info->user_ptr[0];
7908         struct devlink_health_reporter *reporter;
7909
7910         reporter = devlink_health_reporter_get_from_info(devlink, info);
7911         if (!reporter)
7912                 return -EINVAL;
7913
7914         if (!reporter->ops->dump) {
7915                 devlink_health_reporter_put(reporter);
7916                 return -EOPNOTSUPP;
7917         }
7918
7919         mutex_lock(&reporter->dump_lock);
7920         devlink_health_dump_clear(reporter);
7921         mutex_unlock(&reporter->dump_lock);
7922         devlink_health_reporter_put(reporter);
7923         return 0;
7924 }
7925
7926 static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
7927                                                     struct genl_info *info)
7928 {
7929         struct devlink *devlink = info->user_ptr[0];
7930         struct devlink_health_reporter *reporter;
7931         int err;
7932
7933         reporter = devlink_health_reporter_get_from_info(devlink, info);
7934         if (!reporter)
7935                 return -EINVAL;
7936
7937         if (!reporter->ops->test) {
7938                 devlink_health_reporter_put(reporter);
7939                 return -EOPNOTSUPP;
7940         }
7941
7942         err = reporter->ops->test(reporter, info->extack);
7943
7944         devlink_health_reporter_put(reporter);
7945         return err;
7946 }
7947
7948 struct devlink_stats {
7949         u64_stats_t rx_bytes;
7950         u64_stats_t rx_packets;
7951         struct u64_stats_sync syncp;
7952 };
7953
7954 /**
7955  * struct devlink_trap_policer_item - Packet trap policer attributes.
7956  * @policer: Immutable packet trap policer attributes.
7957  * @rate: Rate in packets / sec.
7958  * @burst: Burst size in packets.
7959  * @list: trap_policer_list member.
7960  *
7961  * Describes packet trap policer attributes. Created by devlink during trap
7962  * policer registration.
7963  */
7964 struct devlink_trap_policer_item {
7965         const struct devlink_trap_policer *policer;
7966         u64 rate;
7967         u64 burst;
7968         struct list_head list;
7969 };
7970
7971 /**
7972  * struct devlink_trap_group_item - Packet trap group attributes.
7973  * @group: Immutable packet trap group attributes.
7974  * @policer_item: Associated policer item. Can be NULL.
7975  * @list: trap_group_list member.
7976  * @stats: Trap group statistics.
7977  *
7978  * Describes packet trap group attributes. Created by devlink during trap
7979  * group registration.
7980  */
7981 struct devlink_trap_group_item {
7982         const struct devlink_trap_group *group;
7983         struct devlink_trap_policer_item *policer_item;
7984         struct list_head list;
7985         struct devlink_stats __percpu *stats;
7986 };
7987
7988 /**
7989  * struct devlink_trap_item - Packet trap attributes.
7990  * @trap: Immutable packet trap attributes.
7991  * @group_item: Associated group item.
7992  * @list: trap_list member.
7993  * @action: Trap action.
7994  * @stats: Trap statistics.
7995  * @priv: Driver private information.
7996  *
7997  * Describes both mutable and immutable packet trap attributes. Created by
7998  * devlink during trap registration and used for all trap related operations.
7999  */
8000 struct devlink_trap_item {
8001         const struct devlink_trap *trap;
8002         struct devlink_trap_group_item *group_item;
8003         struct list_head list;
8004         enum devlink_trap_action action;
8005         struct devlink_stats __percpu *stats;
8006         void *priv;
8007 };
8008
8009 static struct devlink_trap_policer_item *
8010 devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
8011 {
8012         struct devlink_trap_policer_item *policer_item;
8013
8014         list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
8015                 if (policer_item->policer->id == id)
8016                         return policer_item;
8017         }
8018
8019         return NULL;
8020 }
8021
8022 static struct devlink_trap_item *
8023 devlink_trap_item_lookup(struct devlink *devlink, const char *name)
8024 {
8025         struct devlink_trap_item *trap_item;
8026
8027         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8028                 if (!strcmp(trap_item->trap->name, name))
8029                         return trap_item;
8030         }
8031
8032         return NULL;
8033 }
8034
8035 static struct devlink_trap_item *
8036 devlink_trap_item_get_from_info(struct devlink *devlink,
8037                                 struct genl_info *info)
8038 {
8039         struct nlattr *attr;
8040
8041         if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
8042                 return NULL;
8043         attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
8044
8045         return devlink_trap_item_lookup(devlink, nla_data(attr));
8046 }
8047
8048 static int
8049 devlink_trap_action_get_from_info(struct genl_info *info,
8050                                   enum devlink_trap_action *p_trap_action)
8051 {
8052         u8 val;
8053
8054         val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
8055         switch (val) {
8056         case DEVLINK_TRAP_ACTION_DROP:
8057         case DEVLINK_TRAP_ACTION_TRAP:
8058         case DEVLINK_TRAP_ACTION_MIRROR:
8059                 *p_trap_action = val;
8060                 break;
8061         default:
8062                 return -EINVAL;
8063         }
8064
8065         return 0;
8066 }
8067
8068 static int devlink_trap_metadata_put(struct sk_buff *msg,
8069                                      const struct devlink_trap *trap)
8070 {
8071         struct nlattr *attr;
8072
8073         attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
8074         if (!attr)
8075                 return -EMSGSIZE;
8076
8077         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
8078             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
8079                 goto nla_put_failure;
8080         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
8081             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
8082                 goto nla_put_failure;
8083
8084         nla_nest_end(msg, attr);
8085
8086         return 0;
8087
8088 nla_put_failure:
8089         nla_nest_cancel(msg, attr);
8090         return -EMSGSIZE;
8091 }
8092
8093 static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
8094                                     struct devlink_stats *stats)
8095 {
8096         int i;
8097
8098         memset(stats, 0, sizeof(*stats));
8099         for_each_possible_cpu(i) {
8100                 struct devlink_stats *cpu_stats;
8101                 u64 rx_packets, rx_bytes;
8102                 unsigned int start;
8103
8104                 cpu_stats = per_cpu_ptr(trap_stats, i);
8105                 do {
8106                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
8107                         rx_packets = u64_stats_read(&cpu_stats->rx_packets);
8108                         rx_bytes = u64_stats_read(&cpu_stats->rx_bytes);
8109                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
8110
8111                 u64_stats_add(&stats->rx_packets, rx_packets);
8112                 u64_stats_add(&stats->rx_bytes, rx_bytes);
8113         }
8114 }
8115
8116 static int
8117 devlink_trap_group_stats_put(struct sk_buff *msg,
8118                              struct devlink_stats __percpu *trap_stats)
8119 {
8120         struct devlink_stats stats;
8121         struct nlattr *attr;
8122
8123         devlink_trap_stats_read(trap_stats, &stats);
8124
8125         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8126         if (!attr)
8127                 return -EMSGSIZE;
8128
8129         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
8130                               u64_stats_read(&stats.rx_packets),
8131                               DEVLINK_ATTR_PAD))
8132                 goto nla_put_failure;
8133
8134         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
8135                               u64_stats_read(&stats.rx_bytes),
8136                               DEVLINK_ATTR_PAD))
8137                 goto nla_put_failure;
8138
8139         nla_nest_end(msg, attr);
8140
8141         return 0;
8142
8143 nla_put_failure:
8144         nla_nest_cancel(msg, attr);
8145         return -EMSGSIZE;
8146 }
8147
8148 static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
8149                                   const struct devlink_trap_item *trap_item)
8150 {
8151         struct devlink_stats stats;
8152         struct nlattr *attr;
8153         u64 drops = 0;
8154         int err;
8155
8156         if (devlink->ops->trap_drop_counter_get) {
8157                 err = devlink->ops->trap_drop_counter_get(devlink,
8158                                                           trap_item->trap,
8159                                                           &drops);
8160                 if (err)
8161                         return err;
8162         }
8163
8164         devlink_trap_stats_read(trap_item->stats, &stats);
8165
8166         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8167         if (!attr)
8168                 return -EMSGSIZE;
8169
8170         if (devlink->ops->trap_drop_counter_get &&
8171             nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8172                               DEVLINK_ATTR_PAD))
8173                 goto nla_put_failure;
8174
8175         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
8176                               u64_stats_read(&stats.rx_packets),
8177                               DEVLINK_ATTR_PAD))
8178                 goto nla_put_failure;
8179
8180         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
8181                               u64_stats_read(&stats.rx_bytes),
8182                               DEVLINK_ATTR_PAD))
8183                 goto nla_put_failure;
8184
8185         nla_nest_end(msg, attr);
8186
8187         return 0;
8188
8189 nla_put_failure:
8190         nla_nest_cancel(msg, attr);
8191         return -EMSGSIZE;
8192 }
8193
8194 static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
8195                                 const struct devlink_trap_item *trap_item,
8196                                 enum devlink_command cmd, u32 portid, u32 seq,
8197                                 int flags)
8198 {
8199         struct devlink_trap_group_item *group_item = trap_item->group_item;
8200         void *hdr;
8201         int err;
8202
8203         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8204         if (!hdr)
8205                 return -EMSGSIZE;
8206
8207         if (devlink_nl_put_handle(msg, devlink))
8208                 goto nla_put_failure;
8209
8210         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
8211                            group_item->group->name))
8212                 goto nla_put_failure;
8213
8214         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
8215                 goto nla_put_failure;
8216
8217         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
8218                 goto nla_put_failure;
8219
8220         if (trap_item->trap->generic &&
8221             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
8222                 goto nla_put_failure;
8223
8224         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
8225                 goto nla_put_failure;
8226
8227         err = devlink_trap_metadata_put(msg, trap_item->trap);
8228         if (err)
8229                 goto nla_put_failure;
8230
8231         err = devlink_trap_stats_put(msg, devlink, trap_item);
8232         if (err)
8233                 goto nla_put_failure;
8234
8235         genlmsg_end(msg, hdr);
8236
8237         return 0;
8238
8239 nla_put_failure:
8240         genlmsg_cancel(msg, hdr);
8241         return -EMSGSIZE;
8242 }
8243
8244 static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
8245                                         struct genl_info *info)
8246 {
8247         struct netlink_ext_ack *extack = info->extack;
8248         struct devlink *devlink = info->user_ptr[0];
8249         struct devlink_trap_item *trap_item;
8250         struct sk_buff *msg;
8251         int err;
8252
8253         if (list_empty(&devlink->trap_list))
8254                 return -EOPNOTSUPP;
8255
8256         trap_item = devlink_trap_item_get_from_info(devlink, info);
8257         if (!trap_item) {
8258                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
8259                 return -ENOENT;
8260         }
8261
8262         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8263         if (!msg)
8264                 return -ENOMEM;
8265
8266         err = devlink_nl_trap_fill(msg, devlink, trap_item,
8267                                    DEVLINK_CMD_TRAP_NEW, info->snd_portid,
8268                                    info->snd_seq, 0);
8269         if (err)
8270                 goto err_trap_fill;
8271
8272         return genlmsg_reply(msg, info);
8273
8274 err_trap_fill:
8275         nlmsg_free(msg);
8276         return err;
8277 }
8278
8279 static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
8280                                           struct netlink_callback *cb)
8281 {
8282         struct devlink_trap_item *trap_item;
8283         struct devlink *devlink;
8284         int start = cb->args[0];
8285         unsigned long index;
8286         int idx = 0;
8287         int err;
8288
8289         mutex_lock(&devlink_mutex);
8290         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8291                 if (!devlink_try_get(devlink))
8292                         continue;
8293
8294                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8295                         goto retry;
8296
8297                 mutex_lock(&devlink->lock);
8298                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8299                         if (idx < start) {
8300                                 idx++;
8301                                 continue;
8302                         }
8303                         err = devlink_nl_trap_fill(msg, devlink, trap_item,
8304                                                    DEVLINK_CMD_TRAP_NEW,
8305                                                    NETLINK_CB(cb->skb).portid,
8306                                                    cb->nlh->nlmsg_seq,
8307                                                    NLM_F_MULTI);
8308                         if (err) {
8309                                 mutex_unlock(&devlink->lock);
8310                                 devlink_put(devlink);
8311                                 goto out;
8312                         }
8313                         idx++;
8314                 }
8315                 mutex_unlock(&devlink->lock);
8316 retry:
8317                 devlink_put(devlink);
8318         }
8319 out:
8320         mutex_unlock(&devlink_mutex);
8321
8322         cb->args[0] = idx;
8323         return msg->len;
8324 }
8325
8326 static int __devlink_trap_action_set(struct devlink *devlink,
8327                                      struct devlink_trap_item *trap_item,
8328                                      enum devlink_trap_action trap_action,
8329                                      struct netlink_ext_ack *extack)
8330 {
8331         int err;
8332
8333         if (trap_item->action != trap_action &&
8334             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
8335                 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
8336                 return 0;
8337         }
8338
8339         err = devlink->ops->trap_action_set(devlink, trap_item->trap,
8340                                             trap_action, extack);
8341         if (err)
8342                 return err;
8343
8344         trap_item->action = trap_action;
8345
8346         return 0;
8347 }
8348
8349 static int devlink_trap_action_set(struct devlink *devlink,
8350                                    struct devlink_trap_item *trap_item,
8351                                    struct genl_info *info)
8352 {
8353         enum devlink_trap_action trap_action;
8354         int err;
8355
8356         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8357                 return 0;
8358
8359         err = devlink_trap_action_get_from_info(info, &trap_action);
8360         if (err) {
8361                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8362                 return -EINVAL;
8363         }
8364
8365         return __devlink_trap_action_set(devlink, trap_item, trap_action,
8366                                          info->extack);
8367 }
8368
8369 static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
8370                                         struct genl_info *info)
8371 {
8372         struct netlink_ext_ack *extack = info->extack;
8373         struct devlink *devlink = info->user_ptr[0];
8374         struct devlink_trap_item *trap_item;
8375
8376         if (list_empty(&devlink->trap_list))
8377                 return -EOPNOTSUPP;
8378
8379         trap_item = devlink_trap_item_get_from_info(devlink, info);
8380         if (!trap_item) {
8381                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
8382                 return -ENOENT;
8383         }
8384
8385         return devlink_trap_action_set(devlink, trap_item, info);
8386 }
8387
8388 static struct devlink_trap_group_item *
8389 devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
8390 {
8391         struct devlink_trap_group_item *group_item;
8392
8393         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
8394                 if (!strcmp(group_item->group->name, name))
8395                         return group_item;
8396         }
8397
8398         return NULL;
8399 }
8400
8401 static struct devlink_trap_group_item *
8402 devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
8403 {
8404         struct devlink_trap_group_item *group_item;
8405
8406         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
8407                 if (group_item->group->id == id)
8408                         return group_item;
8409         }
8410
8411         return NULL;
8412 }
8413
8414 static struct devlink_trap_group_item *
8415 devlink_trap_group_item_get_from_info(struct devlink *devlink,
8416                                       struct genl_info *info)
8417 {
8418         char *name;
8419
8420         if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
8421                 return NULL;
8422         name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
8423
8424         return devlink_trap_group_item_lookup(devlink, name);
8425 }
8426
8427 static int
8428 devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
8429                            const struct devlink_trap_group_item *group_item,
8430                            enum devlink_command cmd, u32 portid, u32 seq,
8431                            int flags)
8432 {
8433         void *hdr;
8434         int err;
8435
8436         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8437         if (!hdr)
8438                 return -EMSGSIZE;
8439
8440         if (devlink_nl_put_handle(msg, devlink))
8441                 goto nla_put_failure;
8442
8443         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
8444                            group_item->group->name))
8445                 goto nla_put_failure;
8446
8447         if (group_item->group->generic &&
8448             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
8449                 goto nla_put_failure;
8450
8451         if (group_item->policer_item &&
8452             nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8453                         group_item->policer_item->policer->id))
8454                 goto nla_put_failure;
8455
8456         err = devlink_trap_group_stats_put(msg, group_item->stats);
8457         if (err)
8458                 goto nla_put_failure;
8459
8460         genlmsg_end(msg, hdr);
8461
8462         return 0;
8463
8464 nla_put_failure:
8465         genlmsg_cancel(msg, hdr);
8466         return -EMSGSIZE;
8467 }
8468
8469 static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
8470                                               struct genl_info *info)
8471 {
8472         struct netlink_ext_ack *extack = info->extack;
8473         struct devlink *devlink = info->user_ptr[0];
8474         struct devlink_trap_group_item *group_item;
8475         struct sk_buff *msg;
8476         int err;
8477
8478         if (list_empty(&devlink->trap_group_list))
8479                 return -EOPNOTSUPP;
8480
8481         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8482         if (!group_item) {
8483                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8484                 return -ENOENT;
8485         }
8486
8487         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8488         if (!msg)
8489                 return -ENOMEM;
8490
8491         err = devlink_nl_trap_group_fill(msg, devlink, group_item,
8492                                          DEVLINK_CMD_TRAP_GROUP_NEW,
8493                                          info->snd_portid, info->snd_seq, 0);
8494         if (err)
8495                 goto err_trap_group_fill;
8496
8497         return genlmsg_reply(msg, info);
8498
8499 err_trap_group_fill:
8500         nlmsg_free(msg);
8501         return err;
8502 }
8503
8504 static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
8505                                                 struct netlink_callback *cb)
8506 {
8507         enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
8508         struct devlink_trap_group_item *group_item;
8509         u32 portid = NETLINK_CB(cb->skb).portid;
8510         struct devlink *devlink;
8511         int start = cb->args[0];
8512         unsigned long index;
8513         int idx = 0;
8514         int err;
8515
8516         mutex_lock(&devlink_mutex);
8517         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8518                 if (!devlink_try_get(devlink))
8519                         continue;
8520
8521                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8522                         goto retry;
8523
8524                 mutex_lock(&devlink->lock);
8525                 list_for_each_entry(group_item, &devlink->trap_group_list,
8526                                     list) {
8527                         if (idx < start) {
8528                                 idx++;
8529                                 continue;
8530                         }
8531                         err = devlink_nl_trap_group_fill(msg, devlink,
8532                                                          group_item, cmd,
8533                                                          portid,
8534                                                          cb->nlh->nlmsg_seq,
8535                                                          NLM_F_MULTI);
8536                         if (err) {
8537                                 mutex_unlock(&devlink->lock);
8538                                 devlink_put(devlink);
8539                                 goto out;
8540                         }
8541                         idx++;
8542                 }
8543                 mutex_unlock(&devlink->lock);
8544 retry:
8545                 devlink_put(devlink);
8546         }
8547 out:
8548         mutex_unlock(&devlink_mutex);
8549
8550         cb->args[0] = idx;
8551         return msg->len;
8552 }
8553
8554 static int
8555 __devlink_trap_group_action_set(struct devlink *devlink,
8556                                 struct devlink_trap_group_item *group_item,
8557                                 enum devlink_trap_action trap_action,
8558                                 struct netlink_ext_ack *extack)
8559 {
8560         const char *group_name = group_item->group->name;
8561         struct devlink_trap_item *trap_item;
8562         int err;
8563
8564         if (devlink->ops->trap_group_action_set) {
8565                 err = devlink->ops->trap_group_action_set(devlink, group_item->group,
8566                                                           trap_action, extack);
8567                 if (err)
8568                         return err;
8569
8570                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8571                         if (strcmp(trap_item->group_item->group->name, group_name))
8572                                 continue;
8573                         if (trap_item->action != trap_action &&
8574                             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
8575                                 continue;
8576                         trap_item->action = trap_action;
8577                 }
8578
8579                 return 0;
8580         }
8581
8582         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8583                 if (strcmp(trap_item->group_item->group->name, group_name))
8584                         continue;
8585                 err = __devlink_trap_action_set(devlink, trap_item,
8586                                                 trap_action, extack);
8587                 if (err)
8588                         return err;
8589         }
8590
8591         return 0;
8592 }
8593
8594 static int
8595 devlink_trap_group_action_set(struct devlink *devlink,
8596                               struct devlink_trap_group_item *group_item,
8597                               struct genl_info *info, bool *p_modified)
8598 {
8599         enum devlink_trap_action trap_action;
8600         int err;
8601
8602         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8603                 return 0;
8604
8605         err = devlink_trap_action_get_from_info(info, &trap_action);
8606         if (err) {
8607                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8608                 return -EINVAL;
8609         }
8610
8611         err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
8612                                               info->extack);
8613         if (err)
8614                 return err;
8615
8616         *p_modified = true;
8617
8618         return 0;
8619 }
8620
8621 static int devlink_trap_group_set(struct devlink *devlink,
8622                                   struct devlink_trap_group_item *group_item,
8623                                   struct genl_info *info)
8624 {
8625         struct devlink_trap_policer_item *policer_item;
8626         struct netlink_ext_ack *extack = info->extack;
8627         const struct devlink_trap_policer *policer;
8628         struct nlattr **attrs = info->attrs;
8629         int err;
8630
8631         if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8632                 return 0;
8633
8634         if (!devlink->ops->trap_group_set)
8635                 return -EOPNOTSUPP;
8636
8637         policer_item = group_item->policer_item;
8638         if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
8639                 u32 policer_id;
8640
8641                 policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8642                 policer_item = devlink_trap_policer_item_lookup(devlink,
8643                                                                 policer_id);
8644                 if (policer_id && !policer_item) {
8645                         NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8646                         return -ENOENT;
8647                 }
8648         }
8649         policer = policer_item ? policer_item->policer : NULL;
8650
8651         err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
8652                                            extack);
8653         if (err)
8654                 return err;
8655
8656         group_item->policer_item = policer_item;
8657
8658         return 0;
8659 }
8660
8661 static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
8662                                               struct genl_info *info)
8663 {
8664         struct netlink_ext_ack *extack = info->extack;
8665         struct devlink *devlink = info->user_ptr[0];
8666         struct devlink_trap_group_item *group_item;
8667         bool modified = false;
8668         int err;
8669
8670         if (list_empty(&devlink->trap_group_list))
8671                 return -EOPNOTSUPP;
8672
8673         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8674         if (!group_item) {
8675                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8676                 return -ENOENT;
8677         }
8678
8679         err = devlink_trap_group_action_set(devlink, group_item, info,
8680                                             &modified);
8681         if (err)
8682                 return err;
8683
8684         err = devlink_trap_group_set(devlink, group_item, info);
8685         if (err)
8686                 goto err_trap_group_set;
8687
8688         return 0;
8689
8690 err_trap_group_set:
8691         if (modified)
8692                 NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
8693         return err;
8694 }
8695
8696 static struct devlink_trap_policer_item *
8697 devlink_trap_policer_item_get_from_info(struct devlink *devlink,
8698                                         struct genl_info *info)
8699 {
8700         u32 id;
8701
8702         if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8703                 return NULL;
8704         id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8705
8706         return devlink_trap_policer_item_lookup(devlink, id);
8707 }
8708
8709 static int
8710 devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
8711                                const struct devlink_trap_policer *policer)
8712 {
8713         struct nlattr *attr;
8714         u64 drops;
8715         int err;
8716
8717         if (!devlink->ops->trap_policer_counter_get)
8718                 return 0;
8719
8720         err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
8721         if (err)
8722                 return err;
8723
8724         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8725         if (!attr)
8726                 return -EMSGSIZE;
8727
8728         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8729                               DEVLINK_ATTR_PAD))
8730                 goto nla_put_failure;
8731
8732         nla_nest_end(msg, attr);
8733
8734         return 0;
8735
8736 nla_put_failure:
8737         nla_nest_cancel(msg, attr);
8738         return -EMSGSIZE;
8739 }
8740
8741 static int
8742 devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
8743                              const struct devlink_trap_policer_item *policer_item,
8744                              enum devlink_command cmd, u32 portid, u32 seq,
8745                              int flags)
8746 {
8747         void *hdr;
8748         int err;
8749
8750         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8751         if (!hdr)
8752                 return -EMSGSIZE;
8753
8754         if (devlink_nl_put_handle(msg, devlink))
8755                 goto nla_put_failure;
8756
8757         if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8758                         policer_item->policer->id))
8759                 goto nla_put_failure;
8760
8761         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
8762                               policer_item->rate, DEVLINK_ATTR_PAD))
8763                 goto nla_put_failure;
8764
8765         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
8766                               policer_item->burst, DEVLINK_ATTR_PAD))
8767                 goto nla_put_failure;
8768
8769         err = devlink_trap_policer_stats_put(msg, devlink,
8770                                              policer_item->policer);
8771         if (err)
8772                 goto nla_put_failure;
8773
8774         genlmsg_end(msg, hdr);
8775
8776         return 0;
8777
8778 nla_put_failure:
8779         genlmsg_cancel(msg, hdr);
8780         return -EMSGSIZE;
8781 }
8782
8783 static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
8784                                                 struct genl_info *info)
8785 {
8786         struct devlink_trap_policer_item *policer_item;
8787         struct netlink_ext_ack *extack = info->extack;
8788         struct devlink *devlink = info->user_ptr[0];
8789         struct sk_buff *msg;
8790         int err;
8791
8792         if (list_empty(&devlink->trap_policer_list))
8793                 return -EOPNOTSUPP;
8794
8795         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8796         if (!policer_item) {
8797                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8798                 return -ENOENT;
8799         }
8800
8801         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8802         if (!msg)
8803                 return -ENOMEM;
8804
8805         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
8806                                            DEVLINK_CMD_TRAP_POLICER_NEW,
8807                                            info->snd_portid, info->snd_seq, 0);
8808         if (err)
8809                 goto err_trap_policer_fill;
8810
8811         return genlmsg_reply(msg, info);
8812
8813 err_trap_policer_fill:
8814         nlmsg_free(msg);
8815         return err;
8816 }
8817
8818 static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
8819                                                   struct netlink_callback *cb)
8820 {
8821         enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
8822         struct devlink_trap_policer_item *policer_item;
8823         u32 portid = NETLINK_CB(cb->skb).portid;
8824         struct devlink *devlink;
8825         int start = cb->args[0];
8826         unsigned long index;
8827         int idx = 0;
8828         int err;
8829
8830         mutex_lock(&devlink_mutex);
8831         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8832                 if (!devlink_try_get(devlink))
8833                         continue;
8834
8835                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8836                         goto retry;
8837
8838                 mutex_lock(&devlink->lock);
8839                 list_for_each_entry(policer_item, &devlink->trap_policer_list,
8840                                     list) {
8841                         if (idx < start) {
8842                                 idx++;
8843                                 continue;
8844                         }
8845                         err = devlink_nl_trap_policer_fill(msg, devlink,
8846                                                            policer_item, cmd,
8847                                                            portid,
8848                                                            cb->nlh->nlmsg_seq,
8849                                                            NLM_F_MULTI);
8850                         if (err) {
8851                                 mutex_unlock(&devlink->lock);
8852                                 devlink_put(devlink);
8853                                 goto out;
8854                         }
8855                         idx++;
8856                 }
8857                 mutex_unlock(&devlink->lock);
8858 retry:
8859                 devlink_put(devlink);
8860         }
8861 out:
8862         mutex_unlock(&devlink_mutex);
8863
8864         cb->args[0] = idx;
8865         return msg->len;
8866 }
8867
8868 static int
8869 devlink_trap_policer_set(struct devlink *devlink,
8870                          struct devlink_trap_policer_item *policer_item,
8871                          struct genl_info *info)
8872 {
8873         struct netlink_ext_ack *extack = info->extack;
8874         struct nlattr **attrs = info->attrs;
8875         u64 rate, burst;
8876         int err;
8877
8878         rate = policer_item->rate;
8879         burst = policer_item->burst;
8880
8881         if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
8882                 rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
8883
8884         if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
8885                 burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
8886
8887         if (rate < policer_item->policer->min_rate) {
8888                 NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
8889                 return -EINVAL;
8890         }
8891
8892         if (rate > policer_item->policer->max_rate) {
8893                 NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
8894                 return -EINVAL;
8895         }
8896
8897         if (burst < policer_item->policer->min_burst) {
8898                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
8899                 return -EINVAL;
8900         }
8901
8902         if (burst > policer_item->policer->max_burst) {
8903                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
8904                 return -EINVAL;
8905         }
8906
8907         err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
8908                                              rate, burst, info->extack);
8909         if (err)
8910                 return err;
8911
8912         policer_item->rate = rate;
8913         policer_item->burst = burst;
8914
8915         return 0;
8916 }
8917
8918 static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
8919                                                 struct genl_info *info)
8920 {
8921         struct devlink_trap_policer_item *policer_item;
8922         struct netlink_ext_ack *extack = info->extack;
8923         struct devlink *devlink = info->user_ptr[0];
8924
8925         if (list_empty(&devlink->trap_policer_list))
8926                 return -EOPNOTSUPP;
8927
8928         if (!devlink->ops->trap_policer_set)
8929                 return -EOPNOTSUPP;
8930
8931         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8932         if (!policer_item) {
8933                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8934                 return -ENOENT;
8935         }
8936
8937         return devlink_trap_policer_set(devlink, policer_item, info);
8938 }
8939
8940 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
8941         [DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
8942                 DEVLINK_ATTR_TRAP_POLICER_ID },
8943         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
8944         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
8945         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
8946         [DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
8947                                                     DEVLINK_PORT_TYPE_IB),
8948         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
8949         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
8950         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
8951         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
8952         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
8953         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
8954         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
8955         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
8956         [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
8957                                                        DEVLINK_ESWITCH_MODE_SWITCHDEV),
8958         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
8959         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
8960         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
8961         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
8962         [DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
8963         [DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
8964         [DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
8965         [DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
8966         [DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
8967         [DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
8968         [DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
8969         [DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
8970         [DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
8971         [DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
8972         [DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
8973         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
8974         [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
8975         [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
8976         [DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
8977                 NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
8978         [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
8979         [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
8980         [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
8981         [DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
8982         [DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
8983         [DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
8984         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
8985         [DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
8986         [DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
8987         [DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
8988         [DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
8989         [DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
8990                                                         DEVLINK_RELOAD_ACTION_MAX),
8991         [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
8992         [DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
8993         [DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
8994         [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
8995         [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
8996         [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
8997         [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
8998         [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
8999         [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
9000         [DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
9001         [DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 },
9002         [DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING },
9003 };
9004
9005 static const struct genl_small_ops devlink_nl_ops[] = {
9006         {
9007                 .cmd = DEVLINK_CMD_GET,
9008                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9009                 .doit = devlink_nl_cmd_get_doit,
9010                 .dumpit = devlink_nl_cmd_get_dumpit,
9011                 /* can be retrieved by unprivileged users */
9012         },
9013         {
9014                 .cmd = DEVLINK_CMD_PORT_GET,
9015                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9016                 .doit = devlink_nl_cmd_port_get_doit,
9017                 .dumpit = devlink_nl_cmd_port_get_dumpit,
9018                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9019                 /* can be retrieved by unprivileged users */
9020         },
9021         {
9022                 .cmd = DEVLINK_CMD_PORT_SET,
9023                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9024                 .doit = devlink_nl_cmd_port_set_doit,
9025                 .flags = GENL_ADMIN_PERM,
9026                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9027         },
9028         {
9029                 .cmd = DEVLINK_CMD_RATE_GET,
9030                 .doit = devlink_nl_cmd_rate_get_doit,
9031                 .dumpit = devlink_nl_cmd_rate_get_dumpit,
9032                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
9033                 /* can be retrieved by unprivileged users */
9034         },
9035         {
9036                 .cmd = DEVLINK_CMD_RATE_SET,
9037                 .doit = devlink_nl_cmd_rate_set_doit,
9038                 .flags = GENL_ADMIN_PERM,
9039                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
9040         },
9041         {
9042                 .cmd = DEVLINK_CMD_RATE_NEW,
9043                 .doit = devlink_nl_cmd_rate_new_doit,
9044                 .flags = GENL_ADMIN_PERM,
9045         },
9046         {
9047                 .cmd = DEVLINK_CMD_RATE_DEL,
9048                 .doit = devlink_nl_cmd_rate_del_doit,
9049                 .flags = GENL_ADMIN_PERM,
9050                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
9051         },
9052         {
9053                 .cmd = DEVLINK_CMD_PORT_SPLIT,
9054                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9055                 .doit = devlink_nl_cmd_port_split_doit,
9056                 .flags = GENL_ADMIN_PERM,
9057                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9058         },
9059         {
9060                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
9061                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9062                 .doit = devlink_nl_cmd_port_unsplit_doit,
9063                 .flags = GENL_ADMIN_PERM,
9064                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9065         },
9066         {
9067                 .cmd = DEVLINK_CMD_PORT_NEW,
9068                 .doit = devlink_nl_cmd_port_new_doit,
9069                 .flags = GENL_ADMIN_PERM,
9070                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
9071         },
9072         {
9073                 .cmd = DEVLINK_CMD_PORT_DEL,
9074                 .doit = devlink_nl_cmd_port_del_doit,
9075                 .flags = GENL_ADMIN_PERM,
9076                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
9077         },
9078         {
9079                 .cmd = DEVLINK_CMD_LINECARD_GET,
9080                 .doit = devlink_nl_cmd_linecard_get_doit,
9081                 .dumpit = devlink_nl_cmd_linecard_get_dumpit,
9082                 .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
9083                 /* can be retrieved by unprivileged users */
9084         },
9085         {
9086                 .cmd = DEVLINK_CMD_LINECARD_SET,
9087                 .doit = devlink_nl_cmd_linecard_set_doit,
9088                 .flags = GENL_ADMIN_PERM,
9089                 .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
9090         },
9091         {
9092                 .cmd = DEVLINK_CMD_SB_GET,
9093                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9094                 .doit = devlink_nl_cmd_sb_get_doit,
9095                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
9096                 /* can be retrieved by unprivileged users */
9097         },
9098         {
9099                 .cmd = DEVLINK_CMD_SB_POOL_GET,
9100                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9101                 .doit = devlink_nl_cmd_sb_pool_get_doit,
9102                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
9103                 /* can be retrieved by unprivileged users */
9104         },
9105         {
9106                 .cmd = DEVLINK_CMD_SB_POOL_SET,
9107                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9108                 .doit = devlink_nl_cmd_sb_pool_set_doit,
9109                 .flags = GENL_ADMIN_PERM,
9110         },
9111         {
9112                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
9113                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9114                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
9115                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
9116                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9117                 /* can be retrieved by unprivileged users */
9118         },
9119         {
9120                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
9121                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9122                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
9123                 .flags = GENL_ADMIN_PERM,
9124                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9125         },
9126         {
9127                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
9128                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9129                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
9130                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
9131                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9132                 /* can be retrieved by unprivileged users */
9133         },
9134         {
9135                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
9136                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9137                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
9138                 .flags = GENL_ADMIN_PERM,
9139                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9140         },
9141         {
9142                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
9143                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9144                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
9145                 .flags = GENL_ADMIN_PERM,
9146         },
9147         {
9148                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
9149                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9150                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
9151                 .flags = GENL_ADMIN_PERM,
9152         },
9153         {
9154                 .cmd = DEVLINK_CMD_ESWITCH_GET,
9155                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9156                 .doit = devlink_nl_cmd_eswitch_get_doit,
9157                 .flags = GENL_ADMIN_PERM,
9158         },
9159         {
9160                 .cmd = DEVLINK_CMD_ESWITCH_SET,
9161                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9162                 .doit = devlink_nl_cmd_eswitch_set_doit,
9163                 .flags = GENL_ADMIN_PERM,
9164         },
9165         {
9166                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
9167                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9168                 .doit = devlink_nl_cmd_dpipe_table_get,
9169                 /* can be retrieved by unprivileged users */
9170         },
9171         {
9172                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
9173                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9174                 .doit = devlink_nl_cmd_dpipe_entries_get,
9175                 /* can be retrieved by unprivileged users */
9176         },
9177         {
9178                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
9179                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9180                 .doit = devlink_nl_cmd_dpipe_headers_get,
9181                 /* can be retrieved by unprivileged users */
9182         },
9183         {
9184                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
9185                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9186                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
9187                 .flags = GENL_ADMIN_PERM,
9188         },
9189         {
9190                 .cmd = DEVLINK_CMD_RESOURCE_SET,
9191                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9192                 .doit = devlink_nl_cmd_resource_set,
9193                 .flags = GENL_ADMIN_PERM,
9194         },
9195         {
9196                 .cmd = DEVLINK_CMD_RESOURCE_DUMP,
9197                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9198                 .doit = devlink_nl_cmd_resource_dump,
9199                 /* can be retrieved by unprivileged users */
9200         },
9201         {
9202                 .cmd = DEVLINK_CMD_RELOAD,
9203                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9204                 .doit = devlink_nl_cmd_reload,
9205                 .flags = GENL_ADMIN_PERM,
9206                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
9207         },
9208         {
9209                 .cmd = DEVLINK_CMD_PARAM_GET,
9210                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9211                 .doit = devlink_nl_cmd_param_get_doit,
9212                 .dumpit = devlink_nl_cmd_param_get_dumpit,
9213                 /* can be retrieved by unprivileged users */
9214         },
9215         {
9216                 .cmd = DEVLINK_CMD_PARAM_SET,
9217                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9218                 .doit = devlink_nl_cmd_param_set_doit,
9219                 .flags = GENL_ADMIN_PERM,
9220         },
9221         {
9222                 .cmd = DEVLINK_CMD_PORT_PARAM_GET,
9223                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9224                 .doit = devlink_nl_cmd_port_param_get_doit,
9225                 .dumpit = devlink_nl_cmd_port_param_get_dumpit,
9226                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9227                 /* can be retrieved by unprivileged users */
9228         },
9229         {
9230                 .cmd = DEVLINK_CMD_PORT_PARAM_SET,
9231                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9232                 .doit = devlink_nl_cmd_port_param_set_doit,
9233                 .flags = GENL_ADMIN_PERM,
9234                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9235         },
9236         {
9237                 .cmd = DEVLINK_CMD_REGION_GET,
9238                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9239                 .doit = devlink_nl_cmd_region_get_doit,
9240                 .dumpit = devlink_nl_cmd_region_get_dumpit,
9241                 .flags = GENL_ADMIN_PERM,
9242         },
9243         {
9244                 .cmd = DEVLINK_CMD_REGION_NEW,
9245                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9246                 .doit = devlink_nl_cmd_region_new,
9247                 .flags = GENL_ADMIN_PERM,
9248         },
9249         {
9250                 .cmd = DEVLINK_CMD_REGION_DEL,
9251                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9252                 .doit = devlink_nl_cmd_region_del,
9253                 .flags = GENL_ADMIN_PERM,
9254         },
9255         {
9256                 .cmd = DEVLINK_CMD_REGION_READ,
9257                 .validate = GENL_DONT_VALIDATE_STRICT |
9258                             GENL_DONT_VALIDATE_DUMP_STRICT,
9259                 .dumpit = devlink_nl_cmd_region_read_dumpit,
9260                 .flags = GENL_ADMIN_PERM,
9261         },
9262         {
9263                 .cmd = DEVLINK_CMD_INFO_GET,
9264                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9265                 .doit = devlink_nl_cmd_info_get_doit,
9266                 .dumpit = devlink_nl_cmd_info_get_dumpit,
9267                 /* can be retrieved by unprivileged users */
9268         },
9269         {
9270                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
9271                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9272                 .doit = devlink_nl_cmd_health_reporter_get_doit,
9273                 .dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
9274                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9275                                   DEVLINK_NL_FLAG_NO_LOCK,
9276                 /* can be retrieved by unprivileged users */
9277         },
9278         {
9279                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
9280                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9281                 .doit = devlink_nl_cmd_health_reporter_set_doit,
9282                 .flags = GENL_ADMIN_PERM,
9283                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9284                                   DEVLINK_NL_FLAG_NO_LOCK,
9285         },
9286         {
9287                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
9288                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9289                 .doit = devlink_nl_cmd_health_reporter_recover_doit,
9290                 .flags = GENL_ADMIN_PERM,
9291                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9292                                   DEVLINK_NL_FLAG_NO_LOCK,
9293         },
9294         {
9295                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
9296                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9297                 .doit = devlink_nl_cmd_health_reporter_diagnose_doit,
9298                 .flags = GENL_ADMIN_PERM,
9299                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9300                                   DEVLINK_NL_FLAG_NO_LOCK,
9301         },
9302         {
9303                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
9304                 .validate = GENL_DONT_VALIDATE_STRICT |
9305                             GENL_DONT_VALIDATE_DUMP_STRICT,
9306                 .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
9307                 .flags = GENL_ADMIN_PERM,
9308         },
9309         {
9310                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
9311                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9312                 .doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
9313                 .flags = GENL_ADMIN_PERM,
9314                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9315                                   DEVLINK_NL_FLAG_NO_LOCK,
9316         },
9317         {
9318                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
9319                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9320                 .doit = devlink_nl_cmd_health_reporter_test_doit,
9321                 .flags = GENL_ADMIN_PERM,
9322                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9323                                   DEVLINK_NL_FLAG_NO_LOCK,
9324         },
9325         {
9326                 .cmd = DEVLINK_CMD_FLASH_UPDATE,
9327                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9328                 .doit = devlink_nl_cmd_flash_update,
9329                 .flags = GENL_ADMIN_PERM,
9330         },
9331         {
9332                 .cmd = DEVLINK_CMD_TRAP_GET,
9333                 .doit = devlink_nl_cmd_trap_get_doit,
9334                 .dumpit = devlink_nl_cmd_trap_get_dumpit,
9335                 /* can be retrieved by unprivileged users */
9336         },
9337         {
9338                 .cmd = DEVLINK_CMD_TRAP_SET,
9339                 .doit = devlink_nl_cmd_trap_set_doit,
9340                 .flags = GENL_ADMIN_PERM,
9341         },
9342         {
9343                 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
9344                 .doit = devlink_nl_cmd_trap_group_get_doit,
9345                 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
9346                 /* can be retrieved by unprivileged users */
9347         },
9348         {
9349                 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
9350                 .doit = devlink_nl_cmd_trap_group_set_doit,
9351                 .flags = GENL_ADMIN_PERM,
9352         },
9353         {
9354                 .cmd = DEVLINK_CMD_TRAP_POLICER_GET,
9355                 .doit = devlink_nl_cmd_trap_policer_get_doit,
9356                 .dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
9357                 /* can be retrieved by unprivileged users */
9358         },
9359         {
9360                 .cmd = DEVLINK_CMD_TRAP_POLICER_SET,
9361                 .doit = devlink_nl_cmd_trap_policer_set_doit,
9362                 .flags = GENL_ADMIN_PERM,
9363         },
9364 };
9365
9366 static struct genl_family devlink_nl_family __ro_after_init = {
9367         .name           = DEVLINK_GENL_NAME,
9368         .version        = DEVLINK_GENL_VERSION,
9369         .maxattr        = DEVLINK_ATTR_MAX,
9370         .policy = devlink_nl_policy,
9371         .netnsok        = true,
9372         .pre_doit       = devlink_nl_pre_doit,
9373         .post_doit      = devlink_nl_post_doit,
9374         .module         = THIS_MODULE,
9375         .small_ops      = devlink_nl_ops,
9376         .n_small_ops    = ARRAY_SIZE(devlink_nl_ops),
9377         .mcgrps         = devlink_nl_mcgrps,
9378         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
9379 };
9380
9381 static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
9382 {
9383         const struct devlink_reload_combination *comb;
9384         int i;
9385
9386         if (!devlink_reload_supported(ops)) {
9387                 if (WARN_ON(ops->reload_actions))
9388                         return false;
9389                 return true;
9390         }
9391
9392         if (WARN_ON(!ops->reload_actions ||
9393                     ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
9394                     ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
9395                 return false;
9396
9397         if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
9398                     ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
9399                 return false;
9400
9401         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
9402                 comb = &devlink_reload_invalid_combinations[i];
9403                 if (ops->reload_actions == BIT(comb->action) &&
9404                     ops->reload_limits == BIT(comb->limit))
9405                         return false;
9406         }
9407         return true;
9408 }
9409
9410 /**
9411  *      devlink_set_features - Set devlink supported features
9412  *
9413  *      @devlink: devlink
9414  *      @features: devlink support features
9415  *
9416  *      This interface allows us to set reload ops separatelly from
9417  *      the devlink_alloc.
9418  */
9419 void devlink_set_features(struct devlink *devlink, u64 features)
9420 {
9421         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9422
9423         WARN_ON(features & DEVLINK_F_RELOAD &&
9424                 !devlink_reload_supported(devlink->ops));
9425         devlink->features = features;
9426 }
9427 EXPORT_SYMBOL_GPL(devlink_set_features);
9428
9429 /**
9430  *      devlink_alloc_ns - Allocate new devlink instance resources
9431  *      in specific namespace
9432  *
9433  *      @ops: ops
9434  *      @priv_size: size of user private data
9435  *      @net: net namespace
9436  *      @dev: parent device
9437  *
9438  *      Allocate new devlink instance resources, including devlink index
9439  *      and name.
9440  */
9441 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
9442                                  size_t priv_size, struct net *net,
9443                                  struct device *dev)
9444 {
9445         struct devlink *devlink;
9446         static u32 last_id;
9447         int ret;
9448
9449         WARN_ON(!ops || !dev);
9450         if (!devlink_reload_actions_valid(ops))
9451                 return NULL;
9452
9453         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
9454         if (!devlink)
9455                 return NULL;
9456
9457         ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
9458                               &last_id, GFP_KERNEL);
9459         if (ret < 0) {
9460                 kfree(devlink);
9461                 return NULL;
9462         }
9463
9464         devlink->dev = dev;
9465         devlink->ops = ops;
9466         xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
9467         write_pnet(&devlink->_net, net);
9468         INIT_LIST_HEAD(&devlink->port_list);
9469         INIT_LIST_HEAD(&devlink->rate_list);
9470         INIT_LIST_HEAD(&devlink->linecard_list);
9471         INIT_LIST_HEAD(&devlink->sb_list);
9472         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
9473         INIT_LIST_HEAD(&devlink->resource_list);
9474         INIT_LIST_HEAD(&devlink->param_list);
9475         INIT_LIST_HEAD(&devlink->region_list);
9476         INIT_LIST_HEAD(&devlink->reporter_list);
9477         INIT_LIST_HEAD(&devlink->trap_list);
9478         INIT_LIST_HEAD(&devlink->trap_group_list);
9479         INIT_LIST_HEAD(&devlink->trap_policer_list);
9480         mutex_init(&devlink->lock);
9481         mutex_init(&devlink->reporters_lock);
9482         mutex_init(&devlink->linecards_lock);
9483         refcount_set(&devlink->refcount, 1);
9484         init_completion(&devlink->comp);
9485
9486         return devlink;
9487 }
9488 EXPORT_SYMBOL_GPL(devlink_alloc_ns);
9489
9490 static void
9491 devlink_trap_policer_notify(struct devlink *devlink,
9492                             const struct devlink_trap_policer_item *policer_item,
9493                             enum devlink_command cmd);
9494 static void
9495 devlink_trap_group_notify(struct devlink *devlink,
9496                           const struct devlink_trap_group_item *group_item,
9497                           enum devlink_command cmd);
9498 static void devlink_trap_notify(struct devlink *devlink,
9499                                 const struct devlink_trap_item *trap_item,
9500                                 enum devlink_command cmd);
9501
9502 static void devlink_notify_register(struct devlink *devlink)
9503 {
9504         struct devlink_trap_policer_item *policer_item;
9505         struct devlink_trap_group_item *group_item;
9506         struct devlink_param_item *param_item;
9507         struct devlink_trap_item *trap_item;
9508         struct devlink_port *devlink_port;
9509         struct devlink_linecard *linecard;
9510         struct devlink_rate *rate_node;
9511         struct devlink_region *region;
9512
9513         devlink_notify(devlink, DEVLINK_CMD_NEW);
9514         list_for_each_entry(linecard, &devlink->linecard_list, list)
9515                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
9516
9517         list_for_each_entry(devlink_port, &devlink->port_list, list)
9518                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9519
9520         list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
9521                 devlink_trap_policer_notify(devlink, policer_item,
9522                                             DEVLINK_CMD_TRAP_POLICER_NEW);
9523
9524         list_for_each_entry(group_item, &devlink->trap_group_list, list)
9525                 devlink_trap_group_notify(devlink, group_item,
9526                                           DEVLINK_CMD_TRAP_GROUP_NEW);
9527
9528         list_for_each_entry(trap_item, &devlink->trap_list, list)
9529                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
9530
9531         list_for_each_entry(rate_node, &devlink->rate_list, list)
9532                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
9533
9534         list_for_each_entry(region, &devlink->region_list, list)
9535                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9536
9537         list_for_each_entry(param_item, &devlink->param_list, list)
9538                 devlink_param_notify(devlink, 0, param_item,
9539                                      DEVLINK_CMD_PARAM_NEW);
9540 }
9541
9542 static void devlink_notify_unregister(struct devlink *devlink)
9543 {
9544         struct devlink_trap_policer_item *policer_item;
9545         struct devlink_trap_group_item *group_item;
9546         struct devlink_param_item *param_item;
9547         struct devlink_trap_item *trap_item;
9548         struct devlink_port *devlink_port;
9549         struct devlink_rate *rate_node;
9550         struct devlink_region *region;
9551
9552         list_for_each_entry_reverse(param_item, &devlink->param_list, list)
9553                 devlink_param_notify(devlink, 0, param_item,
9554                                      DEVLINK_CMD_PARAM_DEL);
9555
9556         list_for_each_entry_reverse(region, &devlink->region_list, list)
9557                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
9558
9559         list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
9560                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
9561
9562         list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
9563                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
9564
9565         list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
9566                 devlink_trap_group_notify(devlink, group_item,
9567                                           DEVLINK_CMD_TRAP_GROUP_DEL);
9568         list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
9569                                     list)
9570                 devlink_trap_policer_notify(devlink, policer_item,
9571                                             DEVLINK_CMD_TRAP_POLICER_DEL);
9572
9573         list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
9574                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9575         devlink_notify(devlink, DEVLINK_CMD_DEL);
9576 }
9577
9578 /**
9579  *      devlink_register - Register devlink instance
9580  *
9581  *      @devlink: devlink
9582  */
9583 void devlink_register(struct devlink *devlink)
9584 {
9585         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9586         /* Make sure that we are in .probe() routine */
9587
9588         mutex_lock(&devlink_mutex);
9589         xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9590         devlink_notify_register(devlink);
9591         mutex_unlock(&devlink_mutex);
9592 }
9593 EXPORT_SYMBOL_GPL(devlink_register);
9594
9595 /**
9596  *      devlink_unregister - Unregister devlink instance
9597  *
9598  *      @devlink: devlink
9599  */
9600 void devlink_unregister(struct devlink *devlink)
9601 {
9602         ASSERT_DEVLINK_REGISTERED(devlink);
9603         /* Make sure that we are in .remove() routine */
9604
9605         devlink_put(devlink);
9606         wait_for_completion(&devlink->comp);
9607
9608         mutex_lock(&devlink_mutex);
9609         devlink_notify_unregister(devlink);
9610         xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9611         mutex_unlock(&devlink_mutex);
9612 }
9613 EXPORT_SYMBOL_GPL(devlink_unregister);
9614
9615 /**
9616  *      devlink_free - Free devlink instance resources
9617  *
9618  *      @devlink: devlink
9619  */
9620 void devlink_free(struct devlink *devlink)
9621 {
9622         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9623
9624         mutex_destroy(&devlink->linecards_lock);
9625         mutex_destroy(&devlink->reporters_lock);
9626         mutex_destroy(&devlink->lock);
9627         WARN_ON(!list_empty(&devlink->trap_policer_list));
9628         WARN_ON(!list_empty(&devlink->trap_group_list));
9629         WARN_ON(!list_empty(&devlink->trap_list));
9630         WARN_ON(!list_empty(&devlink->reporter_list));
9631         WARN_ON(!list_empty(&devlink->region_list));
9632         WARN_ON(!list_empty(&devlink->param_list));
9633         WARN_ON(!list_empty(&devlink->resource_list));
9634         WARN_ON(!list_empty(&devlink->dpipe_table_list));
9635         WARN_ON(!list_empty(&devlink->sb_list));
9636         WARN_ON(!list_empty(&devlink->rate_list));
9637         WARN_ON(!list_empty(&devlink->linecard_list));
9638         WARN_ON(!list_empty(&devlink->port_list));
9639
9640         xa_destroy(&devlink->snapshot_ids);
9641         xa_erase(&devlinks, devlink->index);
9642
9643         kfree(devlink);
9644 }
9645 EXPORT_SYMBOL_GPL(devlink_free);
9646
9647 static void devlink_port_type_warn(struct work_struct *work)
9648 {
9649         WARN(true, "Type was not set for devlink port.");
9650 }
9651
9652 static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
9653 {
9654         /* Ignore CPU and DSA flavours. */
9655         return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
9656                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
9657                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
9658 }
9659
9660 #define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
9661
9662 static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
9663 {
9664         if (!devlink_port_type_should_warn(devlink_port))
9665                 return;
9666         /* Schedule a work to WARN in case driver does not set port
9667          * type within timeout.
9668          */
9669         schedule_delayed_work(&devlink_port->type_warn_dw,
9670                               DEVLINK_PORT_TYPE_WARN_TIMEOUT);
9671 }
9672
9673 static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
9674 {
9675         if (!devlink_port_type_should_warn(devlink_port))
9676                 return;
9677         cancel_delayed_work_sync(&devlink_port->type_warn_dw);
9678 }
9679
9680 int devl_port_register(struct devlink *devlink,
9681                        struct devlink_port *devlink_port,
9682                        unsigned int port_index)
9683 {
9684         lockdep_assert_held(&devlink->lock);
9685
9686         if (devlink_port_index_exists(devlink, port_index))
9687                 return -EEXIST;
9688
9689         WARN_ON(devlink_port->devlink);
9690         devlink_port->devlink = devlink;
9691         devlink_port->index = port_index;
9692         spin_lock_init(&devlink_port->type_lock);
9693         INIT_LIST_HEAD(&devlink_port->reporter_list);
9694         mutex_init(&devlink_port->reporters_lock);
9695         list_add_tail(&devlink_port->list, &devlink->port_list);
9696         INIT_LIST_HEAD(&devlink_port->param_list);
9697         INIT_LIST_HEAD(&devlink_port->region_list);
9698
9699         INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
9700         devlink_port_type_warn_schedule(devlink_port);
9701         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9702         return 0;
9703 }
9704 EXPORT_SYMBOL_GPL(devl_port_register);
9705
9706 /**
9707  *      devlink_port_register - Register devlink port
9708  *
9709  *      @devlink: devlink
9710  *      @devlink_port: devlink port
9711  *      @port_index: driver-specific numerical identifier of the port
9712  *
9713  *      Register devlink port with provided port index. User can use
9714  *      any indexing, even hw-related one. devlink_port structure
9715  *      is convenient to be embedded inside user driver private structure.
9716  *      Note that the caller should take care of zeroing the devlink_port
9717  *      structure.
9718  */
9719 int devlink_port_register(struct devlink *devlink,
9720                           struct devlink_port *devlink_port,
9721                           unsigned int port_index)
9722 {
9723         int err;
9724
9725         mutex_lock(&devlink->lock);
9726         err = devl_port_register(devlink, devlink_port, port_index);
9727         mutex_unlock(&devlink->lock);
9728         return err;
9729 }
9730 EXPORT_SYMBOL_GPL(devlink_port_register);
9731
9732 void devl_port_unregister(struct devlink_port *devlink_port)
9733 {
9734         lockdep_assert_held(&devlink_port->devlink->lock);
9735
9736         devlink_port_type_warn_cancel(devlink_port);
9737         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9738         list_del(&devlink_port->list);
9739         WARN_ON(!list_empty(&devlink_port->reporter_list));
9740         WARN_ON(!list_empty(&devlink_port->region_list));
9741         mutex_destroy(&devlink_port->reporters_lock);
9742 }
9743 EXPORT_SYMBOL_GPL(devl_port_unregister);
9744
9745 /**
9746  *      devlink_port_unregister - Unregister devlink port
9747  *
9748  *      @devlink_port: devlink port
9749  */
9750 void devlink_port_unregister(struct devlink_port *devlink_port)
9751 {
9752         struct devlink *devlink = devlink_port->devlink;
9753
9754         mutex_lock(&devlink->lock);
9755         devl_port_unregister(devlink_port);
9756         mutex_unlock(&devlink->lock);
9757 }
9758 EXPORT_SYMBOL_GPL(devlink_port_unregister);
9759
9760 static void __devlink_port_type_set(struct devlink_port *devlink_port,
9761                                     enum devlink_port_type type,
9762                                     void *type_dev)
9763 {
9764         if (WARN_ON(!devlink_port->devlink))
9765                 return;
9766         devlink_port_type_warn_cancel(devlink_port);
9767         spin_lock_bh(&devlink_port->type_lock);
9768         devlink_port->type = type;
9769         devlink_port->type_dev = type_dev;
9770         spin_unlock_bh(&devlink_port->type_lock);
9771         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9772 }
9773
9774 static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
9775                                             struct net_device *netdev)
9776 {
9777         const struct net_device_ops *ops = netdev->netdev_ops;
9778
9779         /* If driver registers devlink port, it should set devlink port
9780          * attributes accordingly so the compat functions are called
9781          * and the original ops are not used.
9782          */
9783         if (ops->ndo_get_phys_port_name) {
9784                 /* Some drivers use the same set of ndos for netdevs
9785                  * that have devlink_port registered and also for
9786                  * those who don't. Make sure that ndo_get_phys_port_name
9787                  * returns -EOPNOTSUPP here in case it is defined.
9788                  * Warn if not.
9789                  */
9790                 char name[IFNAMSIZ];
9791                 int err;
9792
9793                 err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
9794                 WARN_ON(err != -EOPNOTSUPP);
9795         }
9796         if (ops->ndo_get_port_parent_id) {
9797                 /* Some drivers use the same set of ndos for netdevs
9798                  * that have devlink_port registered and also for
9799                  * those who don't. Make sure that ndo_get_port_parent_id
9800                  * returns -EOPNOTSUPP here in case it is defined.
9801                  * Warn if not.
9802                  */
9803                 struct netdev_phys_item_id ppid;
9804                 int err;
9805
9806                 err = ops->ndo_get_port_parent_id(netdev, &ppid);
9807                 WARN_ON(err != -EOPNOTSUPP);
9808         }
9809 }
9810
9811 /**
9812  *      devlink_port_type_eth_set - Set port type to Ethernet
9813  *
9814  *      @devlink_port: devlink port
9815  *      @netdev: related netdevice
9816  */
9817 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
9818                                struct net_device *netdev)
9819 {
9820         if (netdev)
9821                 devlink_port_type_netdev_checks(devlink_port, netdev);
9822         else
9823                 dev_warn(devlink_port->devlink->dev,
9824                          "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
9825                          devlink_port->index);
9826
9827         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
9828 }
9829 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
9830
9831 /**
9832  *      devlink_port_type_ib_set - Set port type to InfiniBand
9833  *
9834  *      @devlink_port: devlink port
9835  *      @ibdev: related IB device
9836  */
9837 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
9838                               struct ib_device *ibdev)
9839 {
9840         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
9841 }
9842 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
9843
9844 /**
9845  *      devlink_port_type_clear - Clear port type
9846  *
9847  *      @devlink_port: devlink port
9848  */
9849 void devlink_port_type_clear(struct devlink_port *devlink_port)
9850 {
9851         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
9852         devlink_port_type_warn_schedule(devlink_port);
9853 }
9854 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
9855
9856 static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
9857                                     enum devlink_port_flavour flavour)
9858 {
9859         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9860
9861         devlink_port->attrs_set = true;
9862         attrs->flavour = flavour;
9863         if (attrs->switch_id.id_len) {
9864                 devlink_port->switch_port = true;
9865                 if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
9866                         attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
9867         } else {
9868                 devlink_port->switch_port = false;
9869         }
9870         return 0;
9871 }
9872
9873 /**
9874  *      devlink_port_attrs_set - Set port attributes
9875  *
9876  *      @devlink_port: devlink port
9877  *      @attrs: devlink port attrs
9878  */
9879 void devlink_port_attrs_set(struct devlink_port *devlink_port,
9880                             struct devlink_port_attrs *attrs)
9881 {
9882         int ret;
9883
9884         if (WARN_ON(devlink_port->devlink))
9885                 return;
9886         devlink_port->attrs = *attrs;
9887         ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
9888         if (ret)
9889                 return;
9890         WARN_ON(attrs->splittable && attrs->split);
9891 }
9892 EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
9893
9894 /**
9895  *      devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
9896  *
9897  *      @devlink_port: devlink port
9898  *      @controller: associated controller number for the devlink port instance
9899  *      @pf: associated PF for the devlink port instance
9900  *      @external: indicates if the port is for an external controller
9901  */
9902 void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
9903                                    u16 pf, bool external)
9904 {
9905         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9906         int ret;
9907
9908         if (WARN_ON(devlink_port->devlink))
9909                 return;
9910         ret = __devlink_port_attrs_set(devlink_port,
9911                                        DEVLINK_PORT_FLAVOUR_PCI_PF);
9912         if (ret)
9913                 return;
9914         attrs->pci_pf.controller = controller;
9915         attrs->pci_pf.pf = pf;
9916         attrs->pci_pf.external = external;
9917 }
9918 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
9919
9920 /**
9921  *      devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
9922  *
9923  *      @devlink_port: devlink port
9924  *      @controller: associated controller number for the devlink port instance
9925  *      @pf: associated PF for the devlink port instance
9926  *      @vf: associated VF of a PF for the devlink port instance
9927  *      @external: indicates if the port is for an external controller
9928  */
9929 void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
9930                                    u16 pf, u16 vf, bool external)
9931 {
9932         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9933         int ret;
9934
9935         if (WARN_ON(devlink_port->devlink))
9936                 return;
9937         ret = __devlink_port_attrs_set(devlink_port,
9938                                        DEVLINK_PORT_FLAVOUR_PCI_VF);
9939         if (ret)
9940                 return;
9941         attrs->pci_vf.controller = controller;
9942         attrs->pci_vf.pf = pf;
9943         attrs->pci_vf.vf = vf;
9944         attrs->pci_vf.external = external;
9945 }
9946 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
9947
9948 /**
9949  *      devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
9950  *
9951  *      @devlink_port: devlink port
9952  *      @controller: associated controller number for the devlink port instance
9953  *      @pf: associated PF for the devlink port instance
9954  *      @sf: associated SF of a PF for the devlink port instance
9955  *      @external: indicates if the port is for an external controller
9956  */
9957 void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
9958                                    u16 pf, u32 sf, bool external)
9959 {
9960         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9961         int ret;
9962
9963         if (WARN_ON(devlink_port->devlink))
9964                 return;
9965         ret = __devlink_port_attrs_set(devlink_port,
9966                                        DEVLINK_PORT_FLAVOUR_PCI_SF);
9967         if (ret)
9968                 return;
9969         attrs->pci_sf.controller = controller;
9970         attrs->pci_sf.pf = pf;
9971         attrs->pci_sf.sf = sf;
9972         attrs->pci_sf.external = external;
9973 }
9974 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
9975
9976 /**
9977  * devl_rate_leaf_create - create devlink rate leaf
9978  * @devlink_port: devlink port object to create rate object on
9979  * @priv: driver private data
9980  *
9981  * Create devlink rate object of type leaf on provided @devlink_port.
9982  */
9983 int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
9984 {
9985         struct devlink *devlink = devlink_port->devlink;
9986         struct devlink_rate *devlink_rate;
9987
9988         devl_assert_locked(devlink_port->devlink);
9989
9990         if (WARN_ON(devlink_port->devlink_rate))
9991                 return -EBUSY;
9992
9993         devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
9994         if (!devlink_rate)
9995                 return -ENOMEM;
9996
9997         devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
9998         devlink_rate->devlink = devlink;
9999         devlink_rate->devlink_port = devlink_port;
10000         devlink_rate->priv = priv;
10001         list_add_tail(&devlink_rate->list, &devlink->rate_list);
10002         devlink_port->devlink_rate = devlink_rate;
10003         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
10004
10005         return 0;
10006 }
10007 EXPORT_SYMBOL_GPL(devl_rate_leaf_create);
10008
10009 int
10010 devlink_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
10011 {
10012         struct devlink *devlink = devlink_port->devlink;
10013         int ret;
10014
10015         mutex_lock(&devlink->lock);
10016         ret = devl_rate_leaf_create(devlink_port, priv);
10017         mutex_unlock(&devlink->lock);
10018
10019         return ret;
10020 }
10021 EXPORT_SYMBOL_GPL(devlink_rate_leaf_create);
10022
10023 void devl_rate_leaf_destroy(struct devlink_port *devlink_port)
10024 {
10025         struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
10026
10027         devl_assert_locked(devlink_port->devlink);
10028         if (!devlink_rate)
10029                 return;
10030
10031         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
10032         if (devlink_rate->parent)
10033                 refcount_dec(&devlink_rate->parent->refcnt);
10034         list_del(&devlink_rate->list);
10035         devlink_port->devlink_rate = NULL;
10036         kfree(devlink_rate);
10037 }
10038 EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
10039
10040 /**
10041  * devlink_rate_leaf_destroy - destroy devlink rate leaf
10042  *
10043  * @devlink_port: devlink port linked to the rate object
10044  *
10045  * Context: Takes and release devlink->lock <mutex>.
10046  */
10047 void devlink_rate_leaf_destroy(struct devlink_port *devlink_port)
10048 {
10049         struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
10050         struct devlink *devlink = devlink_port->devlink;
10051
10052         if (!devlink_rate)
10053                 return;
10054
10055         mutex_lock(&devlink->lock);
10056         devl_rate_leaf_destroy(devlink_port);
10057         mutex_unlock(&devlink->lock);
10058 }
10059 EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy);
10060
10061 /**
10062  * devl_rate_nodes_destroy - destroy all devlink rate nodes on device
10063  * @devlink: devlink instance
10064  *
10065  * Unset parent for all rate objects and destroy all rate nodes
10066  * on specified device.
10067  */
10068 void devl_rate_nodes_destroy(struct devlink *devlink)
10069 {
10070         static struct devlink_rate *devlink_rate, *tmp;
10071         const struct devlink_ops *ops = devlink->ops;
10072
10073         devl_assert_locked(devlink);
10074
10075         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
10076                 if (!devlink_rate->parent)
10077                         continue;
10078
10079                 refcount_dec(&devlink_rate->parent->refcnt);
10080                 if (devlink_rate_is_leaf(devlink_rate))
10081                         ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
10082                                                   NULL, NULL);
10083                 else if (devlink_rate_is_node(devlink_rate))
10084                         ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
10085                                                   NULL, NULL);
10086         }
10087         list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
10088                 if (devlink_rate_is_node(devlink_rate)) {
10089                         ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
10090                         list_del(&devlink_rate->list);
10091                         kfree(devlink_rate->name);
10092                         kfree(devlink_rate);
10093                 }
10094         }
10095 }
10096 EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
10097
10098 /**
10099  * devlink_rate_nodes_destroy - destroy all devlink rate nodes on device
10100  *
10101  * @devlink: devlink instance
10102  *
10103  * Unset parent for all rate objects and destroy all rate nodes
10104  * on specified device.
10105  *
10106  * Context: Takes and release devlink->lock <mutex>.
10107  */
10108 void devlink_rate_nodes_destroy(struct devlink *devlink)
10109 {
10110         mutex_lock(&devlink->lock);
10111         devl_rate_nodes_destroy(devlink);
10112         mutex_unlock(&devlink->lock);
10113 }
10114 EXPORT_SYMBOL_GPL(devlink_rate_nodes_destroy);
10115
10116 /**
10117  *      devlink_port_linecard_set - Link port with a linecard
10118  *
10119  *      @devlink_port: devlink port
10120  *      @linecard: devlink linecard
10121  */
10122 void devlink_port_linecard_set(struct devlink_port *devlink_port,
10123                                struct devlink_linecard *linecard)
10124 {
10125         if (WARN_ON(devlink_port->devlink))
10126                 return;
10127         devlink_port->linecard = linecard;
10128 }
10129 EXPORT_SYMBOL_GPL(devlink_port_linecard_set);
10130
10131 static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
10132                                              char *name, size_t len)
10133 {
10134         struct devlink_port_attrs *attrs = &devlink_port->attrs;
10135         int n = 0;
10136
10137         if (!devlink_port->attrs_set)
10138                 return -EOPNOTSUPP;
10139
10140         switch (attrs->flavour) {
10141         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
10142                 if (devlink_port->linecard)
10143                         n = snprintf(name, len, "l%u",
10144                                      devlink_port->linecard->index);
10145                 if (n < len)
10146                         n += snprintf(name + n, len - n, "p%u",
10147                                       attrs->phys.port_number);
10148                 if (n < len && attrs->split)
10149                         n += snprintf(name + n, len - n, "s%u",
10150                                       attrs->phys.split_subport_number);
10151                 break;
10152         case DEVLINK_PORT_FLAVOUR_CPU:
10153         case DEVLINK_PORT_FLAVOUR_DSA:
10154         case DEVLINK_PORT_FLAVOUR_UNUSED:
10155                 /* As CPU and DSA ports do not have a netdevice associated
10156                  * case should not ever happen.
10157                  */
10158                 WARN_ON(1);
10159                 return -EINVAL;
10160         case DEVLINK_PORT_FLAVOUR_PCI_PF:
10161                 if (attrs->pci_pf.external) {
10162                         n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
10163                         if (n >= len)
10164                                 return -EINVAL;
10165                         len -= n;
10166                         name += n;
10167                 }
10168                 n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
10169                 break;
10170         case DEVLINK_PORT_FLAVOUR_PCI_VF:
10171                 if (attrs->pci_vf.external) {
10172                         n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
10173                         if (n >= len)
10174                                 return -EINVAL;
10175                         len -= n;
10176                         name += n;
10177                 }
10178                 n = snprintf(name, len, "pf%uvf%u",
10179                              attrs->pci_vf.pf, attrs->pci_vf.vf);
10180                 break;
10181         case DEVLINK_PORT_FLAVOUR_PCI_SF:
10182                 if (attrs->pci_sf.external) {
10183                         n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
10184                         if (n >= len)
10185                                 return -EINVAL;
10186                         len -= n;
10187                         name += n;
10188                 }
10189                 n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
10190                              attrs->pci_sf.sf);
10191                 break;
10192         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
10193                 return -EOPNOTSUPP;
10194         }
10195
10196         if (n >= len)
10197                 return -EINVAL;
10198
10199         return 0;
10200 }
10201
10202 static int devlink_linecard_types_init(struct devlink_linecard *linecard)
10203 {
10204         struct devlink_linecard_type *linecard_type;
10205         unsigned int count;
10206         int i;
10207
10208         count = linecard->ops->types_count(linecard, linecard->priv);
10209         linecard->types = kmalloc_array(count, sizeof(*linecard_type),
10210                                         GFP_KERNEL);
10211         if (!linecard->types)
10212                 return -ENOMEM;
10213         linecard->types_count = count;
10214
10215         for (i = 0; i < count; i++) {
10216                 linecard_type = &linecard->types[i];
10217                 linecard->ops->types_get(linecard, linecard->priv, i,
10218                                          &linecard_type->type,
10219                                          &linecard_type->priv);
10220         }
10221         return 0;
10222 }
10223
10224 static void devlink_linecard_types_fini(struct devlink_linecard *linecard)
10225 {
10226         kfree(linecard->types);
10227 }
10228
10229 /**
10230  *      devlink_linecard_create - Create devlink linecard
10231  *
10232  *      @devlink: devlink
10233  *      @linecard_index: driver-specific numerical identifier of the linecard
10234  *      @ops: linecards ops
10235  *      @priv: user priv pointer
10236  *
10237  *      Create devlink linecard instance with provided linecard index.
10238  *      Caller can use any indexing, even hw-related one.
10239  *
10240  *      Return: Line card structure or an ERR_PTR() encoded error code.
10241  */
10242 struct devlink_linecard *
10243 devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index,
10244                         const struct devlink_linecard_ops *ops, void *priv)
10245 {
10246         struct devlink_linecard *linecard;
10247         int err;
10248
10249         if (WARN_ON(!ops || !ops->provision || !ops->unprovision ||
10250                     !ops->types_count || !ops->types_get))
10251                 return ERR_PTR(-EINVAL);
10252
10253         mutex_lock(&devlink->linecards_lock);
10254         if (devlink_linecard_index_exists(devlink, linecard_index)) {
10255                 mutex_unlock(&devlink->linecards_lock);
10256                 return ERR_PTR(-EEXIST);
10257         }
10258
10259         linecard = kzalloc(sizeof(*linecard), GFP_KERNEL);
10260         if (!linecard) {
10261                 mutex_unlock(&devlink->linecards_lock);
10262                 return ERR_PTR(-ENOMEM);
10263         }
10264
10265         linecard->devlink = devlink;
10266         linecard->index = linecard_index;
10267         linecard->ops = ops;
10268         linecard->priv = priv;
10269         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
10270         mutex_init(&linecard->state_lock);
10271
10272         err = devlink_linecard_types_init(linecard);
10273         if (err) {
10274                 mutex_destroy(&linecard->state_lock);
10275                 kfree(linecard);
10276                 mutex_unlock(&devlink->linecards_lock);
10277                 return ERR_PTR(err);
10278         }
10279
10280         list_add_tail(&linecard->list, &devlink->linecard_list);
10281         refcount_set(&linecard->refcount, 1);
10282         mutex_unlock(&devlink->linecards_lock);
10283         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10284         return linecard;
10285 }
10286 EXPORT_SYMBOL_GPL(devlink_linecard_create);
10287
10288 /**
10289  *      devlink_linecard_destroy - Destroy devlink linecard
10290  *
10291  *      @linecard: devlink linecard
10292  */
10293 void devlink_linecard_destroy(struct devlink_linecard *linecard)
10294 {
10295         struct devlink *devlink = linecard->devlink;
10296
10297         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_DEL);
10298         mutex_lock(&devlink->linecards_lock);
10299         list_del(&linecard->list);
10300         devlink_linecard_types_fini(linecard);
10301         mutex_unlock(&devlink->linecards_lock);
10302         devlink_linecard_put(linecard);
10303 }
10304 EXPORT_SYMBOL_GPL(devlink_linecard_destroy);
10305
10306 /**
10307  *      devlink_linecard_provision_set - Set provisioning on linecard
10308  *
10309  *      @linecard: devlink linecard
10310  *      @type: linecard type
10311  *
10312  *      This is either called directly from the provision() op call or
10313  *      as a result of the provision() op call asynchronously.
10314  */
10315 void devlink_linecard_provision_set(struct devlink_linecard *linecard,
10316                                     const char *type)
10317 {
10318         mutex_lock(&linecard->state_lock);
10319         WARN_ON(linecard->type && strcmp(linecard->type, type));
10320         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
10321         linecard->type = type;
10322         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10323         mutex_unlock(&linecard->state_lock);
10324 }
10325 EXPORT_SYMBOL_GPL(devlink_linecard_provision_set);
10326
10327 /**
10328  *      devlink_linecard_provision_clear - Clear provisioning on linecard
10329  *
10330  *      @linecard: devlink linecard
10331  *
10332  *      This is either called directly from the unprovision() op call or
10333  *      as a result of the unprovision() op call asynchronously.
10334  */
10335 void devlink_linecard_provision_clear(struct devlink_linecard *linecard)
10336 {
10337         mutex_lock(&linecard->state_lock);
10338         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
10339         linecard->type = NULL;
10340         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10341         mutex_unlock(&linecard->state_lock);
10342 }
10343 EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear);
10344
10345 /**
10346  *      devlink_linecard_provision_fail - Fail provisioning on linecard
10347  *
10348  *      @linecard: devlink linecard
10349  *
10350  *      This is either called directly from the provision() op call or
10351  *      as a result of the provision() op call asynchronously.
10352  */
10353 void devlink_linecard_provision_fail(struct devlink_linecard *linecard)
10354 {
10355         mutex_lock(&linecard->state_lock);
10356         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING_FAILED;
10357         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10358         mutex_unlock(&linecard->state_lock);
10359 }
10360 EXPORT_SYMBOL_GPL(devlink_linecard_provision_fail);
10361
10362 /**
10363  *      devlink_linecard_activate - Set linecard active
10364  *
10365  *      @linecard: devlink linecard
10366  */
10367 void devlink_linecard_activate(struct devlink_linecard *linecard)
10368 {
10369         mutex_lock(&linecard->state_lock);
10370         WARN_ON(linecard->state != DEVLINK_LINECARD_STATE_PROVISIONED);
10371         linecard->state = DEVLINK_LINECARD_STATE_ACTIVE;
10372         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10373         mutex_unlock(&linecard->state_lock);
10374 }
10375 EXPORT_SYMBOL_GPL(devlink_linecard_activate);
10376
10377 /**
10378  *      devlink_linecard_deactivate - Set linecard inactive
10379  *
10380  *      @linecard: devlink linecard
10381  */
10382 void devlink_linecard_deactivate(struct devlink_linecard *linecard)
10383 {
10384         mutex_lock(&linecard->state_lock);
10385         switch (linecard->state) {
10386         case DEVLINK_LINECARD_STATE_ACTIVE:
10387                 linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
10388                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10389                 break;
10390         case DEVLINK_LINECARD_STATE_UNPROVISIONING:
10391                 /* Line card is being deactivated as part
10392                  * of unprovisioning flow.
10393                  */
10394                 break;
10395         default:
10396                 WARN_ON(1);
10397                 break;
10398         }
10399         mutex_unlock(&linecard->state_lock);
10400 }
10401 EXPORT_SYMBOL_GPL(devlink_linecard_deactivate);
10402
10403 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
10404                         u32 size, u16 ingress_pools_count,
10405                         u16 egress_pools_count, u16 ingress_tc_count,
10406                         u16 egress_tc_count)
10407 {
10408         struct devlink_sb *devlink_sb;
10409         int err = 0;
10410
10411         mutex_lock(&devlink->lock);
10412         if (devlink_sb_index_exists(devlink, sb_index)) {
10413                 err = -EEXIST;
10414                 goto unlock;
10415         }
10416
10417         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
10418         if (!devlink_sb) {
10419                 err = -ENOMEM;
10420                 goto unlock;
10421         }
10422         devlink_sb->index = sb_index;
10423         devlink_sb->size = size;
10424         devlink_sb->ingress_pools_count = ingress_pools_count;
10425         devlink_sb->egress_pools_count = egress_pools_count;
10426         devlink_sb->ingress_tc_count = ingress_tc_count;
10427         devlink_sb->egress_tc_count = egress_tc_count;
10428         list_add_tail(&devlink_sb->list, &devlink->sb_list);
10429 unlock:
10430         mutex_unlock(&devlink->lock);
10431         return err;
10432 }
10433 EXPORT_SYMBOL_GPL(devlink_sb_register);
10434
10435 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
10436 {
10437         struct devlink_sb *devlink_sb;
10438
10439         mutex_lock(&devlink->lock);
10440         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
10441         WARN_ON(!devlink_sb);
10442         list_del(&devlink_sb->list);
10443         mutex_unlock(&devlink->lock);
10444         kfree(devlink_sb);
10445 }
10446 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
10447
10448 /**
10449  *      devlink_dpipe_headers_register - register dpipe headers
10450  *
10451  *      @devlink: devlink
10452  *      @dpipe_headers: dpipe header array
10453  *
10454  *      Register the headers supported by hardware.
10455  */
10456 int devlink_dpipe_headers_register(struct devlink *devlink,
10457                                    struct devlink_dpipe_headers *dpipe_headers)
10458 {
10459         mutex_lock(&devlink->lock);
10460         devlink->dpipe_headers = dpipe_headers;
10461         mutex_unlock(&devlink->lock);
10462         return 0;
10463 }
10464 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
10465
10466 /**
10467  *      devlink_dpipe_headers_unregister - unregister dpipe headers
10468  *
10469  *      @devlink: devlink
10470  *
10471  *      Unregister the headers supported by hardware.
10472  */
10473 void devlink_dpipe_headers_unregister(struct devlink *devlink)
10474 {
10475         mutex_lock(&devlink->lock);
10476         devlink->dpipe_headers = NULL;
10477         mutex_unlock(&devlink->lock);
10478 }
10479 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
10480
10481 /**
10482  *      devlink_dpipe_table_counter_enabled - check if counter allocation
10483  *                                            required
10484  *      @devlink: devlink
10485  *      @table_name: tables name
10486  *
10487  *      Used by driver to check if counter allocation is required.
10488  *      After counter allocation is turned on the table entries
10489  *      are updated to include counter statistics.
10490  *
10491  *      After that point on the driver must respect the counter
10492  *      state so that each entry added to the table is added
10493  *      with a counter.
10494  */
10495 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
10496                                          const char *table_name)
10497 {
10498         struct devlink_dpipe_table *table;
10499         bool enabled;
10500
10501         rcu_read_lock();
10502         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10503                                          table_name, devlink);
10504         enabled = false;
10505         if (table)
10506                 enabled = table->counters_enabled;
10507         rcu_read_unlock();
10508         return enabled;
10509 }
10510 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
10511
10512 /**
10513  *      devlink_dpipe_table_register - register dpipe table
10514  *
10515  *      @devlink: devlink
10516  *      @table_name: table name
10517  *      @table_ops: table ops
10518  *      @priv: priv
10519  *      @counter_control_extern: external control for counters
10520  */
10521 int devlink_dpipe_table_register(struct devlink *devlink,
10522                                  const char *table_name,
10523                                  struct devlink_dpipe_table_ops *table_ops,
10524                                  void *priv, bool counter_control_extern)
10525 {
10526         struct devlink_dpipe_table *table;
10527         int err = 0;
10528
10529         if (WARN_ON(!table_ops->size_get))
10530                 return -EINVAL;
10531
10532         mutex_lock(&devlink->lock);
10533
10534         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
10535                                      devlink)) {
10536                 err = -EEXIST;
10537                 goto unlock;
10538         }
10539
10540         table = kzalloc(sizeof(*table), GFP_KERNEL);
10541         if (!table) {
10542                 err = -ENOMEM;
10543                 goto unlock;
10544         }
10545
10546         table->name = table_name;
10547         table->table_ops = table_ops;
10548         table->priv = priv;
10549         table->counter_control_extern = counter_control_extern;
10550
10551         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
10552 unlock:
10553         mutex_unlock(&devlink->lock);
10554         return err;
10555 }
10556 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
10557
10558 /**
10559  *      devlink_dpipe_table_unregister - unregister dpipe table
10560  *
10561  *      @devlink: devlink
10562  *      @table_name: table name
10563  */
10564 void devlink_dpipe_table_unregister(struct devlink *devlink,
10565                                     const char *table_name)
10566 {
10567         struct devlink_dpipe_table *table;
10568
10569         mutex_lock(&devlink->lock);
10570         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10571                                          table_name, devlink);
10572         if (!table)
10573                 goto unlock;
10574         list_del_rcu(&table->list);
10575         mutex_unlock(&devlink->lock);
10576         kfree_rcu(table, rcu);
10577         return;
10578 unlock:
10579         mutex_unlock(&devlink->lock);
10580 }
10581 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
10582
10583 /**
10584  *      devlink_resource_register - devlink resource register
10585  *
10586  *      @devlink: devlink
10587  *      @resource_name: resource's name
10588  *      @resource_size: resource's size
10589  *      @resource_id: resource's id
10590  *      @parent_resource_id: resource's parent id
10591  *      @size_params: size parameters
10592  *
10593  *      Generic resources should reuse the same names across drivers.
10594  *      Please see the generic resources list at:
10595  *      Documentation/networking/devlink/devlink-resource.rst
10596  */
10597 int devlink_resource_register(struct devlink *devlink,
10598                               const char *resource_name,
10599                               u64 resource_size,
10600                               u64 resource_id,
10601                               u64 parent_resource_id,
10602                               const struct devlink_resource_size_params *size_params)
10603 {
10604         struct devlink_resource *resource;
10605         struct list_head *resource_list;
10606         bool top_hierarchy;
10607         int err = 0;
10608
10609         top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
10610
10611         mutex_lock(&devlink->lock);
10612         resource = devlink_resource_find(devlink, NULL, resource_id);
10613         if (resource) {
10614                 err = -EINVAL;
10615                 goto out;
10616         }
10617
10618         resource = kzalloc(sizeof(*resource), GFP_KERNEL);
10619         if (!resource) {
10620                 err = -ENOMEM;
10621                 goto out;
10622         }
10623
10624         if (top_hierarchy) {
10625                 resource_list = &devlink->resource_list;
10626         } else {
10627                 struct devlink_resource *parent_resource;
10628
10629                 parent_resource = devlink_resource_find(devlink, NULL,
10630                                                         parent_resource_id);
10631                 if (parent_resource) {
10632                         resource_list = &parent_resource->resource_list;
10633                         resource->parent = parent_resource;
10634                 } else {
10635                         kfree(resource);
10636                         err = -EINVAL;
10637                         goto out;
10638                 }
10639         }
10640
10641         resource->name = resource_name;
10642         resource->size = resource_size;
10643         resource->size_new = resource_size;
10644         resource->id = resource_id;
10645         resource->size_valid = true;
10646         memcpy(&resource->size_params, size_params,
10647                sizeof(resource->size_params));
10648         INIT_LIST_HEAD(&resource->resource_list);
10649         list_add_tail(&resource->list, resource_list);
10650 out:
10651         mutex_unlock(&devlink->lock);
10652         return err;
10653 }
10654 EXPORT_SYMBOL_GPL(devlink_resource_register);
10655
10656 static void devlink_resource_unregister(struct devlink *devlink,
10657                                         struct devlink_resource *resource)
10658 {
10659         struct devlink_resource *tmp, *child_resource;
10660
10661         list_for_each_entry_safe(child_resource, tmp, &resource->resource_list,
10662                                  list) {
10663                 devlink_resource_unregister(devlink, child_resource);
10664                 list_del(&child_resource->list);
10665                 kfree(child_resource);
10666         }
10667 }
10668
10669 /**
10670  *      devlink_resources_unregister - free all resources
10671  *
10672  *      @devlink: devlink
10673  */
10674 void devlink_resources_unregister(struct devlink *devlink)
10675 {
10676         struct devlink_resource *tmp, *child_resource;
10677
10678         mutex_lock(&devlink->lock);
10679
10680         list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list,
10681                                  list) {
10682                 devlink_resource_unregister(devlink, child_resource);
10683                 list_del(&child_resource->list);
10684                 kfree(child_resource);
10685         }
10686
10687         mutex_unlock(&devlink->lock);
10688 }
10689 EXPORT_SYMBOL_GPL(devlink_resources_unregister);
10690
10691 /**
10692  *      devlink_resource_size_get - get and update size
10693  *
10694  *      @devlink: devlink
10695  *      @resource_id: the requested resource id
10696  *      @p_resource_size: ptr to update
10697  */
10698 int devlink_resource_size_get(struct devlink *devlink,
10699                               u64 resource_id,
10700                               u64 *p_resource_size)
10701 {
10702         struct devlink_resource *resource;
10703         int err = 0;
10704
10705         mutex_lock(&devlink->lock);
10706         resource = devlink_resource_find(devlink, NULL, resource_id);
10707         if (!resource) {
10708                 err = -EINVAL;
10709                 goto out;
10710         }
10711         *p_resource_size = resource->size_new;
10712         resource->size = resource->size_new;
10713 out:
10714         mutex_unlock(&devlink->lock);
10715         return err;
10716 }
10717 EXPORT_SYMBOL_GPL(devlink_resource_size_get);
10718
10719 /**
10720  *      devlink_dpipe_table_resource_set - set the resource id
10721  *
10722  *      @devlink: devlink
10723  *      @table_name: table name
10724  *      @resource_id: resource id
10725  *      @resource_units: number of resource's units consumed per table's entry
10726  */
10727 int devlink_dpipe_table_resource_set(struct devlink *devlink,
10728                                      const char *table_name, u64 resource_id,
10729                                      u64 resource_units)
10730 {
10731         struct devlink_dpipe_table *table;
10732         int err = 0;
10733
10734         mutex_lock(&devlink->lock);
10735         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10736                                          table_name, devlink);
10737         if (!table) {
10738                 err = -EINVAL;
10739                 goto out;
10740         }
10741         table->resource_id = resource_id;
10742         table->resource_units = resource_units;
10743         table->resource_valid = true;
10744 out:
10745         mutex_unlock(&devlink->lock);
10746         return err;
10747 }
10748 EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set);
10749
10750 /**
10751  *      devlink_resource_occ_get_register - register occupancy getter
10752  *
10753  *      @devlink: devlink
10754  *      @resource_id: resource id
10755  *      @occ_get: occupancy getter callback
10756  *      @occ_get_priv: occupancy getter callback priv
10757  */
10758 void devlink_resource_occ_get_register(struct devlink *devlink,
10759                                        u64 resource_id,
10760                                        devlink_resource_occ_get_t *occ_get,
10761                                        void *occ_get_priv)
10762 {
10763         struct devlink_resource *resource;
10764
10765         mutex_lock(&devlink->lock);
10766         resource = devlink_resource_find(devlink, NULL, resource_id);
10767         if (WARN_ON(!resource))
10768                 goto out;
10769         WARN_ON(resource->occ_get);
10770
10771         resource->occ_get = occ_get;
10772         resource->occ_get_priv = occ_get_priv;
10773 out:
10774         mutex_unlock(&devlink->lock);
10775 }
10776 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
10777
10778 /**
10779  *      devlink_resource_occ_get_unregister - unregister occupancy getter
10780  *
10781  *      @devlink: devlink
10782  *      @resource_id: resource id
10783  */
10784 void devlink_resource_occ_get_unregister(struct devlink *devlink,
10785                                          u64 resource_id)
10786 {
10787         struct devlink_resource *resource;
10788
10789         mutex_lock(&devlink->lock);
10790         resource = devlink_resource_find(devlink, NULL, resource_id);
10791         if (WARN_ON(!resource))
10792                 goto out;
10793         WARN_ON(!resource->occ_get);
10794
10795         resource->occ_get = NULL;
10796         resource->occ_get_priv = NULL;
10797 out:
10798         mutex_unlock(&devlink->lock);
10799 }
10800 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
10801
10802 static int devlink_param_verify(const struct devlink_param *param)
10803 {
10804         if (!param || !param->name || !param->supported_cmodes)
10805                 return -EINVAL;
10806         if (param->generic)
10807                 return devlink_param_generic_verify(param);
10808         else
10809                 return devlink_param_driver_verify(param);
10810 }
10811
10812 /**
10813  *      devlink_params_register - register configuration parameters
10814  *
10815  *      @devlink: devlink
10816  *      @params: configuration parameters array
10817  *      @params_count: number of parameters provided
10818  *
10819  *      Register the configuration parameters supported by the driver.
10820  */
10821 int devlink_params_register(struct devlink *devlink,
10822                             const struct devlink_param *params,
10823                             size_t params_count)
10824 {
10825         const struct devlink_param *param = params;
10826         int i, err;
10827
10828         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10829
10830         for (i = 0; i < params_count; i++, param++) {
10831                 err = devlink_param_register(devlink, param);
10832                 if (err)
10833                         goto rollback;
10834         }
10835         return 0;
10836
10837 rollback:
10838         if (!i)
10839                 return err;
10840
10841         for (param--; i > 0; i--, param--)
10842                 devlink_param_unregister(devlink, param);
10843         return err;
10844 }
10845 EXPORT_SYMBOL_GPL(devlink_params_register);
10846
10847 /**
10848  *      devlink_params_unregister - unregister configuration parameters
10849  *      @devlink: devlink
10850  *      @params: configuration parameters to unregister
10851  *      @params_count: number of parameters provided
10852  */
10853 void devlink_params_unregister(struct devlink *devlink,
10854                                const struct devlink_param *params,
10855                                size_t params_count)
10856 {
10857         const struct devlink_param *param = params;
10858         int i;
10859
10860         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10861
10862         for (i = 0; i < params_count; i++, param++)
10863                 devlink_param_unregister(devlink, param);
10864 }
10865 EXPORT_SYMBOL_GPL(devlink_params_unregister);
10866
10867 /**
10868  * devlink_param_register - register one configuration parameter
10869  *
10870  * @devlink: devlink
10871  * @param: one configuration parameter
10872  *
10873  * Register the configuration parameter supported by the driver.
10874  * Return: returns 0 on successful registration or error code otherwise.
10875  */
10876 int devlink_param_register(struct devlink *devlink,
10877                            const struct devlink_param *param)
10878 {
10879         struct devlink_param_item *param_item;
10880
10881         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10882
10883         WARN_ON(devlink_param_verify(param));
10884         WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
10885
10886         if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
10887                 WARN_ON(param->get || param->set);
10888         else
10889                 WARN_ON(!param->get || !param->set);
10890
10891         param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
10892         if (!param_item)
10893                 return -ENOMEM;
10894
10895         param_item->param = param;
10896
10897         list_add_tail(&param_item->list, &devlink->param_list);
10898         return 0;
10899 }
10900 EXPORT_SYMBOL_GPL(devlink_param_register);
10901
10902 /**
10903  * devlink_param_unregister - unregister one configuration parameter
10904  * @devlink: devlink
10905  * @param: configuration parameter to unregister
10906  */
10907 void devlink_param_unregister(struct devlink *devlink,
10908                               const struct devlink_param *param)
10909 {
10910         struct devlink_param_item *param_item;
10911
10912         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10913
10914         param_item =
10915                 devlink_param_find_by_name(&devlink->param_list, param->name);
10916         WARN_ON(!param_item);
10917         list_del(&param_item->list);
10918         kfree(param_item);
10919 }
10920 EXPORT_SYMBOL_GPL(devlink_param_unregister);
10921
10922 /**
10923  *      devlink_param_driverinit_value_get - get configuration parameter
10924  *                                           value for driver initializing
10925  *
10926  *      @devlink: devlink
10927  *      @param_id: parameter ID
10928  *      @init_val: value of parameter in driverinit configuration mode
10929  *
10930  *      This function should be used by the driver to get driverinit
10931  *      configuration for initialization after reload command.
10932  */
10933 int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
10934                                        union devlink_param_value *init_val)
10935 {
10936         struct devlink_param_item *param_item;
10937
10938         if (!devlink_reload_supported(devlink->ops))
10939                 return -EOPNOTSUPP;
10940
10941         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10942         if (!param_item)
10943                 return -EINVAL;
10944
10945         if (!param_item->driverinit_value_valid ||
10946             !devlink_param_cmode_is_supported(param_item->param,
10947                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10948                 return -EOPNOTSUPP;
10949
10950         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10951                 strcpy(init_val->vstr, param_item->driverinit_value.vstr);
10952         else
10953                 *init_val = param_item->driverinit_value;
10954
10955         return 0;
10956 }
10957 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
10958
10959 /**
10960  *      devlink_param_driverinit_value_set - set value of configuration
10961  *                                           parameter for driverinit
10962  *                                           configuration mode
10963  *
10964  *      @devlink: devlink
10965  *      @param_id: parameter ID
10966  *      @init_val: value of parameter to set for driverinit configuration mode
10967  *
10968  *      This function should be used by the driver to set driverinit
10969  *      configuration mode default value.
10970  */
10971 int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
10972                                        union devlink_param_value init_val)
10973 {
10974         struct devlink_param_item *param_item;
10975
10976         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10977
10978         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10979         if (!param_item)
10980                 return -EINVAL;
10981
10982         if (!devlink_param_cmode_is_supported(param_item->param,
10983                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10984                 return -EOPNOTSUPP;
10985
10986         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10987                 strcpy(param_item->driverinit_value.vstr, init_val.vstr);
10988         else
10989                 param_item->driverinit_value = init_val;
10990         param_item->driverinit_value_valid = true;
10991         return 0;
10992 }
10993 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
10994
10995 /**
10996  *      devlink_param_value_changed - notify devlink on a parameter's value
10997  *                                    change. Should be called by the driver
10998  *                                    right after the change.
10999  *
11000  *      @devlink: devlink
11001  *      @param_id: parameter ID
11002  *
11003  *      This function should be used by the driver to notify devlink on value
11004  *      change, excluding driverinit configuration mode.
11005  *      For driverinit configuration mode driver should use the function
11006  */
11007 void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
11008 {
11009         struct devlink_param_item *param_item;
11010
11011         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
11012         WARN_ON(!param_item);
11013
11014         devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
11015 }
11016 EXPORT_SYMBOL_GPL(devlink_param_value_changed);
11017
11018 /**
11019  *      devlink_region_create - create a new address region
11020  *
11021  *      @devlink: devlink
11022  *      @ops: region operations and name
11023  *      @region_max_snapshots: Maximum supported number of snapshots for region
11024  *      @region_size: size of region
11025  */
11026 struct devlink_region *
11027 devlink_region_create(struct devlink *devlink,
11028                       const struct devlink_region_ops *ops,
11029                       u32 region_max_snapshots, u64 region_size)
11030 {
11031         struct devlink_region *region;
11032         int err = 0;
11033
11034         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
11035                 return ERR_PTR(-EINVAL);
11036
11037         mutex_lock(&devlink->lock);
11038
11039         if (devlink_region_get_by_name(devlink, ops->name)) {
11040                 err = -EEXIST;
11041                 goto unlock;
11042         }
11043
11044         region = kzalloc(sizeof(*region), GFP_KERNEL);
11045         if (!region) {
11046                 err = -ENOMEM;
11047                 goto unlock;
11048         }
11049
11050         region->devlink = devlink;
11051         region->max_snapshots = region_max_snapshots;
11052         region->ops = ops;
11053         region->size = region_size;
11054         INIT_LIST_HEAD(&region->snapshot_list);
11055         list_add_tail(&region->list, &devlink->region_list);
11056         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
11057
11058         mutex_unlock(&devlink->lock);
11059         return region;
11060
11061 unlock:
11062         mutex_unlock(&devlink->lock);
11063         return ERR_PTR(err);
11064 }
11065 EXPORT_SYMBOL_GPL(devlink_region_create);
11066
11067 /**
11068  *      devlink_port_region_create - create a new address region for a port
11069  *
11070  *      @port: devlink port
11071  *      @ops: region operations and name
11072  *      @region_max_snapshots: Maximum supported number of snapshots for region
11073  *      @region_size: size of region
11074  */
11075 struct devlink_region *
11076 devlink_port_region_create(struct devlink_port *port,
11077                            const struct devlink_port_region_ops *ops,
11078                            u32 region_max_snapshots, u64 region_size)
11079 {
11080         struct devlink *devlink = port->devlink;
11081         struct devlink_region *region;
11082         int err = 0;
11083
11084         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
11085                 return ERR_PTR(-EINVAL);
11086
11087         mutex_lock(&devlink->lock);
11088
11089         if (devlink_port_region_get_by_name(port, ops->name)) {
11090                 err = -EEXIST;
11091                 goto unlock;
11092         }
11093
11094         region = kzalloc(sizeof(*region), GFP_KERNEL);
11095         if (!region) {
11096                 err = -ENOMEM;
11097                 goto unlock;
11098         }
11099
11100         region->devlink = devlink;
11101         region->port = port;
11102         region->max_snapshots = region_max_snapshots;
11103         region->port_ops = ops;
11104         region->size = region_size;
11105         INIT_LIST_HEAD(&region->snapshot_list);
11106         list_add_tail(&region->list, &port->region_list);
11107         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
11108
11109         mutex_unlock(&devlink->lock);
11110         return region;
11111
11112 unlock:
11113         mutex_unlock(&devlink->lock);
11114         return ERR_PTR(err);
11115 }
11116 EXPORT_SYMBOL_GPL(devlink_port_region_create);
11117
11118 /**
11119  *      devlink_region_destroy - destroy address region
11120  *
11121  *      @region: devlink region to destroy
11122  */
11123 void devlink_region_destroy(struct devlink_region *region)
11124 {
11125         struct devlink *devlink = region->devlink;
11126         struct devlink_snapshot *snapshot, *ts;
11127
11128         mutex_lock(&devlink->lock);
11129
11130         /* Free all snapshots of region */
11131         list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
11132                 devlink_region_snapshot_del(region, snapshot);
11133
11134         list_del(&region->list);
11135
11136         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
11137         mutex_unlock(&devlink->lock);
11138         kfree(region);
11139 }
11140 EXPORT_SYMBOL_GPL(devlink_region_destroy);
11141
11142 /**
11143  *      devlink_region_snapshot_id_get - get snapshot ID
11144  *
11145  *      This callback should be called when adding a new snapshot,
11146  *      Driver should use the same id for multiple snapshots taken
11147  *      on multiple regions at the same time/by the same trigger.
11148  *
11149  *      The caller of this function must use devlink_region_snapshot_id_put
11150  *      when finished creating regions using this id.
11151  *
11152  *      Returns zero on success, or a negative error code on failure.
11153  *
11154  *      @devlink: devlink
11155  *      @id: storage to return id
11156  */
11157 int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
11158 {
11159         int err;
11160
11161         mutex_lock(&devlink->lock);
11162         err = __devlink_region_snapshot_id_get(devlink, id);
11163         mutex_unlock(&devlink->lock);
11164
11165         return err;
11166 }
11167 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
11168
11169 /**
11170  *      devlink_region_snapshot_id_put - put snapshot ID reference
11171  *
11172  *      This should be called by a driver after finishing creating snapshots
11173  *      with an id. Doing so ensures that the ID can later be released in the
11174  *      event that all snapshots using it have been destroyed.
11175  *
11176  *      @devlink: devlink
11177  *      @id: id to release reference on
11178  */
11179 void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
11180 {
11181         mutex_lock(&devlink->lock);
11182         __devlink_snapshot_id_decrement(devlink, id);
11183         mutex_unlock(&devlink->lock);
11184 }
11185 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
11186
11187 /**
11188  *      devlink_region_snapshot_create - create a new snapshot
11189  *      This will add a new snapshot of a region. The snapshot
11190  *      will be stored on the region struct and can be accessed
11191  *      from devlink. This is useful for future analyses of snapshots.
11192  *      Multiple snapshots can be created on a region.
11193  *      The @snapshot_id should be obtained using the getter function.
11194  *
11195  *      @region: devlink region of the snapshot
11196  *      @data: snapshot data
11197  *      @snapshot_id: snapshot id to be created
11198  */
11199 int devlink_region_snapshot_create(struct devlink_region *region,
11200                                    u8 *data, u32 snapshot_id)
11201 {
11202         struct devlink *devlink = region->devlink;
11203         int err;
11204
11205         mutex_lock(&devlink->lock);
11206         err = __devlink_region_snapshot_create(region, data, snapshot_id);
11207         mutex_unlock(&devlink->lock);
11208
11209         return err;
11210 }
11211 EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
11212
11213 #define DEVLINK_TRAP(_id, _type)                                              \
11214         {                                                                     \
11215                 .type = DEVLINK_TRAP_TYPE_##_type,                            \
11216                 .id = DEVLINK_TRAP_GENERIC_ID_##_id,                          \
11217                 .name = DEVLINK_TRAP_GENERIC_NAME_##_id,                      \
11218         }
11219
11220 static const struct devlink_trap devlink_trap_generic[] = {
11221         DEVLINK_TRAP(SMAC_MC, DROP),
11222         DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
11223         DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
11224         DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
11225         DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
11226         DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
11227         DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
11228         DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
11229         DEVLINK_TRAP(TAIL_DROP, DROP),
11230         DEVLINK_TRAP(NON_IP_PACKET, DROP),
11231         DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
11232         DEVLINK_TRAP(DIP_LB, DROP),
11233         DEVLINK_TRAP(SIP_MC, DROP),
11234         DEVLINK_TRAP(SIP_LB, DROP),
11235         DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
11236         DEVLINK_TRAP(IPV4_SIP_BC, DROP),
11237         DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
11238         DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
11239         DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
11240         DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
11241         DEVLINK_TRAP(RPF, EXCEPTION),
11242         DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
11243         DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
11244         DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
11245         DEVLINK_TRAP(NON_ROUTABLE, DROP),
11246         DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
11247         DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
11248         DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
11249         DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
11250         DEVLINK_TRAP(STP, CONTROL),
11251         DEVLINK_TRAP(LACP, CONTROL),
11252         DEVLINK_TRAP(LLDP, CONTROL),
11253         DEVLINK_TRAP(IGMP_QUERY, CONTROL),
11254         DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
11255         DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
11256         DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
11257         DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
11258         DEVLINK_TRAP(MLD_QUERY, CONTROL),
11259         DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
11260         DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
11261         DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
11262         DEVLINK_TRAP(IPV4_DHCP, CONTROL),
11263         DEVLINK_TRAP(IPV6_DHCP, CONTROL),
11264         DEVLINK_TRAP(ARP_REQUEST, CONTROL),
11265         DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
11266         DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
11267         DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
11268         DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
11269         DEVLINK_TRAP(IPV4_BFD, CONTROL),
11270         DEVLINK_TRAP(IPV6_BFD, CONTROL),
11271         DEVLINK_TRAP(IPV4_OSPF, CONTROL),
11272         DEVLINK_TRAP(IPV6_OSPF, CONTROL),
11273         DEVLINK_TRAP(IPV4_BGP, CONTROL),
11274         DEVLINK_TRAP(IPV6_BGP, CONTROL),
11275         DEVLINK_TRAP(IPV4_VRRP, CONTROL),
11276         DEVLINK_TRAP(IPV6_VRRP, CONTROL),
11277         DEVLINK_TRAP(IPV4_PIM, CONTROL),
11278         DEVLINK_TRAP(IPV6_PIM, CONTROL),
11279         DEVLINK_TRAP(UC_LB, CONTROL),
11280         DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
11281         DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
11282         DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
11283         DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
11284         DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
11285         DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
11286         DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
11287         DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
11288         DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
11289         DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
11290         DEVLINK_TRAP(PTP_EVENT, CONTROL),
11291         DEVLINK_TRAP(PTP_GENERAL, CONTROL),
11292         DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
11293         DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
11294         DEVLINK_TRAP(EARLY_DROP, DROP),
11295         DEVLINK_TRAP(VXLAN_PARSING, DROP),
11296         DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
11297         DEVLINK_TRAP(VLAN_PARSING, DROP),
11298         DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
11299         DEVLINK_TRAP(MPLS_PARSING, DROP),
11300         DEVLINK_TRAP(ARP_PARSING, DROP),
11301         DEVLINK_TRAP(IP_1_PARSING, DROP),
11302         DEVLINK_TRAP(IP_N_PARSING, DROP),
11303         DEVLINK_TRAP(GRE_PARSING, DROP),
11304         DEVLINK_TRAP(UDP_PARSING, DROP),
11305         DEVLINK_TRAP(TCP_PARSING, DROP),
11306         DEVLINK_TRAP(IPSEC_PARSING, DROP),
11307         DEVLINK_TRAP(SCTP_PARSING, DROP),
11308         DEVLINK_TRAP(DCCP_PARSING, DROP),
11309         DEVLINK_TRAP(GTP_PARSING, DROP),
11310         DEVLINK_TRAP(ESP_PARSING, DROP),
11311         DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
11312         DEVLINK_TRAP(DMAC_FILTER, DROP),
11313 };
11314
11315 #define DEVLINK_TRAP_GROUP(_id)                                               \
11316         {                                                                     \
11317                 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,                    \
11318                 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,                \
11319         }
11320
11321 static const struct devlink_trap_group devlink_trap_group_generic[] = {
11322         DEVLINK_TRAP_GROUP(L2_DROPS),
11323         DEVLINK_TRAP_GROUP(L3_DROPS),
11324         DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
11325         DEVLINK_TRAP_GROUP(BUFFER_DROPS),
11326         DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
11327         DEVLINK_TRAP_GROUP(ACL_DROPS),
11328         DEVLINK_TRAP_GROUP(STP),
11329         DEVLINK_TRAP_GROUP(LACP),
11330         DEVLINK_TRAP_GROUP(LLDP),
11331         DEVLINK_TRAP_GROUP(MC_SNOOPING),
11332         DEVLINK_TRAP_GROUP(DHCP),
11333         DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
11334         DEVLINK_TRAP_GROUP(BFD),
11335         DEVLINK_TRAP_GROUP(OSPF),
11336         DEVLINK_TRAP_GROUP(BGP),
11337         DEVLINK_TRAP_GROUP(VRRP),
11338         DEVLINK_TRAP_GROUP(PIM),
11339         DEVLINK_TRAP_GROUP(UC_LB),
11340         DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
11341         DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
11342         DEVLINK_TRAP_GROUP(IPV6),
11343         DEVLINK_TRAP_GROUP(PTP_EVENT),
11344         DEVLINK_TRAP_GROUP(PTP_GENERAL),
11345         DEVLINK_TRAP_GROUP(ACL_SAMPLE),
11346         DEVLINK_TRAP_GROUP(ACL_TRAP),
11347         DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
11348 };
11349
11350 static int devlink_trap_generic_verify(const struct devlink_trap *trap)
11351 {
11352         if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
11353                 return -EINVAL;
11354
11355         if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
11356                 return -EINVAL;
11357
11358         if (trap->type != devlink_trap_generic[trap->id].type)
11359                 return -EINVAL;
11360
11361         return 0;
11362 }
11363
11364 static int devlink_trap_driver_verify(const struct devlink_trap *trap)
11365 {
11366         int i;
11367
11368         if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
11369                 return -EINVAL;
11370
11371         for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
11372                 if (!strcmp(trap->name, devlink_trap_generic[i].name))
11373                         return -EEXIST;
11374         }
11375
11376         return 0;
11377 }
11378
11379 static int devlink_trap_verify(const struct devlink_trap *trap)
11380 {
11381         if (!trap || !trap->name)
11382                 return -EINVAL;
11383
11384         if (trap->generic)
11385                 return devlink_trap_generic_verify(trap);
11386         else
11387                 return devlink_trap_driver_verify(trap);
11388 }
11389
11390 static int
11391 devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
11392 {
11393         if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
11394                 return -EINVAL;
11395
11396         if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
11397                 return -EINVAL;
11398
11399         return 0;
11400 }
11401
11402 static int
11403 devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
11404 {
11405         int i;
11406
11407         if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
11408                 return -EINVAL;
11409
11410         for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
11411                 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
11412                         return -EEXIST;
11413         }
11414
11415         return 0;
11416 }
11417
11418 static int devlink_trap_group_verify(const struct devlink_trap_group *group)
11419 {
11420         if (group->generic)
11421                 return devlink_trap_group_generic_verify(group);
11422         else
11423                 return devlink_trap_group_driver_verify(group);
11424 }
11425
11426 static void
11427 devlink_trap_group_notify(struct devlink *devlink,
11428                           const struct devlink_trap_group_item *group_item,
11429                           enum devlink_command cmd)
11430 {
11431         struct sk_buff *msg;
11432         int err;
11433
11434         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
11435                      cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
11436         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11437                 return;
11438
11439         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11440         if (!msg)
11441                 return;
11442
11443         err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
11444                                          0);
11445         if (err) {
11446                 nlmsg_free(msg);
11447                 return;
11448         }
11449
11450         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11451                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11452 }
11453
11454 static int
11455 devlink_trap_item_group_link(struct devlink *devlink,
11456                              struct devlink_trap_item *trap_item)
11457 {
11458         u16 group_id = trap_item->trap->init_group_id;
11459         struct devlink_trap_group_item *group_item;
11460
11461         group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
11462         if (WARN_ON_ONCE(!group_item))
11463                 return -EINVAL;
11464
11465         trap_item->group_item = group_item;
11466
11467         return 0;
11468 }
11469
11470 static void devlink_trap_notify(struct devlink *devlink,
11471                                 const struct devlink_trap_item *trap_item,
11472                                 enum devlink_command cmd)
11473 {
11474         struct sk_buff *msg;
11475         int err;
11476
11477         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
11478                      cmd != DEVLINK_CMD_TRAP_DEL);
11479         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11480                 return;
11481
11482         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11483         if (!msg)
11484                 return;
11485
11486         err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
11487         if (err) {
11488                 nlmsg_free(msg);
11489                 return;
11490         }
11491
11492         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11493                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11494 }
11495
11496 static int
11497 devlink_trap_register(struct devlink *devlink,
11498                       const struct devlink_trap *trap, void *priv)
11499 {
11500         struct devlink_trap_item *trap_item;
11501         int err;
11502
11503         if (devlink_trap_item_lookup(devlink, trap->name))
11504                 return -EEXIST;
11505
11506         trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
11507         if (!trap_item)
11508                 return -ENOMEM;
11509
11510         trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
11511         if (!trap_item->stats) {
11512                 err = -ENOMEM;
11513                 goto err_stats_alloc;
11514         }
11515
11516         trap_item->trap = trap;
11517         trap_item->action = trap->init_action;
11518         trap_item->priv = priv;
11519
11520         err = devlink_trap_item_group_link(devlink, trap_item);
11521         if (err)
11522                 goto err_group_link;
11523
11524         err = devlink->ops->trap_init(devlink, trap, trap_item);
11525         if (err)
11526                 goto err_trap_init;
11527
11528         list_add_tail(&trap_item->list, &devlink->trap_list);
11529         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
11530
11531         return 0;
11532
11533 err_trap_init:
11534 err_group_link:
11535         free_percpu(trap_item->stats);
11536 err_stats_alloc:
11537         kfree(trap_item);
11538         return err;
11539 }
11540
11541 static void devlink_trap_unregister(struct devlink *devlink,
11542                                     const struct devlink_trap *trap)
11543 {
11544         struct devlink_trap_item *trap_item;
11545
11546         trap_item = devlink_trap_item_lookup(devlink, trap->name);
11547         if (WARN_ON_ONCE(!trap_item))
11548                 return;
11549
11550         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
11551         list_del(&trap_item->list);
11552         if (devlink->ops->trap_fini)
11553                 devlink->ops->trap_fini(devlink, trap, trap_item);
11554         free_percpu(trap_item->stats);
11555         kfree(trap_item);
11556 }
11557
11558 static void devlink_trap_disable(struct devlink *devlink,
11559                                  const struct devlink_trap *trap)
11560 {
11561         struct devlink_trap_item *trap_item;
11562
11563         trap_item = devlink_trap_item_lookup(devlink, trap->name);
11564         if (WARN_ON_ONCE(!trap_item))
11565                 return;
11566
11567         devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
11568                                       NULL);
11569         trap_item->action = DEVLINK_TRAP_ACTION_DROP;
11570 }
11571
11572 /**
11573  * devlink_traps_register - Register packet traps with devlink.
11574  * @devlink: devlink.
11575  * @traps: Packet traps.
11576  * @traps_count: Count of provided packet traps.
11577  * @priv: Driver private information.
11578  *
11579  * Return: Non-zero value on failure.
11580  */
11581 int devlink_traps_register(struct devlink *devlink,
11582                            const struct devlink_trap *traps,
11583                            size_t traps_count, void *priv)
11584 {
11585         int i, err;
11586
11587         if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
11588                 return -EINVAL;
11589
11590         mutex_lock(&devlink->lock);
11591         for (i = 0; i < traps_count; i++) {
11592                 const struct devlink_trap *trap = &traps[i];
11593
11594                 err = devlink_trap_verify(trap);
11595                 if (err)
11596                         goto err_trap_verify;
11597
11598                 err = devlink_trap_register(devlink, trap, priv);
11599                 if (err)
11600                         goto err_trap_register;
11601         }
11602         mutex_unlock(&devlink->lock);
11603
11604         return 0;
11605
11606 err_trap_register:
11607 err_trap_verify:
11608         for (i--; i >= 0; i--)
11609                 devlink_trap_unregister(devlink, &traps[i]);
11610         mutex_unlock(&devlink->lock);
11611         return err;
11612 }
11613 EXPORT_SYMBOL_GPL(devlink_traps_register);
11614
11615 /**
11616  * devlink_traps_unregister - Unregister packet traps from devlink.
11617  * @devlink: devlink.
11618  * @traps: Packet traps.
11619  * @traps_count: Count of provided packet traps.
11620  */
11621 void devlink_traps_unregister(struct devlink *devlink,
11622                               const struct devlink_trap *traps,
11623                               size_t traps_count)
11624 {
11625         int i;
11626
11627         mutex_lock(&devlink->lock);
11628         /* Make sure we do not have any packets in-flight while unregistering
11629          * traps by disabling all of them and waiting for a grace period.
11630          */
11631         for (i = traps_count - 1; i >= 0; i--)
11632                 devlink_trap_disable(devlink, &traps[i]);
11633         synchronize_rcu();
11634         for (i = traps_count - 1; i >= 0; i--)
11635                 devlink_trap_unregister(devlink, &traps[i]);
11636         mutex_unlock(&devlink->lock);
11637 }
11638 EXPORT_SYMBOL_GPL(devlink_traps_unregister);
11639
11640 static void
11641 devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
11642                           size_t skb_len)
11643 {
11644         struct devlink_stats *stats;
11645
11646         stats = this_cpu_ptr(trap_stats);
11647         u64_stats_update_begin(&stats->syncp);
11648         u64_stats_add(&stats->rx_bytes, skb_len);
11649         u64_stats_inc(&stats->rx_packets);
11650         u64_stats_update_end(&stats->syncp);
11651 }
11652
11653 static void
11654 devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
11655                                  const struct devlink_trap_item *trap_item,
11656                                  struct devlink_port *in_devlink_port,
11657                                  const struct flow_action_cookie *fa_cookie)
11658 {
11659         metadata->trap_name = trap_item->trap->name;
11660         metadata->trap_group_name = trap_item->group_item->group->name;
11661         metadata->fa_cookie = fa_cookie;
11662         metadata->trap_type = trap_item->trap->type;
11663
11664         spin_lock(&in_devlink_port->type_lock);
11665         if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
11666                 metadata->input_dev = in_devlink_port->type_dev;
11667         spin_unlock(&in_devlink_port->type_lock);
11668 }
11669
11670 /**
11671  * devlink_trap_report - Report trapped packet to drop monitor.
11672  * @devlink: devlink.
11673  * @skb: Trapped packet.
11674  * @trap_ctx: Trap context.
11675  * @in_devlink_port: Input devlink port.
11676  * @fa_cookie: Flow action cookie. Could be NULL.
11677  */
11678 void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
11679                          void *trap_ctx, struct devlink_port *in_devlink_port,
11680                          const struct flow_action_cookie *fa_cookie)
11681
11682 {
11683         struct devlink_trap_item *trap_item = trap_ctx;
11684
11685         devlink_trap_stats_update(trap_item->stats, skb->len);
11686         devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
11687
11688         if (trace_devlink_trap_report_enabled()) {
11689                 struct devlink_trap_metadata metadata = {};
11690
11691                 devlink_trap_report_metadata_set(&metadata, trap_item,
11692                                                  in_devlink_port, fa_cookie);
11693                 trace_devlink_trap_report(devlink, skb, &metadata);
11694         }
11695 }
11696 EXPORT_SYMBOL_GPL(devlink_trap_report);
11697
11698 /**
11699  * devlink_trap_ctx_priv - Trap context to driver private information.
11700  * @trap_ctx: Trap context.
11701  *
11702  * Return: Driver private information passed during registration.
11703  */
11704 void *devlink_trap_ctx_priv(void *trap_ctx)
11705 {
11706         struct devlink_trap_item *trap_item = trap_ctx;
11707
11708         return trap_item->priv;
11709 }
11710 EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
11711
11712 static int
11713 devlink_trap_group_item_policer_link(struct devlink *devlink,
11714                                      struct devlink_trap_group_item *group_item)
11715 {
11716         u32 policer_id = group_item->group->init_policer_id;
11717         struct devlink_trap_policer_item *policer_item;
11718
11719         if (policer_id == 0)
11720                 return 0;
11721
11722         policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
11723         if (WARN_ON_ONCE(!policer_item))
11724                 return -EINVAL;
11725
11726         group_item->policer_item = policer_item;
11727
11728         return 0;
11729 }
11730
11731 static int
11732 devlink_trap_group_register(struct devlink *devlink,
11733                             const struct devlink_trap_group *group)
11734 {
11735         struct devlink_trap_group_item *group_item;
11736         int err;
11737
11738         if (devlink_trap_group_item_lookup(devlink, group->name))
11739                 return -EEXIST;
11740
11741         group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
11742         if (!group_item)
11743                 return -ENOMEM;
11744
11745         group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
11746         if (!group_item->stats) {
11747                 err = -ENOMEM;
11748                 goto err_stats_alloc;
11749         }
11750
11751         group_item->group = group;
11752
11753         err = devlink_trap_group_item_policer_link(devlink, group_item);
11754         if (err)
11755                 goto err_policer_link;
11756
11757         if (devlink->ops->trap_group_init) {
11758                 err = devlink->ops->trap_group_init(devlink, group);
11759                 if (err)
11760                         goto err_group_init;
11761         }
11762
11763         list_add_tail(&group_item->list, &devlink->trap_group_list);
11764         devlink_trap_group_notify(devlink, group_item,
11765                                   DEVLINK_CMD_TRAP_GROUP_NEW);
11766
11767         return 0;
11768
11769 err_group_init:
11770 err_policer_link:
11771         free_percpu(group_item->stats);
11772 err_stats_alloc:
11773         kfree(group_item);
11774         return err;
11775 }
11776
11777 static void
11778 devlink_trap_group_unregister(struct devlink *devlink,
11779                               const struct devlink_trap_group *group)
11780 {
11781         struct devlink_trap_group_item *group_item;
11782
11783         group_item = devlink_trap_group_item_lookup(devlink, group->name);
11784         if (WARN_ON_ONCE(!group_item))
11785                 return;
11786
11787         devlink_trap_group_notify(devlink, group_item,
11788                                   DEVLINK_CMD_TRAP_GROUP_DEL);
11789         list_del(&group_item->list);
11790         free_percpu(group_item->stats);
11791         kfree(group_item);
11792 }
11793
11794 /**
11795  * devlink_trap_groups_register - Register packet trap groups with devlink.
11796  * @devlink: devlink.
11797  * @groups: Packet trap groups.
11798  * @groups_count: Count of provided packet trap groups.
11799  *
11800  * Return: Non-zero value on failure.
11801  */
11802 int devlink_trap_groups_register(struct devlink *devlink,
11803                                  const struct devlink_trap_group *groups,
11804                                  size_t groups_count)
11805 {
11806         int i, err;
11807
11808         mutex_lock(&devlink->lock);
11809         for (i = 0; i < groups_count; i++) {
11810                 const struct devlink_trap_group *group = &groups[i];
11811
11812                 err = devlink_trap_group_verify(group);
11813                 if (err)
11814                         goto err_trap_group_verify;
11815
11816                 err = devlink_trap_group_register(devlink, group);
11817                 if (err)
11818                         goto err_trap_group_register;
11819         }
11820         mutex_unlock(&devlink->lock);
11821
11822         return 0;
11823
11824 err_trap_group_register:
11825 err_trap_group_verify:
11826         for (i--; i >= 0; i--)
11827                 devlink_trap_group_unregister(devlink, &groups[i]);
11828         mutex_unlock(&devlink->lock);
11829         return err;
11830 }
11831 EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
11832
11833 /**
11834  * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
11835  * @devlink: devlink.
11836  * @groups: Packet trap groups.
11837  * @groups_count: Count of provided packet trap groups.
11838  */
11839 void devlink_trap_groups_unregister(struct devlink *devlink,
11840                                     const struct devlink_trap_group *groups,
11841                                     size_t groups_count)
11842 {
11843         int i;
11844
11845         mutex_lock(&devlink->lock);
11846         for (i = groups_count - 1; i >= 0; i--)
11847                 devlink_trap_group_unregister(devlink, &groups[i]);
11848         mutex_unlock(&devlink->lock);
11849 }
11850 EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
11851
11852 static void
11853 devlink_trap_policer_notify(struct devlink *devlink,
11854                             const struct devlink_trap_policer_item *policer_item,
11855                             enum devlink_command cmd)
11856 {
11857         struct sk_buff *msg;
11858         int err;
11859
11860         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
11861                      cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
11862         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11863                 return;
11864
11865         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11866         if (!msg)
11867                 return;
11868
11869         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
11870                                            0, 0);
11871         if (err) {
11872                 nlmsg_free(msg);
11873                 return;
11874         }
11875
11876         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11877                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11878 }
11879
11880 static int
11881 devlink_trap_policer_register(struct devlink *devlink,
11882                               const struct devlink_trap_policer *policer)
11883 {
11884         struct devlink_trap_policer_item *policer_item;
11885         int err;
11886
11887         if (devlink_trap_policer_item_lookup(devlink, policer->id))
11888                 return -EEXIST;
11889
11890         policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
11891         if (!policer_item)
11892                 return -ENOMEM;
11893
11894         policer_item->policer = policer;
11895         policer_item->rate = policer->init_rate;
11896         policer_item->burst = policer->init_burst;
11897
11898         if (devlink->ops->trap_policer_init) {
11899                 err = devlink->ops->trap_policer_init(devlink, policer);
11900                 if (err)
11901                         goto err_policer_init;
11902         }
11903
11904         list_add_tail(&policer_item->list, &devlink->trap_policer_list);
11905         devlink_trap_policer_notify(devlink, policer_item,
11906                                     DEVLINK_CMD_TRAP_POLICER_NEW);
11907
11908         return 0;
11909
11910 err_policer_init:
11911         kfree(policer_item);
11912         return err;
11913 }
11914
11915 static void
11916 devlink_trap_policer_unregister(struct devlink *devlink,
11917                                 const struct devlink_trap_policer *policer)
11918 {
11919         struct devlink_trap_policer_item *policer_item;
11920
11921         policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
11922         if (WARN_ON_ONCE(!policer_item))
11923                 return;
11924
11925         devlink_trap_policer_notify(devlink, policer_item,
11926                                     DEVLINK_CMD_TRAP_POLICER_DEL);
11927         list_del(&policer_item->list);
11928         if (devlink->ops->trap_policer_fini)
11929                 devlink->ops->trap_policer_fini(devlink, policer);
11930         kfree(policer_item);
11931 }
11932
11933 /**
11934  * devlink_trap_policers_register - Register packet trap policers with devlink.
11935  * @devlink: devlink.
11936  * @policers: Packet trap policers.
11937  * @policers_count: Count of provided packet trap policers.
11938  *
11939  * Return: Non-zero value on failure.
11940  */
11941 int
11942 devlink_trap_policers_register(struct devlink *devlink,
11943                                const struct devlink_trap_policer *policers,
11944                                size_t policers_count)
11945 {
11946         int i, err;
11947
11948         mutex_lock(&devlink->lock);
11949         for (i = 0; i < policers_count; i++) {
11950                 const struct devlink_trap_policer *policer = &policers[i];
11951
11952                 if (WARN_ON(policer->id == 0 ||
11953                             policer->max_rate < policer->min_rate ||
11954                             policer->max_burst < policer->min_burst)) {
11955                         err = -EINVAL;
11956                         goto err_trap_policer_verify;
11957                 }
11958
11959                 err = devlink_trap_policer_register(devlink, policer);
11960                 if (err)
11961                         goto err_trap_policer_register;
11962         }
11963         mutex_unlock(&devlink->lock);
11964
11965         return 0;
11966
11967 err_trap_policer_register:
11968 err_trap_policer_verify:
11969         for (i--; i >= 0; i--)
11970                 devlink_trap_policer_unregister(devlink, &policers[i]);
11971         mutex_unlock(&devlink->lock);
11972         return err;
11973 }
11974 EXPORT_SYMBOL_GPL(devlink_trap_policers_register);
11975
11976 /**
11977  * devlink_trap_policers_unregister - Unregister packet trap policers from devlink.
11978  * @devlink: devlink.
11979  * @policers: Packet trap policers.
11980  * @policers_count: Count of provided packet trap policers.
11981  */
11982 void
11983 devlink_trap_policers_unregister(struct devlink *devlink,
11984                                  const struct devlink_trap_policer *policers,
11985                                  size_t policers_count)
11986 {
11987         int i;
11988
11989         mutex_lock(&devlink->lock);
11990         for (i = policers_count - 1; i >= 0; i--)
11991                 devlink_trap_policer_unregister(devlink, &policers[i]);
11992         mutex_unlock(&devlink->lock);
11993 }
11994 EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister);
11995
11996 static void __devlink_compat_running_version(struct devlink *devlink,
11997                                              char *buf, size_t len)
11998 {
11999         const struct nlattr *nlattr;
12000         struct devlink_info_req req;
12001         struct sk_buff *msg;
12002         int rem, err;
12003
12004         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
12005         if (!msg)
12006                 return;
12007
12008         req.msg = msg;
12009         err = devlink->ops->info_get(devlink, &req, NULL);
12010         if (err)
12011                 goto free_msg;
12012
12013         nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
12014                 const struct nlattr *kv;
12015                 int rem_kv;
12016
12017                 if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
12018                         continue;
12019
12020                 nla_for_each_nested(kv, nlattr, rem_kv) {
12021                         if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
12022                                 continue;
12023
12024                         strlcat(buf, nla_data(kv), len);
12025                         strlcat(buf, " ", len);
12026                 }
12027         }
12028 free_msg:
12029         nlmsg_free(msg);
12030 }
12031
12032 static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
12033 {
12034         if (!dev->netdev_ops->ndo_get_devlink_port)
12035                 return NULL;
12036
12037         return dev->netdev_ops->ndo_get_devlink_port(dev);
12038 }
12039
12040 void devlink_compat_running_version(struct devlink *devlink,
12041                                     char *buf, size_t len)
12042 {
12043         if (!devlink->ops->info_get)
12044                 return;
12045
12046         mutex_lock(&devlink->lock);
12047         __devlink_compat_running_version(devlink, buf, len);
12048         mutex_unlock(&devlink->lock);
12049 }
12050
12051 int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
12052 {
12053         struct devlink_flash_update_params params = {};
12054         int ret;
12055
12056         if (!devlink->ops->flash_update)
12057                 return -EOPNOTSUPP;
12058
12059         ret = request_firmware(&params.fw, file_name, devlink->dev);
12060         if (ret)
12061                 return ret;
12062
12063         mutex_lock(&devlink->lock);
12064         devlink_flash_update_begin_notify(devlink);
12065         ret = devlink->ops->flash_update(devlink, &params, NULL);
12066         devlink_flash_update_end_notify(devlink);
12067         mutex_unlock(&devlink->lock);
12068
12069         release_firmware(params.fw);
12070
12071         return ret;
12072 }
12073
12074 int devlink_compat_phys_port_name_get(struct net_device *dev,
12075                                       char *name, size_t len)
12076 {
12077         struct devlink_port *devlink_port;
12078
12079         /* RTNL mutex is held here which ensures that devlink_port
12080          * instance cannot disappear in the middle. No need to take
12081          * any devlink lock as only permanent values are accessed.
12082          */
12083         ASSERT_RTNL();
12084
12085         devlink_port = netdev_to_devlink_port(dev);
12086         if (!devlink_port)
12087                 return -EOPNOTSUPP;
12088
12089         return __devlink_port_phys_port_name_get(devlink_port, name, len);
12090 }
12091
12092 int devlink_compat_switch_id_get(struct net_device *dev,
12093                                  struct netdev_phys_item_id *ppid)
12094 {
12095         struct devlink_port *devlink_port;
12096
12097         /* Caller must hold RTNL mutex or reference to dev, which ensures that
12098          * devlink_port instance cannot disappear in the middle. No need to take
12099          * any devlink lock as only permanent values are accessed.
12100          */
12101         devlink_port = netdev_to_devlink_port(dev);
12102         if (!devlink_port || !devlink_port->switch_port)
12103                 return -EOPNOTSUPP;
12104
12105         memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
12106
12107         return 0;
12108 }
12109
12110 static void __net_exit devlink_pernet_pre_exit(struct net *net)
12111 {
12112         struct devlink *devlink;
12113         u32 actions_performed;
12114         unsigned long index;
12115         int err;
12116
12117         /* In case network namespace is getting destroyed, reload
12118          * all devlink instances from this namespace into init_net.
12119          */
12120         mutex_lock(&devlink_mutex);
12121         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
12122                 if (!devlink_try_get(devlink))
12123                         continue;
12124
12125                 if (!net_eq(devlink_net(devlink), net))
12126                         goto retry;
12127
12128                 WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
12129                 err = devlink_reload(devlink, &init_net,
12130                                      DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
12131                                      DEVLINK_RELOAD_LIMIT_UNSPEC,
12132                                      &actions_performed, NULL);
12133                 if (err && err != -EOPNOTSUPP)
12134                         pr_warn("Failed to reload devlink instance into init_net\n");
12135 retry:
12136                 devlink_put(devlink);
12137         }
12138         mutex_unlock(&devlink_mutex);
12139 }
12140
12141 static struct pernet_operations devlink_pernet_ops __net_initdata = {
12142         .pre_exit = devlink_pernet_pre_exit,
12143 };
12144
12145 static int __init devlink_init(void)
12146 {
12147         int err;
12148
12149         err = genl_register_family(&devlink_nl_family);
12150         if (err)
12151                 goto out;
12152         err = register_pernet_subsys(&devlink_pernet_ops);
12153
12154 out:
12155         WARN_ON(err);
12156         return err;
12157 }
12158
12159 subsys_initcall(devlink_init);