net/core/devlink.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/core/devlink.c - Network physical/parent device Netlink interface
   4  *
   5  * Heavily inspired by net/wireless/
   6  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   7  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   8  */
   9
  10 #include <linux/etherdevice.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/types.h>
  14 #include <linux/slab.h>
  15 #include <linux/gfp.h>
  16 #include <linux/device.h>
  17 #include <linux/list.h>
  18 #include <linux/netdevice.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/refcount.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/u64_stats_sync.h>
  23 #include <linux/timekeeping.h>
  24 #include <rdma/ib_verbs.h>
  25 #include <net/netlink.h>
  26 #include <net/genetlink.h>
  27 #include <net/rtnetlink.h>
  28 #include <net/net_namespace.h>
  29 #include <net/sock.h>
  30 #include <net/devlink.h>
  31 #define CREATE_TRACE_POINTS
  32 #include <trace/events/devlink.h>
  33
  34 #define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
  35         (__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
  36
  37 struct devlink_dev_stats {
  38         u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  39         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  40 };
  41
  42 struct devlink {
  43         u32 index;
  44         struct list_head port_list;
  45         struct list_head rate_list;
  46         struct list_head sb_list;
  47         struct list_head dpipe_table_list;
  48         struct list_head resource_list;
  49         struct list_head param_list;
  50         struct list_head region_list;
  51         struct list_head reporter_list;
  52         struct mutex reporters_lock; /* protects reporter_list */
  53         struct devlink_dpipe_headers *dpipe_headers;
  54         struct list_head trap_list;
  55         struct list_head trap_group_list;
  56         struct list_head trap_policer_list;
  57         struct list_head linecard_list;
  58         struct mutex linecards_lock; /* protects linecard_list */
  59         const struct devlink_ops *ops;
  60         u64 features;
  61         struct xarray snapshot_ids;
  62         struct devlink_dev_stats stats;
  63         struct device *dev;
  64         possible_net_t _net;
  65         /* Serializes access to devlink instance specific objects such as
  66          * port, sb, dpipe, resource, params, region, traps and more.
  67          */
  68         struct mutex lock;
  69         u8 reload_failed:1;
  70         refcount_t refcount;
  71         struct completion comp;
  72         char priv[] __aligned(NETDEV_ALIGN);
  73 };
  74
  75 struct devlink_linecard_ops;
  76 struct devlink_linecard_type;
  77
  78 struct devlink_linecard {
  79         struct list_head list;
  80         struct devlink *devlink;
  81         unsigned int index;
  82         refcount_t refcount;
  83         const struct devlink_linecard_ops *ops;
  84         void *priv;
  85         enum devlink_linecard_state state;
  86         struct mutex state_lock; /* Protects state */
  87         const char *type;
  88         struct devlink_linecard_type *types;
  89         unsigned int types_count;
  90 };
  91
  92 /**
  93  * struct devlink_resource - devlink resource
  94  * @name: name of the resource
  95  * @id: id, per devlink instance
  96  * @size: size of the resource
  97  * @size_new: updated size of the resource, reload is needed
  98  * @size_valid: valid in case the total size of the resource is valid
  99  *              including its children
 100  * @parent: parent resource
 101  * @size_params: size parameters
 102  * @list: parent list
 103  * @resource_list: list of child resources
 104  * @occ_get: occupancy getter callback
 105  * @occ_get_priv: occupancy getter callback priv
 106  */
 107 struct devlink_resource {
 108         const char *name;
 109         u64 id;
 110         u64 size;
 111         u64 size_new;
 112         bool size_valid;
 113         struct devlink_resource *parent;
 114         struct devlink_resource_size_params size_params;
 115         struct list_head list;
 116         struct list_head resource_list;
 117         devlink_resource_occ_get_t *occ_get;
 118         void *occ_get_priv;
 119 };
 120
 121 void *devlink_priv(struct devlink *devlink)
 122 {
 123         return &devlink->priv;
 124 }
 125 EXPORT_SYMBOL_GPL(devlink_priv);
 126
 127 struct devlink *priv_to_devlink(void *priv)
 128 {
 129         return container_of(priv, struct devlink, priv);
 130 }
 131 EXPORT_SYMBOL_GPL(priv_to_devlink);
 132
 133 struct device *devlink_to_dev(const struct devlink *devlink)
 134 {
 135         return devlink->dev;
 136 }
 137 EXPORT_SYMBOL_GPL(devlink_to_dev);
 138
 139 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
 140         {
 141                 .name = "destination mac",
 142                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
 143                 .bitwidth = 48,
 144         },
 145 };
 146
 147 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
 148         .name = "ethernet",
 149         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
 150         .fields = devlink_dpipe_fields_ethernet,
 151         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
 152         .global = true,
 153 };
 154 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
 155
 156 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
 157         {
 158                 .name = "destination ip",
 159                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
 160                 .bitwidth = 32,
 161         },
 162 };
 163
 164 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
 165         .name = "ipv4",
 166         .id = DEVLINK_DPIPE_HEADER_IPV4,
 167         .fields = devlink_dpipe_fields_ipv4,
 168         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
 169         .global = true,
 170 };
 171 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
 172
 173 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
 174         {
 175                 .name = "destination ip",
 176                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
 177                 .bitwidth = 128,
 178         },
 179 };
 180
 181 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
 182         .name = "ipv6",
 183         .id = DEVLINK_DPIPE_HEADER_IPV6,
 184         .fields = devlink_dpipe_fields_ipv6,
 185         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
 186         .global = true,
 187 };
 188 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
 189
 190 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
 191 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
 192 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
 193
 194 static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
 195         [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
 196         [DEVLINK_PORT_FN_ATTR_STATE] =
 197                 NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
 198                                  DEVLINK_PORT_FN_STATE_ACTIVE),
 199 };
 200
 201 static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
 202 #define DEVLINK_REGISTERED XA_MARK_1
 203
 204 /* devlink instances are open to the access from the user space after
 205  * devlink_register() call. Such logical barrier allows us to have certain
 206  * expectations related to locking.
 207  *
 208  * Before *_register() - we are in initialization stage and no parallel
 209  * access possible to the devlink instance. All drivers perform that phase
 210  * by implicitly holding device_lock.
 211  *
 212  * After *_register() - users and driver can access devlink instance at
 213  * the same time.
 214  */
 215 #define ASSERT_DEVLINK_REGISTERED(d)                                           \
 216         WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 217 #define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
 218         WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 219
 220 /* devlink_mutex
 221  *
 222  * An overall lock guarding every operation coming from userspace.
 223  * It also guards devlink devices list and it is taken when
 224  * driver registers/unregisters it.
 225  */
 226 static DEFINE_MUTEX(devlink_mutex);
 227
 228 struct net *devlink_net(const struct devlink *devlink)
 229 {
 230         return read_pnet(&devlink->_net);
 231 }
 232 EXPORT_SYMBOL_GPL(devlink_net);
 233
 234 void devlink_put(struct devlink *devlink)
 235 {
 236         if (refcount_dec_and_test(&devlink->refcount))
 237                 complete(&devlink->comp);
 238 }
 239
 240 struct devlink *__must_check devlink_try_get(struct devlink *devlink)
 241 {
 242         if (refcount_inc_not_zero(&devlink->refcount))
 243                 return devlink;
 244         return NULL;
 245 }
 246
 247 void devl_assert_locked(struct devlink *devlink)
 248 {
 249         lockdep_assert_held(&devlink->lock);
 250 }
 251 EXPORT_SYMBOL_GPL(devl_assert_locked);
 252
 253 #ifdef CONFIG_LOCKDEP
 254 /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
 255 bool devl_lock_is_held(struct devlink *devlink)
 256 {
 257         return lockdep_is_held(&devlink->lock);
 258 }
 259 EXPORT_SYMBOL_GPL(devl_lock_is_held);
 260 #endif
 261
 262 void devl_lock(struct devlink *devlink)
 263 {
 264         mutex_lock(&devlink->lock);
 265 }
 266 EXPORT_SYMBOL_GPL(devl_lock);
 267
 268 void devl_unlock(struct devlink *devlink)
 269 {
 270         mutex_unlock(&devlink->lock);
 271 }
 272 EXPORT_SYMBOL_GPL(devl_unlock);
 273
 274 static struct devlink *devlink_get_from_attrs(struct net *net,
 275                                               struct nlattr **attrs)
 276 {
 277         struct devlink *devlink;
 278         unsigned long index;
 279         bool found = false;
 280         char *busname;
 281         char *devname;
 282
 283         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 284                 return ERR_PTR(-EINVAL);
 285
 286         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 287         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 288
 289         lockdep_assert_held(&devlink_mutex);
 290
 291         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
 292                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 293                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 294                     net_eq(devlink_net(devlink), net)) {
 295                         found = true;
 296                         break;
 297                 }
 298         }
 299
 300         if (!found || !devlink_try_get(devlink))
 301                 devlink = ERR_PTR(-ENODEV);
 302
 303         return devlink;
 304 }
 305
 306 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 307                                                       unsigned int port_index)
 308 {
 309         struct devlink_port *devlink_port;
 310
 311         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 312                 if (devlink_port->index == port_index)
 313                         return devlink_port;
 314         }
 315         return NULL;
 316 }
 317
 318 static bool devlink_port_index_exists(struct devlink *devlink,
 319                                       unsigned int port_index)
 320 {
 321         return devlink_port_get_by_index(devlink, port_index);
 322 }
 323
 324 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 325                                                         struct nlattr **attrs)
 326 {
 327         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 328                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 329                 struct devlink_port *devlink_port;
 330
 331                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 332                 if (!devlink_port)
 333                         return ERR_PTR(-ENODEV);
 334                 return devlink_port;
 335         }
 336         return ERR_PTR(-EINVAL);
 337 }
 338
 339 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 340                                                        struct genl_info *info)
 341 {
 342         return devlink_port_get_from_attrs(devlink, info->attrs);
 343 }
 344
 345 static inline bool
 346 devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
 347 {
 348         return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
 349 }
 350
 351 static inline bool
 352 devlink_rate_is_node(struct devlink_rate *devlink_rate)
 353 {
 354         return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
 355 }
 356
 357 static struct devlink_rate *
 358 devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
 359 {
 360         struct devlink_rate *devlink_rate;
 361         struct devlink_port *devlink_port;
 362
 363         devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
 364         if (IS_ERR(devlink_port))
 365                 return ERR_CAST(devlink_port);
 366         devlink_rate = devlink_port->devlink_rate;
 367         return devlink_rate ?: ERR_PTR(-ENODEV);
 368 }
 369
 370 static struct devlink_rate *
 371 devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
 372 {
 373         static struct devlink_rate *devlink_rate;
 374
 375         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
 376                 if (devlink_rate_is_node(devlink_rate) &&
 377                     !strcmp(node_name, devlink_rate->name))
 378                         return devlink_rate;
 379         }
 380         return ERR_PTR(-ENODEV);
 381 }
 382
 383 static struct devlink_rate *
 384 devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 385 {
 386         const char *rate_node_name;
 387         size_t len;
 388
 389         if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 390                 return ERR_PTR(-EINVAL);
 391         rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
 392         len = strlen(rate_node_name);
 393         /* Name cannot be empty or decimal number */
 394         if (!len || strspn(rate_node_name, "0123456789") == len)
 395                 return ERR_PTR(-EINVAL);
 396
 397         return devlink_rate_node_get_by_name(devlink, rate_node_name);
 398 }
 399
 400 static struct devlink_rate *
 401 devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
 402 {
 403         return devlink_rate_node_get_from_attrs(devlink, info->attrs);
 404 }
 405
 406 static struct devlink_rate *
 407 devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
 408 {
 409         struct nlattr **attrs = info->attrs;
 410
 411         if (attrs[DEVLINK_ATTR_PORT_INDEX])
 412                 return devlink_rate_leaf_get_from_info(devlink, info);
 413         else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 414                 return devlink_rate_node_get_from_info(devlink, info);
 415         else
 416                 return ERR_PTR(-EINVAL);
 417 }
 418
 419 static struct devlink_linecard *
 420 devlink_linecard_get_by_index(struct devlink *devlink,
 421                               unsigned int linecard_index)
 422 {
 423         struct devlink_linecard *devlink_linecard;
 424
 425         list_for_each_entry(devlink_linecard, &devlink->linecard_list, list) {
 426                 if (devlink_linecard->index == linecard_index)
 427                         return devlink_linecard;
 428         }
 429         return NULL;
 430 }
 431
 432 static bool devlink_linecard_index_exists(struct devlink *devlink,
 433                                           unsigned int linecard_index)
 434 {
 435         return devlink_linecard_get_by_index(devlink, linecard_index);
 436 }
 437
 438 static struct devlink_linecard *
 439 devlink_linecard_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 440 {
 441         if (attrs[DEVLINK_ATTR_LINECARD_INDEX]) {
 442                 u32 linecard_index = nla_get_u32(attrs[DEVLINK_ATTR_LINECARD_INDEX]);
 443                 struct devlink_linecard *linecard;
 444
 445                 mutex_lock(&devlink->linecards_lock);
 446                 linecard = devlink_linecard_get_by_index(devlink, linecard_index);
 447                 if (linecard)
 448                         refcount_inc(&linecard->refcount);
 449                 mutex_unlock(&devlink->linecards_lock);
 450                 if (!linecard)
 451                         return ERR_PTR(-ENODEV);
 452                 return linecard;
 453         }
 454         return ERR_PTR(-EINVAL);
 455 }
 456
 457 static struct devlink_linecard *
 458 devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info)
 459 {
 460         return devlink_linecard_get_from_attrs(devlink, info->attrs);
 461 }
 462
 463 static void devlink_linecard_put(struct devlink_linecard *linecard)
 464 {
 465         if (refcount_dec_and_test(&linecard->refcount)) {
 466                 mutex_destroy(&linecard->state_lock);
 467                 kfree(linecard);
 468         }
 469 }
 470
 471 struct devlink_sb {
 472         struct list_head list;
 473         unsigned int index;
 474         u32 size;
 475         u16 ingress_pools_count;
 476         u16 egress_pools_count;
 477         u16 ingress_tc_count;
 478         u16 egress_tc_count;
 479 };
 480
 481 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 482 {
 483         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 484 }
 485
 486 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 487                                                   unsigned int sb_index)
 488 {
 489         struct devlink_sb *devlink_sb;
 490
 491         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 492                 if (devlink_sb->index == sb_index)
 493                         return devlink_sb;
 494         }
 495         return NULL;
 496 }
 497
 498 static bool devlink_sb_index_exists(struct devlink *devlink,
 499                                     unsigned int sb_index)
 500 {
 501         return devlink_sb_get_by_index(devlink, sb_index);
 502 }
 503
 504 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 505                                                     struct nlattr **attrs)
 506 {
 507         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 508                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 509                 struct devlink_sb *devlink_sb;
 510
 511                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 512                 if (!devlink_sb)
 513                         return ERR_PTR(-ENODEV);
 514                 return devlink_sb;
 515         }
 516         return ERR_PTR(-EINVAL);
 517 }
 518
 519 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 520                                                    struct genl_info *info)
 521 {
 522         return devlink_sb_get_from_attrs(devlink, info->attrs);
 523 }
 524
 525 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 526                                                 struct nlattr **attrs,
 527                                                 u16 *p_pool_index)
 528 {
 529         u16 val;
 530
 531         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 532                 return -EINVAL;
 533
 534         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 535         if (val >= devlink_sb_pool_count(devlink_sb))
 536                 return -EINVAL;
 537         *p_pool_index = val;
 538         return 0;
 539 }
 540
 541 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 542                                                struct genl_info *info,
 543                                                u16 *p_pool_index)
 544 {
 545         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 546                                                     p_pool_index);
 547 }
 548
 549 static int
 550 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 551                                     enum devlink_sb_pool_type *p_pool_type)
 552 {
 553         u8 val;
 554
 555         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 556                 return -EINVAL;
 557
 558         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 559         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 560             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 561                 return -EINVAL;
 562         *p_pool_type = val;
 563         return 0;
 564 }
 565
 566 static int
 567 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 568                                    enum devlink_sb_pool_type *p_pool_type)
 569 {
 570         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 571 }
 572
 573 static int
 574 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 575                                   enum devlink_sb_threshold_type *p_th_type)
 576 {
 577         u8 val;
 578
 579         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 580                 return -EINVAL;
 581
 582         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 583         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 584             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 585                 return -EINVAL;
 586         *p_th_type = val;
 587         return 0;
 588 }
 589
 590 static int
 591 devlink_sb_th_type_get_from_info(struct genl_info *info,
 592                                  enum devlink_sb_threshold_type *p_th_type)
 593 {
 594         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 595 }
 596
 597 static int
 598 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 599                                    struct nlattr **attrs,
 600                                    enum devlink_sb_pool_type pool_type,
 601                                    u16 *p_tc_index)
 602 {
 603         u16 val;
 604
 605         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 606                 return -EINVAL;
 607
 608         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 609         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 610             val >= devlink_sb->ingress_tc_count)
 611                 return -EINVAL;
 612         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 613             val >= devlink_sb->egress_tc_count)
 614                 return -EINVAL;
 615         *p_tc_index = val;
 616         return 0;
 617 }
 618
 619 static int
 620 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 621                                   struct genl_info *info,
 622                                   enum devlink_sb_pool_type pool_type,
 623                                   u16 *p_tc_index)
 624 {
 625         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 626                                                   pool_type, p_tc_index);
 627 }
 628
 629 struct devlink_region {
 630         struct devlink *devlink;
 631         struct devlink_port *port;
 632         struct list_head list;
 633         union {
 634                 const struct devlink_region_ops *ops;
 635                 const struct devlink_port_region_ops *port_ops;
 636         };
 637         struct list_head snapshot_list;
 638         u32 max_snapshots;
 639         u32 cur_snapshots;
 640         u64 size;
 641 };
 642
 643 struct devlink_snapshot {
 644         struct list_head list;
 645         struct devlink_region *region;
 646         u8 *data;
 647         u32 id;
 648 };
 649
 650 static struct devlink_region *
 651 devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
 652 {
 653         struct devlink_region *region;
 654
 655         list_for_each_entry(region, &devlink->region_list, list)
 656                 if (!strcmp(region->ops->name, region_name))
 657                         return region;
 658
 659         return NULL;
 660 }
 661
 662 static struct devlink_region *
 663 devlink_port_region_get_by_name(struct devlink_port *port,
 664                                 const char *region_name)
 665 {
 666         struct devlink_region *region;
 667
 668         list_for_each_entry(region, &port->region_list, list)
 669                 if (!strcmp(region->ops->name, region_name))
 670                         return region;
 671
 672         return NULL;
 673 }
 674
 675 static struct devlink_snapshot *
 676 devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
 677 {
 678         struct devlink_snapshot *snapshot;
 679
 680         list_for_each_entry(snapshot, &region->snapshot_list, list)
 681                 if (snapshot->id == id)
 682                         return snapshot;
 683
 684         return NULL;
 685 }
 686
 687 #define DEVLINK_NL_FLAG_NEED_PORT               BIT(0)
 688 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT    BIT(1)
 689 #define DEVLINK_NL_FLAG_NEED_RATE               BIT(2)
 690 #define DEVLINK_NL_FLAG_NEED_RATE_NODE          BIT(3)
 691 #define DEVLINK_NL_FLAG_NEED_LINECARD           BIT(4)
 692
 693 /* The per devlink instance lock is taken by default in the pre-doit
 694  * operation, yet several commands do not require this. The global
 695  * devlink lock is taken and protects from disruption by user-calls.
 696  */
 697 #define DEVLINK_NL_FLAG_NO_LOCK                 BIT(5)
 698
 699 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 700                                struct sk_buff *skb, struct genl_info *info)
 701 {
 702         struct devlink_linecard *linecard;
 703         struct devlink_port *devlink_port;
 704         struct devlink *devlink;
 705         int err;
 706
 707         mutex_lock(&devlink_mutex);
 708         devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
 709         if (IS_ERR(devlink)) {
 710                 mutex_unlock(&devlink_mutex);
 711                 return PTR_ERR(devlink);
 712         }
 713         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 714                 devl_lock(devlink);
 715         info->user_ptr[0] = devlink;
 716         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 717                 devlink_port = devlink_port_get_from_info(devlink, info);
 718                 if (IS_ERR(devlink_port)) {
 719                         err = PTR_ERR(devlink_port);
 720                         goto unlock;
 721                 }
 722                 info->user_ptr[1] = devlink_port;
 723         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 724                 devlink_port = devlink_port_get_from_info(devlink, info);
 725                 if (!IS_ERR(devlink_port))
 726                         info->user_ptr[1] = devlink_port;
 727         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
 728                 struct devlink_rate *devlink_rate;
 729
 730                 devlink_rate = devlink_rate_get_from_info(devlink, info);
 731                 if (IS_ERR(devlink_rate)) {
 732                         err = PTR_ERR(devlink_rate);
 733                         goto unlock;
 734                 }
 735                 info->user_ptr[1] = devlink_rate;
 736         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
 737                 struct devlink_rate *rate_node;
 738
 739                 rate_node = devlink_rate_node_get_from_info(devlink, info);
 740                 if (IS_ERR(rate_node)) {
 741                         err = PTR_ERR(rate_node);
 742                         goto unlock;
 743                 }
 744                 info->user_ptr[1] = rate_node;
 745         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
 746                 linecard = devlink_linecard_get_from_info(devlink, info);
 747                 if (IS_ERR(linecard)) {
 748                         err = PTR_ERR(linecard);
 749                         goto unlock;
 750                 }
 751                 info->user_ptr[1] = linecard;
 752         }
 753         return 0;
 754
 755 unlock:
 756         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 757                 devl_unlock(devlink);
 758         devlink_put(devlink);
 759         mutex_unlock(&devlink_mutex);
 760         return err;
 761 }
 762
 763 static void devlink_nl_post_doit(const struct genl_ops *ops,
 764                                  struct sk_buff *skb, struct genl_info *info)
 765 {
 766         struct devlink_linecard *linecard;
 767         struct devlink *devlink;
 768
 769         devlink = info->user_ptr[0];
 770         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
 771                 linecard = info->user_ptr[1];
 772                 devlink_linecard_put(linecard);
 773         }
 774         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 775                 devl_unlock(devlink);
 776         devlink_put(devlink);
 777         mutex_unlock(&devlink_mutex);
 778 }
 779
 780 static struct genl_family devlink_nl_family;
 781
 782 enum devlink_multicast_groups {
 783         DEVLINK_MCGRP_CONFIG,
 784 };
 785
 786 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 787         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 788 };
 789
 790 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 791 {
 792         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 793                 return -EMSGSIZE;
 794         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 795                 return -EMSGSIZE;
 796         return 0;
 797 }
 798
 799 struct devlink_reload_combination {
 800         enum devlink_reload_action action;
 801         enum devlink_reload_limit limit;
 802 };
 803
 804 static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
 805         {
 806                 /* can't reinitialize driver with no down time */
 807                 .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 808                 .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
 809         },
 810 };
 811
 812 static bool
 813 devlink_reload_combination_is_invalid(enum devlink_reload_action action,
 814                                       enum devlink_reload_limit limit)
 815 {
 816         int i;
 817
 818         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
 819                 if (devlink_reload_invalid_combinations[i].action == action &&
 820                     devlink_reload_invalid_combinations[i].limit == limit)
 821                         return true;
 822         return false;
 823 }
 824
 825 static bool
 826 devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
 827 {
 828         return test_bit(action, &devlink->ops->reload_actions);
 829 }
 830
 831 static bool
 832 devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
 833 {
 834         return test_bit(limit, &devlink->ops->reload_limits);
 835 }
 836
 837 static int devlink_reload_stat_put(struct sk_buff *msg,
 838                                    enum devlink_reload_limit limit, u32 value)
 839 {
 840         struct nlattr *reload_stats_entry;
 841
 842         reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
 843         if (!reload_stats_entry)
 844                 return -EMSGSIZE;
 845
 846         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
 847             nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
 848                 goto nla_put_failure;
 849         nla_nest_end(msg, reload_stats_entry);
 850         return 0;
 851
 852 nla_put_failure:
 853         nla_nest_cancel(msg, reload_stats_entry);
 854         return -EMSGSIZE;
 855 }
 856
 857 static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
 858 {
 859         struct nlattr *reload_stats_attr, *act_info, *act_stats;
 860         int i, j, stat_idx;
 861         u32 value;
 862
 863         if (!is_remote)
 864                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
 865         else
 866                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
 867
 868         if (!reload_stats_attr)
 869                 return -EMSGSIZE;
 870
 871         for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
 872                 if ((!is_remote &&
 873                      !devlink_reload_action_is_supported(devlink, i)) ||
 874                     i == DEVLINK_RELOAD_ACTION_UNSPEC)
 875                         continue;
 876                 act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
 877                 if (!act_info)
 878                         goto nla_put_failure;
 879
 880                 if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
 881                         goto action_info_nest_cancel;
 882                 act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
 883                 if (!act_stats)
 884                         goto action_info_nest_cancel;
 885
 886                 for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
 887                         /* Remote stats are shown even if not locally supported.
 888                          * Stats of actions with unspecified limit are shown
 889                          * though drivers don't need to register unspecified
 890                          * limit.
 891                          */
 892                         if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
 893                              !devlink_reload_limit_is_supported(devlink, j)) ||
 894                             devlink_reload_combination_is_invalid(i, j))
 895                                 continue;
 896
 897                         stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
 898                         if (!is_remote)
 899                                 value = devlink->stats.reload_stats[stat_idx];
 900                         else
 901                                 value = devlink->stats.remote_reload_stats[stat_idx];
 902                         if (devlink_reload_stat_put(msg, j, value))
 903                                 goto action_stats_nest_cancel;
 904                 }
 905                 nla_nest_end(msg, act_stats);
 906                 nla_nest_end(msg, act_info);
 907         }
 908         nla_nest_end(msg, reload_stats_attr);
 909         return 0;
 910
 911 action_stats_nest_cancel:
 912         nla_nest_cancel(msg, act_stats);
 913 action_info_nest_cancel:
 914         nla_nest_cancel(msg, act_info);
 915 nla_put_failure:
 916         nla_nest_cancel(msg, reload_stats_attr);
 917         return -EMSGSIZE;
 918 }
 919
 920 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 921                            enum devlink_command cmd, u32 portid,
 922                            u32 seq, int flags)
 923 {
 924         struct nlattr *dev_stats;
 925         void *hdr;
 926
 927         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 928         if (!hdr)
 929                 return -EMSGSIZE;
 930
 931         if (devlink_nl_put_handle(msg, devlink))
 932                 goto nla_put_failure;
 933         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
 934                 goto nla_put_failure;
 935
 936         dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
 937         if (!dev_stats)
 938                 goto nla_put_failure;
 939
 940         if (devlink_reload_stats_put(msg, devlink, false))
 941                 goto dev_stats_nest_cancel;
 942         if (devlink_reload_stats_put(msg, devlink, true))
 943                 goto dev_stats_nest_cancel;
 944
 945         nla_nest_end(msg, dev_stats);
 946         genlmsg_end(msg, hdr);
 947         return 0;
 948
 949 dev_stats_nest_cancel:
 950         nla_nest_cancel(msg, dev_stats);
 951 nla_put_failure:
 952         genlmsg_cancel(msg, hdr);
 953         return -EMSGSIZE;
 954 }
 955
 956 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 957 {
 958         struct sk_buff *msg;
 959         int err;
 960
 961         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 962         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
 963
 964         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 965         if (!msg)
 966                 return;
 967
 968         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 969         if (err) {
 970                 nlmsg_free(msg);
 971                 return;
 972         }
 973
 974         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 975                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 976 }
 977
 978 static int devlink_nl_port_attrs_put(struct sk_buff *msg,
 979                                      struct devlink_port *devlink_port)
 980 {
 981         struct devlink_port_attrs *attrs = &devlink_port->attrs;
 982
 983         if (!devlink_port->attrs_set)
 984                 return 0;
 985         if (attrs->lanes) {
 986                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
 987                         return -EMSGSIZE;
 988         }
 989         if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
 990                 return -EMSGSIZE;
 991         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
 992                 return -EMSGSIZE;
 993         switch (devlink_port->attrs.flavour) {
 994         case DEVLINK_PORT_FLAVOUR_PCI_PF:
 995                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 996                                 attrs->pci_pf.controller) ||
 997                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
 998                         return -EMSGSIZE;
 999                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
1000                         return -EMSGSIZE;
1001                 break;
1002         case DEVLINK_PORT_FLAVOUR_PCI_VF:
1003                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
1004                                 attrs->pci_vf.controller) ||
1005                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
1006                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
1007                         return -EMSGSIZE;
1008                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
1009                         return -EMSGSIZE;
1010                 break;
1011         case DEVLINK_PORT_FLAVOUR_PCI_SF:
1012                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
1013                                 attrs->pci_sf.controller) ||
1014                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
1015                                 attrs->pci_sf.pf) ||
1016                     nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
1017                                 attrs->pci_sf.sf))
1018                         return -EMSGSIZE;
1019                 break;
1020         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
1021         case DEVLINK_PORT_FLAVOUR_CPU:
1022         case DEVLINK_PORT_FLAVOUR_DSA:
1023                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
1024                                 attrs->phys.port_number))
1025                         return -EMSGSIZE;
1026                 if (!attrs->split)
1027                         return 0;
1028                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
1029                                 attrs->phys.port_number))
1030                         return -EMSGSIZE;
1031                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
1032                                 attrs->phys.split_subport_number))
1033                         return -EMSGSIZE;
1034                 break;
1035         default:
1036                 break;
1037         }
1038         return 0;
1039 }
1040
1041 static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
1042                                         struct devlink_port *port,
1043                                         struct sk_buff *msg,
1044                                         struct netlink_ext_ack *extack,
1045                                         bool *msg_updated)
1046 {
1047         u8 hw_addr[MAX_ADDR_LEN];
1048         int hw_addr_len;
1049         int err;
1050
1051         if (!ops->port_function_hw_addr_get)
1052                 return 0;
1053
1054         err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
1055                                              extack);
1056         if (err) {
1057                 if (err == -EOPNOTSUPP)
1058                         return 0;
1059                 return err;
1060         }
1061         err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
1062         if (err)
1063                 return err;
1064         *msg_updated = true;
1065         return 0;
1066 }
1067
1068 static int devlink_nl_rate_fill(struct sk_buff *msg,
1069                                 struct devlink_rate *devlink_rate,
1070                                 enum devlink_command cmd, u32 portid, u32 seq,
1071                                 int flags, struct netlink_ext_ack *extack)
1072 {
1073         struct devlink *devlink = devlink_rate->devlink;
1074         void *hdr;
1075
1076         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1077         if (!hdr)
1078                 return -EMSGSIZE;
1079
1080         if (devlink_nl_put_handle(msg, devlink))
1081                 goto nla_put_failure;
1082
1083         if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
1084                 goto nla_put_failure;
1085
1086         if (devlink_rate_is_leaf(devlink_rate)) {
1087                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
1088                                 devlink_rate->devlink_port->index))
1089                         goto nla_put_failure;
1090         } else if (devlink_rate_is_node(devlink_rate)) {
1091                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
1092                                    devlink_rate->name))
1093                         goto nla_put_failure;
1094         }
1095
1096         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
1097                               devlink_rate->tx_share, DEVLINK_ATTR_PAD))
1098                 goto nla_put_failure;
1099
1100         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
1101                               devlink_rate->tx_max, DEVLINK_ATTR_PAD))
1102                 goto nla_put_failure;
1103
1104         if (devlink_rate->parent)
1105                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
1106                                    devlink_rate->parent->name))
1107                         goto nla_put_failure;
1108
1109         genlmsg_end(msg, hdr);
1110         return 0;
1111
1112 nla_put_failure:
1113         genlmsg_cancel(msg, hdr);
1114         return -EMSGSIZE;
1115 }
1116
1117 static bool
1118 devlink_port_fn_state_valid(enum devlink_port_fn_state state)
1119 {
1120         return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
1121                state == DEVLINK_PORT_FN_STATE_ACTIVE;
1122 }
1123
1124 static bool
1125 devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
1126 {
1127         return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
1128                opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
1129 }
1130
1131 static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
1132                                       struct devlink_port *port,
1133                                       struct sk_buff *msg,
1134                                       struct netlink_ext_ack *extack,
1135                                       bool *msg_updated)
1136 {
1137         enum devlink_port_fn_opstate opstate;
1138         enum devlink_port_fn_state state;
1139         int err;
1140
1141         if (!ops->port_fn_state_get)
1142                 return 0;
1143
1144         err = ops->port_fn_state_get(port, &state, &opstate, extack);
1145         if (err) {
1146                 if (err == -EOPNOTSUPP)
1147                         return 0;
1148                 return err;
1149         }
1150         if (!devlink_port_fn_state_valid(state)) {
1151                 WARN_ON_ONCE(1);
1152                 NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
1153                 return -EINVAL;
1154         }
1155         if (!devlink_port_fn_opstate_valid(opstate)) {
1156                 WARN_ON_ONCE(1);
1157                 NL_SET_ERR_MSG_MOD(extack,
1158                                    "Invalid operational state read from driver");
1159                 return -EINVAL;
1160         }
1161         if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
1162             nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
1163                 return -EMSGSIZE;
1164         *msg_updated = true;
1165         return 0;
1166 }
1167
1168 static int
1169 devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
1170                                    struct netlink_ext_ack *extack)
1171 {
1172         const struct devlink_ops *ops;
1173         struct nlattr *function_attr;
1174         bool msg_updated = false;
1175         int err;
1176
1177         function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
1178         if (!function_attr)
1179                 return -EMSGSIZE;
1180
1181         ops = port->devlink->ops;
1182         err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
1183                                            &msg_updated);
1184         if (err)
1185                 goto out;
1186         err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
1187 out:
1188         if (err || !msg_updated)
1189                 nla_nest_cancel(msg, function_attr);
1190         else
1191                 nla_nest_end(msg, function_attr);
1192         return err;
1193 }
1194
1195 static int devlink_nl_port_fill(struct sk_buff *msg,
1196                                 struct devlink_port *devlink_port,
1197                                 enum devlink_command cmd, u32 portid, u32 seq,
1198                                 int flags, struct netlink_ext_ack *extack)
1199 {
1200         struct devlink *devlink = devlink_port->devlink;
1201         void *hdr;
1202
1203         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1204         if (!hdr)
1205                 return -EMSGSIZE;
1206
1207         if (devlink_nl_put_handle(msg, devlink))
1208                 goto nla_put_failure;
1209         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1210                 goto nla_put_failure;
1211
1212         /* Hold rtnl lock while accessing port's netdev attributes. */
1213         rtnl_lock();
1214         spin_lock_bh(&devlink_port->type_lock);
1215         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
1216                 goto nla_put_failure_type_locked;
1217         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
1218             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
1219                         devlink_port->desired_type))
1220                 goto nla_put_failure_type_locked;
1221         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
1222                 struct net *net = devlink_net(devlink_port->devlink);
1223                 struct net_device *netdev = devlink_port->type_dev;
1224
1225                 if (netdev && net_eq(net, dev_net(netdev)) &&
1226                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
1227                                  netdev->ifindex) ||
1228                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
1229                                     netdev->name)))
1230                         goto nla_put_failure_type_locked;
1231         }
1232         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
1233                 struct ib_device *ibdev = devlink_port->type_dev;
1234
1235                 if (ibdev &&
1236                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
1237                                    ibdev->name))
1238                         goto nla_put_failure_type_locked;
1239         }
1240         spin_unlock_bh(&devlink_port->type_lock);
1241         rtnl_unlock();
1242         if (devlink_nl_port_attrs_put(msg, devlink_port))
1243                 goto nla_put_failure;
1244         if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
1245                 goto nla_put_failure;
1246         if (devlink_port->linecard &&
1247             nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX,
1248                         devlink_port->linecard->index))
1249                 goto nla_put_failure;
1250
1251         genlmsg_end(msg, hdr);
1252         return 0;
1253
1254 nla_put_failure_type_locked:
1255         spin_unlock_bh(&devlink_port->type_lock);
1256         rtnl_unlock();
1257 nla_put_failure:
1258         genlmsg_cancel(msg, hdr);
1259         return -EMSGSIZE;
1260 }
1261
1262 static void devlink_port_notify(struct devlink_port *devlink_port,
1263                                 enum devlink_command cmd)
1264 {
1265         struct devlink *devlink = devlink_port->devlink;
1266         struct sk_buff *msg;
1267         int err;
1268
1269         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
1270
1271         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1272                 return;
1273
1274         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1275         if (!msg)
1276                 return;
1277
1278         err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
1279         if (err) {
1280                 nlmsg_free(msg);
1281                 return;
1282         }
1283
1284         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1285                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1286 }
1287
1288 static void devlink_rate_notify(struct devlink_rate *devlink_rate,
1289                                 enum devlink_command cmd)
1290 {
1291         struct devlink *devlink = devlink_rate->devlink;
1292         struct sk_buff *msg;
1293         int err;
1294
1295         WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
1296
1297         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1298                 return;
1299
1300         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1301         if (!msg)
1302                 return;
1303
1304         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
1305         if (err) {
1306                 nlmsg_free(msg);
1307                 return;
1308         }
1309
1310         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1311                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1312 }
1313
1314 static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
1315                                           struct netlink_callback *cb)
1316 {
1317         struct devlink_rate *devlink_rate;
1318         struct devlink *devlink;
1319         int start = cb->args[0];
1320         unsigned long index;
1321         int idx = 0;
1322         int err = 0;
1323
1324         mutex_lock(&devlink_mutex);
1325         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1326                 if (!devlink_try_get(devlink))
1327                         continue;
1328
1329                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1330                         goto retry;
1331
1332                 devl_lock(devlink);
1333                 list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
1334                         enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
1335                         u32 id = NETLINK_CB(cb->skb).portid;
1336
1337                         if (idx < start) {
1338                                 idx++;
1339                                 continue;
1340                         }
1341                         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
1342                                                    cb->nlh->nlmsg_seq,
1343                                                    NLM_F_MULTI, NULL);
1344                         if (err) {
1345                                 devl_unlock(devlink);
1346                                 devlink_put(devlink);
1347                                 goto out;
1348                         }
1349                         idx++;
1350                 }
1351                 devl_unlock(devlink);
1352 retry:
1353                 devlink_put(devlink);
1354         }
1355 out:
1356         mutex_unlock(&devlink_mutex);
1357         if (err != -EMSGSIZE)
1358                 return err;
1359
1360         cb->args[0] = idx;
1361         return msg->len;
1362 }
1363
1364 static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
1365                                         struct genl_info *info)
1366 {
1367         struct devlink_rate *devlink_rate = info->user_ptr[1];
1368         struct sk_buff *msg;
1369         int err;
1370
1371         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1372         if (!msg)
1373                 return -ENOMEM;
1374
1375         err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
1376                                    info->snd_portid, info->snd_seq, 0,
1377                                    info->extack);
1378         if (err) {
1379                 nlmsg_free(msg);
1380                 return err;
1381         }
1382
1383         return genlmsg_reply(msg, info);
1384 }
1385
1386 static bool
1387 devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
1388                             struct devlink_rate *parent)
1389 {
1390         while (parent) {
1391                 if (parent == devlink_rate)
1392                         return true;
1393                 parent = parent->parent;
1394         }
1395         return false;
1396 }
1397
1398 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
1399 {
1400         struct devlink *devlink = info->user_ptr[0];
1401         struct sk_buff *msg;
1402         int err;
1403
1404         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1405         if (!msg)
1406                 return -ENOMEM;
1407
1408         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1409                               info->snd_portid, info->snd_seq, 0);
1410         if (err) {
1411                 nlmsg_free(msg);
1412                 return err;
1413         }
1414
1415         return genlmsg_reply(msg, info);
1416 }
1417
1418 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
1419                                      struct netlink_callback *cb)
1420 {
1421         struct devlink *devlink;
1422         int start = cb->args[0];
1423         unsigned long index;
1424         int idx = 0;
1425         int err;
1426
1427         mutex_lock(&devlink_mutex);
1428         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1429                 if (!devlink_try_get(devlink))
1430                         continue;
1431
1432                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk))) {
1433                         devlink_put(devlink);
1434                         continue;
1435                 }
1436
1437                 if (idx < start) {
1438                         idx++;
1439                         devlink_put(devlink);
1440                         continue;
1441                 }
1442
1443                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1444                                       NETLINK_CB(cb->skb).portid,
1445                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
1446                 devlink_put(devlink);
1447                 if (err)
1448                         goto out;
1449                 idx++;
1450         }
1451 out:
1452         mutex_unlock(&devlink_mutex);
1453
1454         cb->args[0] = idx;
1455         return msg->len;
1456 }
1457
1458 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
1459                                         struct genl_info *info)
1460 {
1461         struct devlink_port *devlink_port = info->user_ptr[1];
1462         struct sk_buff *msg;
1463         int err;
1464
1465         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1466         if (!msg)
1467                 return -ENOMEM;
1468
1469         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
1470                                    info->snd_portid, info->snd_seq, 0,
1471                                    info->extack);
1472         if (err) {
1473                 nlmsg_free(msg);
1474                 return err;
1475         }
1476
1477         return genlmsg_reply(msg, info);
1478 }
1479
1480 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
1481                                           struct netlink_callback *cb)
1482 {
1483         struct devlink *devlink;
1484         struct devlink_port *devlink_port;
1485         int start = cb->args[0];
1486         unsigned long index;
1487         int idx = 0;
1488         int err;
1489
1490         mutex_lock(&devlink_mutex);
1491         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1492                 if (!devlink_try_get(devlink))
1493                         continue;
1494
1495                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1496                         goto retry;
1497
1498                 devl_lock(devlink);
1499                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
1500                         if (idx < start) {
1501                                 idx++;
1502                                 continue;
1503                         }
1504                         err = devlink_nl_port_fill(msg, devlink_port,
1505                                                    DEVLINK_CMD_NEW,
1506                                                    NETLINK_CB(cb->skb).portid,
1507                                                    cb->nlh->nlmsg_seq,
1508                                                    NLM_F_MULTI, cb->extack);
1509                         if (err) {
1510                                 devl_unlock(devlink);
1511                                 devlink_put(devlink);
1512                                 goto out;
1513                         }
1514                         idx++;
1515                 }
1516                 devl_unlock(devlink);
1517 retry:
1518                 devlink_put(devlink);
1519         }
1520 out:
1521         mutex_unlock(&devlink_mutex);
1522
1523         cb->args[0] = idx;
1524         return msg->len;
1525 }
1526
1527 static int devlink_port_type_set(struct devlink_port *devlink_port,
1528                                  enum devlink_port_type port_type)
1529
1530 {
1531         int err;
1532
1533         if (!devlink_port->devlink->ops->port_type_set)
1534                 return -EOPNOTSUPP;
1535
1536         if (port_type == devlink_port->type)
1537                 return 0;
1538
1539         err = devlink_port->devlink->ops->port_type_set(devlink_port,
1540                                                         port_type);
1541         if (err)
1542                 return err;
1543
1544         devlink_port->desired_type = port_type;
1545         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
1546         return 0;
1547 }
1548
1549 static int devlink_port_function_hw_addr_set(struct devlink_port *port,
1550                                              const struct nlattr *attr,
1551                                              struct netlink_ext_ack *extack)
1552 {
1553         const struct devlink_ops *ops = port->devlink->ops;
1554         const u8 *hw_addr;
1555         int hw_addr_len;
1556
1557         hw_addr = nla_data(attr);
1558         hw_addr_len = nla_len(attr);
1559         if (hw_addr_len > MAX_ADDR_LEN) {
1560                 NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
1561                 return -EINVAL;
1562         }
1563         if (port->type == DEVLINK_PORT_TYPE_ETH) {
1564                 if (hw_addr_len != ETH_ALEN) {
1565                         NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
1566                         return -EINVAL;
1567                 }
1568                 if (!is_unicast_ether_addr(hw_addr)) {
1569                         NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
1570                         return -EINVAL;
1571                 }
1572         }
1573
1574         if (!ops->port_function_hw_addr_set) {
1575                 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
1576                 return -EOPNOTSUPP;
1577         }
1578
1579         return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
1580                                               extack);
1581 }
1582
1583 static int devlink_port_fn_state_set(struct devlink_port *port,
1584                                      const struct nlattr *attr,
1585                                      struct netlink_ext_ack *extack)
1586 {
1587         enum devlink_port_fn_state state;
1588         const struct devlink_ops *ops;
1589
1590         state = nla_get_u8(attr);
1591         ops = port->devlink->ops;
1592         if (!ops->port_fn_state_set) {
1593                 NL_SET_ERR_MSG_MOD(extack,
1594                                    "Function does not support state setting");
1595                 return -EOPNOTSUPP;
1596         }
1597         return ops->port_fn_state_set(port, state, extack);
1598 }
1599
1600 static int devlink_port_function_set(struct devlink_port *port,
1601                                      const struct nlattr *attr,
1602                                      struct netlink_ext_ack *extack)
1603 {
1604         struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
1605         int err;
1606
1607         err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
1608                                devlink_function_nl_policy, extack);
1609         if (err < 0) {
1610                 NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
1611                 return err;
1612         }
1613
1614         attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
1615         if (attr) {
1616                 err = devlink_port_function_hw_addr_set(port, attr, extack);
1617                 if (err)
1618                         return err;
1619         }
1620         /* Keep this as the last function attribute set, so that when
1621          * multiple port function attributes are set along with state,
1622          * Those can be applied first before activating the state.
1623          */
1624         attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
1625         if (attr)
1626                 err = devlink_port_fn_state_set(port, attr, extack);
1627
1628         if (!err)
1629                 devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
1630         return err;
1631 }
1632
1633 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
1634                                         struct genl_info *info)
1635 {
1636         struct devlink_port *devlink_port = info->user_ptr[1];
1637         int err;
1638
1639         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
1640                 enum devlink_port_type port_type;
1641
1642                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
1643                 err = devlink_port_type_set(devlink_port, port_type);
1644                 if (err)
1645                         return err;
1646         }
1647
1648         if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
1649                 struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
1650                 struct netlink_ext_ack *extack = info->extack;
1651
1652                 err = devlink_port_function_set(devlink_port, attr, extack);
1653                 if (err)
1654                         return err;
1655         }
1656
1657         return 0;
1658 }
1659
1660 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
1661                                           struct genl_info *info)
1662 {
1663         struct devlink_port *devlink_port = info->user_ptr[1];
1664         struct devlink *devlink = info->user_ptr[0];
1665         u32 count;
1666
1667         if (!info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
1668                 return -EINVAL;
1669         if (!devlink->ops->port_split)
1670                 return -EOPNOTSUPP;
1671
1672         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
1673
1674         if (!devlink_port->attrs.splittable) {
1675                 /* Split ports cannot be split. */
1676                 if (devlink_port->attrs.split)
1677                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
1678                 else
1679                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
1680                 return -EINVAL;
1681         }
1682
1683         if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
1684                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
1685                 return -EINVAL;
1686         }
1687
1688         return devlink->ops->port_split(devlink, devlink_port, count,
1689                                         info->extack);
1690 }
1691
1692 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
1693                                             struct genl_info *info)
1694 {
1695         struct devlink_port *devlink_port = info->user_ptr[1];
1696         struct devlink *devlink = info->user_ptr[0];
1697
1698         if (!devlink->ops->port_unsplit)
1699                 return -EOPNOTSUPP;
1700         return devlink->ops->port_unsplit(devlink, devlink_port, info->extack);
1701 }
1702
1703 static int devlink_port_new_notifiy(struct devlink *devlink,
1704                                     unsigned int port_index,
1705                                     struct genl_info *info)
1706 {
1707         struct devlink_port *devlink_port;
1708         struct sk_buff *msg;
1709         int err;
1710
1711         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1712         if (!msg)
1713                 return -ENOMEM;
1714
1715         lockdep_assert_held(&devlink->lock);
1716         devlink_port = devlink_port_get_by_index(devlink, port_index);
1717         if (!devlink_port) {
1718                 err = -ENODEV;
1719                 goto out;
1720         }
1721
1722         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
1723                                    info->snd_portid, info->snd_seq, 0, NULL);
1724         if (err)
1725                 goto out;
1726
1727         err = genlmsg_reply(msg, info);
1728         return err;
1729
1730 out:
1731         nlmsg_free(msg);
1732         return err;
1733 }
1734
1735 static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
1736                                         struct genl_info *info)
1737 {
1738         struct netlink_ext_ack *extack = info->extack;
1739         struct devlink_port_new_attrs new_attrs = {};
1740         struct devlink *devlink = info->user_ptr[0];
1741         unsigned int new_port_index;
1742         int err;
1743
1744         if (!devlink->ops->port_new || !devlink->ops->port_del)
1745                 return -EOPNOTSUPP;
1746
1747         if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
1748             !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
1749                 NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
1750                 return -EINVAL;
1751         }
1752         new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
1753         new_attrs.pfnum =
1754                 nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
1755
1756         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1757                 /* Port index of the new port being created by driver. */
1758                 new_attrs.port_index =
1759                         nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1760                 new_attrs.port_index_valid = true;
1761         }
1762         if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
1763                 new_attrs.controller =
1764                         nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
1765                 new_attrs.controller_valid = true;
1766         }
1767         if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
1768             info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
1769                 new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
1770                 new_attrs.sfnum_valid = true;
1771         }
1772
1773         err = devlink->ops->port_new(devlink, &new_attrs, extack,
1774                                      &new_port_index);
1775         if (err)
1776                 return err;
1777
1778         err = devlink_port_new_notifiy(devlink, new_port_index, info);
1779         if (err && err != -ENODEV) {
1780                 /* Fail to send the response; destroy newly created port. */
1781                 devlink->ops->port_del(devlink, new_port_index, extack);
1782         }
1783         return err;
1784 }
1785
1786 static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
1787                                         struct genl_info *info)
1788 {
1789         struct netlink_ext_ack *extack = info->extack;
1790         struct devlink *devlink = info->user_ptr[0];
1791         unsigned int port_index;
1792
1793         if (!devlink->ops->port_del)
1794                 return -EOPNOTSUPP;
1795
1796         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1797                 NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
1798                 return -EINVAL;
1799         }
1800         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1801
1802         return devlink->ops->port_del(devlink, port_index, extack);
1803 }
1804
1805 static int
1806 devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
1807                                 struct genl_info *info,
1808                                 struct nlattr *nla_parent)
1809 {
1810         struct devlink *devlink = devlink_rate->devlink;
1811         const char *parent_name = nla_data(nla_parent);
1812         const struct devlink_ops *ops = devlink->ops;
1813         size_t len = strlen(parent_name);
1814         struct devlink_rate *parent;
1815         int err = -EOPNOTSUPP;
1816
1817         parent = devlink_rate->parent;
1818         if (parent && len) {
1819                 NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
1820                 return -EBUSY;
1821         } else if (parent && !len) {
1822                 if (devlink_rate_is_leaf(devlink_rate))
1823                         err = ops->rate_leaf_parent_set(devlink_rate, NULL,
1824                                                         devlink_rate->priv, NULL,
1825                                                         info->extack);
1826                 else if (devlink_rate_is_node(devlink_rate))
1827                         err = ops->rate_node_parent_set(devlink_rate, NULL,
1828                                                         devlink_rate->priv, NULL,
1829                                                         info->extack);
1830                 if (err)
1831                         return err;
1832
1833                 refcount_dec(&parent->refcnt);
1834                 devlink_rate->parent = NULL;
1835         } else if (!parent && len) {
1836                 parent = devlink_rate_node_get_by_name(devlink, parent_name);
1837                 if (IS_ERR(parent))
1838                         return -ENODEV;
1839
1840                 if (parent == devlink_rate) {
1841                         NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
1842                         return -EINVAL;
1843                 }
1844
1845                 if (devlink_rate_is_node(devlink_rate) &&
1846                     devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
1847                         NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
1848                         return -EEXIST;
1849                 }
1850
1851                 if (devlink_rate_is_leaf(devlink_rate))
1852                         err = ops->rate_leaf_parent_set(devlink_rate, parent,
1853                                                         devlink_rate->priv, parent->priv,
1854                                                         info->extack);
1855                 else if (devlink_rate_is_node(devlink_rate))
1856                         err = ops->rate_node_parent_set(devlink_rate, parent,
1857                                                         devlink_rate->priv, parent->priv,
1858                                                         info->extack);
1859                 if (err)
1860                         return err;
1861
1862                 refcount_inc(&parent->refcnt);
1863                 devlink_rate->parent = parent;
1864         }
1865
1866         return 0;
1867 }
1868
1869 static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
1870                                const struct devlink_ops *ops,
1871                                struct genl_info *info)
1872 {
1873         struct nlattr *nla_parent, **attrs = info->attrs;
1874         int err = -EOPNOTSUPP;
1875         u64 rate;
1876
1877         if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
1878                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
1879                 if (devlink_rate_is_leaf(devlink_rate))
1880                         err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
1881                                                           rate, info->extack);
1882                 else if (devlink_rate_is_node(devlink_rate))
1883                         err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
1884                                                           rate, info->extack);
1885                 if (err)
1886                         return err;
1887                 devlink_rate->tx_share = rate;
1888         }
1889
1890         if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
1891                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
1892                 if (devlink_rate_is_leaf(devlink_rate))
1893                         err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
1894                                                         rate, info->extack);
1895                 else if (devlink_rate_is_node(devlink_rate))
1896                         err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
1897                                                         rate, info->extack);
1898                 if (err)
1899                         return err;
1900                 devlink_rate->tx_max = rate;
1901         }
1902
1903         nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
1904         if (nla_parent) {
1905                 err = devlink_nl_rate_parent_node_set(devlink_rate, info,
1906                                                       nla_parent);
1907                 if (err)
1908                         return err;
1909         }
1910
1911         return 0;
1912 }
1913
1914 static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
1915                                            struct genl_info *info,
1916                                            enum devlink_rate_type type)
1917 {
1918         struct nlattr **attrs = info->attrs;
1919
1920         if (type == DEVLINK_RATE_TYPE_LEAF) {
1921                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
1922                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
1923                         return false;
1924                 }
1925                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
1926                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
1927                         return false;
1928                 }
1929                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1930                     !ops->rate_leaf_parent_set) {
1931                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
1932                         return false;
1933                 }
1934         } else if (type == DEVLINK_RATE_TYPE_NODE) {
1935                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
1936                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
1937                         return false;
1938                 }
1939                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
1940                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
1941                         return false;
1942                 }
1943                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1944                     !ops->rate_node_parent_set) {
1945                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
1946                         return false;
1947                 }
1948         } else {
1949                 WARN(1, "Unknown type of rate object");
1950                 return false;
1951         }
1952
1953         return true;
1954 }
1955
1956 static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
1957                                         struct genl_info *info)
1958 {
1959         struct devlink_rate *devlink_rate = info->user_ptr[1];
1960         struct devlink *devlink = devlink_rate->devlink;
1961         const struct devlink_ops *ops = devlink->ops;
1962         int err;
1963
1964         if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
1965                 return -EOPNOTSUPP;
1966
1967         err = devlink_nl_rate_set(devlink_rate, ops, info);
1968
1969         if (!err)
1970                 devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
1971         return err;
1972 }
1973
1974 static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
1975                                         struct genl_info *info)
1976 {
1977         struct devlink *devlink = info->user_ptr[0];
1978         struct devlink_rate *rate_node;
1979         const struct devlink_ops *ops;
1980         int err;
1981
1982         ops = devlink->ops;
1983         if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
1984                 NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
1985                 return -EOPNOTSUPP;
1986         }
1987
1988         if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
1989                 return -EOPNOTSUPP;
1990
1991         rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
1992         if (!IS_ERR(rate_node))
1993                 return -EEXIST;
1994         else if (rate_node == ERR_PTR(-EINVAL))
1995                 return -EINVAL;
1996
1997         rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
1998         if (!rate_node)
1999                 return -ENOMEM;
2000
2001         rate_node->devlink = devlink;
2002         rate_node->type = DEVLINK_RATE_TYPE_NODE;
2003         rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
2004         if (!rate_node->name) {
2005                 err = -ENOMEM;
2006                 goto err_strdup;
2007         }
2008
2009         err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
2010         if (err)
2011                 goto err_node_new;
2012
2013         err = devlink_nl_rate_set(rate_node, ops, info);
2014         if (err)
2015                 goto err_rate_set;
2016
2017         refcount_set(&rate_node->refcnt, 1);
2018         list_add(&rate_node->list, &devlink->rate_list);
2019         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
2020         return 0;
2021
2022 err_rate_set:
2023         ops->rate_node_del(rate_node, rate_node->priv, info->extack);
2024 err_node_new:
2025         kfree(rate_node->name);
2026 err_strdup:
2027         kfree(rate_node);
2028         return err;
2029 }
2030
2031 static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
2032                                         struct genl_info *info)
2033 {
2034         struct devlink_rate *rate_node = info->user_ptr[1];
2035         struct devlink *devlink = rate_node->devlink;
2036         const struct devlink_ops *ops = devlink->ops;
2037         int err;
2038
2039         if (refcount_read(&rate_node->refcnt) > 1) {
2040                 NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
2041                 return -EBUSY;
2042         }
2043
2044         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
2045         err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
2046         if (rate_node->parent)
2047                 refcount_dec(&rate_node->parent->refcnt);
2048         list_del(&rate_node->list);
2049         kfree(rate_node->name);
2050         kfree(rate_node);
2051         return err;
2052 }
2053
2054 struct devlink_linecard_type {
2055         const char *type;
2056         const void *priv;
2057 };
2058
2059 static int devlink_nl_linecard_fill(struct sk_buff *msg,
2060                                     struct devlink *devlink,
2061                                     struct devlink_linecard *linecard,
2062                                     enum devlink_command cmd, u32 portid,
2063                                     u32 seq, int flags,
2064                                     struct netlink_ext_ack *extack)
2065 {
2066         struct devlink_linecard_type *linecard_type;
2067         struct nlattr *attr;
2068         void *hdr;
2069         int i;
2070
2071         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2072         if (!hdr)
2073                 return -EMSGSIZE;
2074
2075         if (devlink_nl_put_handle(msg, devlink))
2076                 goto nla_put_failure;
2077         if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index))
2078                 goto nla_put_failure;
2079         if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state))
2080                 goto nla_put_failure;
2081         if (linecard->type &&
2082             nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, linecard->type))
2083                 goto nla_put_failure;
2084
2085         if (linecard->types_count) {
2086                 attr = nla_nest_start(msg,
2087                                       DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES);
2088                 if (!attr)
2089                         goto nla_put_failure;
2090                 for (i = 0; i < linecard->types_count; i++) {
2091                         linecard_type = &linecard->types[i];
2092                         if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE,
2093                                            linecard_type->type)) {
2094                                 nla_nest_cancel(msg, attr);
2095                                 goto nla_put_failure;
2096                         }
2097                 }
2098                 nla_nest_end(msg, attr);
2099         }
2100
2101         genlmsg_end(msg, hdr);
2102         return 0;
2103
2104 nla_put_failure:
2105         genlmsg_cancel(msg, hdr);
2106         return -EMSGSIZE;
2107 }
2108
2109 static void devlink_linecard_notify(struct devlink_linecard *linecard,
2110                                     enum devlink_command cmd)
2111 {
2112         struct devlink *devlink = linecard->devlink;
2113         struct sk_buff *msg;
2114         int err;
2115
2116         WARN_ON(cmd != DEVLINK_CMD_LINECARD_NEW &&
2117                 cmd != DEVLINK_CMD_LINECARD_DEL);
2118
2119         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
2120                 return;
2121
2122         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2123         if (!msg)
2124                 return;
2125
2126         err = devlink_nl_linecard_fill(msg, devlink, linecard, cmd, 0, 0, 0,
2127                                        NULL);
2128         if (err) {
2129                 nlmsg_free(msg);
2130                 return;
2131         }
2132
2133         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
2134                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
2135 }
2136
2137 static int devlink_nl_cmd_linecard_get_doit(struct sk_buff *skb,
2138                                             struct genl_info *info)
2139 {
2140         struct devlink_linecard *linecard = info->user_ptr[1];
2141         struct devlink *devlink = linecard->devlink;
2142         struct sk_buff *msg;
2143         int err;
2144
2145         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2146         if (!msg)
2147                 return -ENOMEM;
2148
2149         mutex_lock(&linecard->state_lock);
2150         err = devlink_nl_linecard_fill(msg, devlink, linecard,
2151                                        DEVLINK_CMD_LINECARD_NEW,
2152                                        info->snd_portid, info->snd_seq, 0,
2153                                        info->extack);
2154         mutex_unlock(&linecard->state_lock);
2155         if (err) {
2156                 nlmsg_free(msg);
2157                 return err;
2158         }
2159
2160         return genlmsg_reply(msg, info);
2161 }
2162
2163 static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg,
2164                                               struct netlink_callback *cb)
2165 {
2166         struct devlink_linecard *linecard;
2167         struct devlink *devlink;
2168         int start = cb->args[0];
2169         unsigned long index;
2170         int idx = 0;
2171         int err;
2172
2173         mutex_lock(&devlink_mutex);
2174         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2175                 if (!devlink_try_get(devlink))
2176                         continue;
2177
2178                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2179                         goto retry;
2180
2181                 mutex_lock(&devlink->linecards_lock);
2182                 list_for_each_entry(linecard, &devlink->linecard_list, list) {
2183                         if (idx < start) {
2184                                 idx++;
2185                                 continue;
2186                         }
2187                         mutex_lock(&linecard->state_lock);
2188                         err = devlink_nl_linecard_fill(msg, devlink, linecard,
2189                                                        DEVLINK_CMD_LINECARD_NEW,
2190                                                        NETLINK_CB(cb->skb).portid,
2191                                                        cb->nlh->nlmsg_seq,
2192                                                        NLM_F_MULTI,
2193                                                        cb->extack);
2194                         mutex_unlock(&linecard->state_lock);
2195                         if (err) {
2196                                 mutex_unlock(&devlink->linecards_lock);
2197                                 devlink_put(devlink);
2198                                 goto out;
2199                         }
2200                         idx++;
2201                 }
2202                 mutex_unlock(&devlink->linecards_lock);
2203 retry:
2204                 devlink_put(devlink);
2205         }
2206 out:
2207         mutex_unlock(&devlink_mutex);
2208
2209         cb->args[0] = idx;
2210         return msg->len;
2211 }
2212
2213 static struct devlink_linecard_type *
2214 devlink_linecard_type_lookup(struct devlink_linecard *linecard,
2215                              const char *type)
2216 {
2217         struct devlink_linecard_type *linecard_type;
2218         int i;
2219
2220         for (i = 0; i < linecard->types_count; i++) {
2221                 linecard_type = &linecard->types[i];
2222                 if (!strcmp(type, linecard_type->type))
2223                         return linecard_type;
2224         }
2225         return NULL;
2226 }
2227
2228 static int devlink_linecard_type_set(struct devlink_linecard *linecard,
2229                                      const char *type,
2230                                      struct netlink_ext_ack *extack)
2231 {
2232         const struct devlink_linecard_ops *ops = linecard->ops;
2233         struct devlink_linecard_type *linecard_type;
2234         int err;
2235
2236         mutex_lock(&linecard->state_lock);
2237         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
2238                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
2239                 err = -EBUSY;
2240                 goto out;
2241         }
2242         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
2243                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
2244                 err = -EBUSY;
2245                 goto out;
2246         }
2247
2248         linecard_type = devlink_linecard_type_lookup(linecard, type);
2249         if (!linecard_type) {
2250                 NL_SET_ERR_MSG_MOD(extack, "Unsupported line card type provided");
2251                 err = -EINVAL;
2252                 goto out;
2253         }
2254
2255         if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED &&
2256             linecard->state != DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
2257                 NL_SET_ERR_MSG_MOD(extack, "Line card already provisioned");
2258                 err = -EBUSY;
2259                 /* Check if the line card is provisioned in the same
2260                  * way the user asks. In case it is, make the operation
2261                  * to return success.
2262                  */
2263                 if (ops->same_provision &&
2264                     ops->same_provision(linecard, linecard->priv,
2265                                         linecard_type->type,
2266                                         linecard_type->priv))
2267                         err = 0;
2268                 goto out;
2269         }
2270
2271         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING;
2272         linecard->type = linecard_type->type;
2273         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2274         mutex_unlock(&linecard->state_lock);
2275         err = ops->provision(linecard, linecard->priv, linecard_type->type,
2276                              linecard_type->priv, extack);
2277         if (err) {
2278                 /* Provisioning failed. Assume the linecard is unprovisioned
2279                  * for future operations.
2280                  */
2281                 mutex_lock(&linecard->state_lock);
2282                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2283                 linecard->type = NULL;
2284                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2285                 mutex_unlock(&linecard->state_lock);
2286         }
2287         return err;
2288
2289 out:
2290         mutex_unlock(&linecard->state_lock);
2291         return err;
2292 }
2293
2294 static int devlink_linecard_type_unset(struct devlink_linecard *linecard,
2295                                        struct netlink_ext_ack *extack)
2296 {
2297         int err;
2298
2299         mutex_lock(&linecard->state_lock);
2300         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
2301                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
2302                 err = -EBUSY;
2303                 goto out;
2304         }
2305         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
2306                 NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
2307                 err = -EBUSY;
2308                 goto out;
2309         }
2310         if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
2311                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2312                 linecard->type = NULL;
2313                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2314                 err = 0;
2315                 goto out;
2316         }
2317
2318         if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) {
2319                 NL_SET_ERR_MSG_MOD(extack, "Line card is not provisioned");
2320                 err = 0;
2321                 goto out;
2322         }
2323         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING;
2324         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2325         mutex_unlock(&linecard->state_lock);
2326         err = linecard->ops->unprovision(linecard, linecard->priv,
2327                                          extack);
2328         if (err) {
2329                 /* Unprovisioning failed. Assume the linecard is unprovisioned
2330                  * for future operations.
2331                  */
2332                 mutex_lock(&linecard->state_lock);
2333                 linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
2334                 linecard->type = NULL;
2335                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
2336                 mutex_unlock(&linecard->state_lock);
2337         }
2338         return err;
2339
2340 out:
2341         mutex_unlock(&linecard->state_lock);
2342         return err;
2343 }
2344
2345 static int devlink_nl_cmd_linecard_set_doit(struct sk_buff *skb,
2346                                             struct genl_info *info)
2347 {
2348         struct devlink_linecard *linecard = info->user_ptr[1];
2349         struct netlink_ext_ack *extack = info->extack;
2350         int err;
2351
2352         if (info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) {
2353                 const char *type;
2354
2355                 type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]);
2356                 if (strcmp(type, "")) {
2357                         err = devlink_linecard_type_set(linecard, type, extack);
2358                         if (err)
2359                                 return err;
2360                 } else {
2361                         err = devlink_linecard_type_unset(linecard, extack);
2362                         if (err)
2363                                 return err;
2364                 }
2365         }
2366
2367         return 0;
2368 }
2369
2370 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
2371                               struct devlink_sb *devlink_sb,
2372                               enum devlink_command cmd, u32 portid,
2373                               u32 seq, int flags)
2374 {
2375         void *hdr;
2376
2377         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2378         if (!hdr)
2379                 return -EMSGSIZE;
2380
2381         if (devlink_nl_put_handle(msg, devlink))
2382                 goto nla_put_failure;
2383         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2384                 goto nla_put_failure;
2385         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
2386                 goto nla_put_failure;
2387         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
2388                         devlink_sb->ingress_pools_count))
2389                 goto nla_put_failure;
2390         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
2391                         devlink_sb->egress_pools_count))
2392                 goto nla_put_failure;
2393         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
2394                         devlink_sb->ingress_tc_count))
2395                 goto nla_put_failure;
2396         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
2397                         devlink_sb->egress_tc_count))
2398                 goto nla_put_failure;
2399
2400         genlmsg_end(msg, hdr);
2401         return 0;
2402
2403 nla_put_failure:
2404         genlmsg_cancel(msg, hdr);
2405         return -EMSGSIZE;
2406 }
2407
2408 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
2409                                       struct genl_info *info)
2410 {
2411         struct devlink *devlink = info->user_ptr[0];
2412         struct devlink_sb *devlink_sb;
2413         struct sk_buff *msg;
2414         int err;
2415
2416         devlink_sb = devlink_sb_get_from_info(devlink, info);
2417         if (IS_ERR(devlink_sb))
2418                 return PTR_ERR(devlink_sb);
2419
2420         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2421         if (!msg)
2422                 return -ENOMEM;
2423
2424         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2425                                  DEVLINK_CMD_SB_NEW,
2426                                  info->snd_portid, info->snd_seq, 0);
2427         if (err) {
2428                 nlmsg_free(msg);
2429                 return err;
2430         }
2431
2432         return genlmsg_reply(msg, info);
2433 }
2434
2435 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
2436                                         struct netlink_callback *cb)
2437 {
2438         struct devlink *devlink;
2439         struct devlink_sb *devlink_sb;
2440         int start = cb->args[0];
2441         unsigned long index;
2442         int idx = 0;
2443         int err;
2444
2445         mutex_lock(&devlink_mutex);
2446         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2447                 if (!devlink_try_get(devlink))
2448                         continue;
2449
2450                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2451                         goto retry;
2452
2453                 devl_lock(devlink);
2454                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2455                         if (idx < start) {
2456                                 idx++;
2457                                 continue;
2458                         }
2459                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2460                                                  DEVLINK_CMD_SB_NEW,
2461                                                  NETLINK_CB(cb->skb).portid,
2462                                                  cb->nlh->nlmsg_seq,
2463                                                  NLM_F_MULTI);
2464                         if (err) {
2465                                 devl_unlock(devlink);
2466                                 devlink_put(devlink);
2467                                 goto out;
2468                         }
2469                         idx++;
2470                 }
2471                 devl_unlock(devlink);
2472 retry:
2473                 devlink_put(devlink);
2474         }
2475 out:
2476         mutex_unlock(&devlink_mutex);
2477
2478         cb->args[0] = idx;
2479         return msg->len;
2480 }
2481
2482 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
2483                                    struct devlink_sb *devlink_sb,
2484                                    u16 pool_index, enum devlink_command cmd,
2485                                    u32 portid, u32 seq, int flags)
2486 {
2487         struct devlink_sb_pool_info pool_info;
2488         void *hdr;
2489         int err;
2490
2491         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
2492                                         pool_index, &pool_info);
2493         if (err)
2494                 return err;
2495
2496         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2497         if (!hdr)
2498                 return -EMSGSIZE;
2499
2500         if (devlink_nl_put_handle(msg, devlink))
2501                 goto nla_put_failure;
2502         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2503                 goto nla_put_failure;
2504         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2505                 goto nla_put_failure;
2506         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
2507                 goto nla_put_failure;
2508         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
2509                 goto nla_put_failure;
2510         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
2511                        pool_info.threshold_type))
2512                 goto nla_put_failure;
2513         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
2514                         pool_info.cell_size))
2515                 goto nla_put_failure;
2516
2517         genlmsg_end(msg, hdr);
2518         return 0;
2519
2520 nla_put_failure:
2521         genlmsg_cancel(msg, hdr);
2522         return -EMSGSIZE;
2523 }
2524
2525 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
2526                                            struct genl_info *info)
2527 {
2528         struct devlink *devlink = info->user_ptr[0];
2529         struct devlink_sb *devlink_sb;
2530         struct sk_buff *msg;
2531         u16 pool_index;
2532         int err;
2533
2534         devlink_sb = devlink_sb_get_from_info(devlink, info);
2535         if (IS_ERR(devlink_sb))
2536                 return PTR_ERR(devlink_sb);
2537
2538         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2539                                                   &pool_index);
2540         if (err)
2541                 return err;
2542
2543         if (!devlink->ops->sb_pool_get)
2544                 return -EOPNOTSUPP;
2545
2546         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2547         if (!msg)
2548                 return -ENOMEM;
2549
2550         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
2551                                       DEVLINK_CMD_SB_POOL_NEW,
2552                                       info->snd_portid, info->snd_seq, 0);
2553         if (err) {
2554                 nlmsg_free(msg);
2555                 return err;
2556         }
2557
2558         return genlmsg_reply(msg, info);
2559 }
2560
2561 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2562                                 struct devlink *devlink,
2563                                 struct devlink_sb *devlink_sb,
2564                                 u32 portid, u32 seq)
2565 {
2566         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2567         u16 pool_index;
2568         int err;
2569
2570         for (pool_index = 0; pool_index < pool_count; pool_index++) {
2571                 if (*p_idx < start) {
2572                         (*p_idx)++;
2573                         continue;
2574                 }
2575                 err = devlink_nl_sb_pool_fill(msg, devlink,
2576                                               devlink_sb,
2577                                               pool_index,
2578                                               DEVLINK_CMD_SB_POOL_NEW,
2579                                               portid, seq, NLM_F_MULTI);
2580                 if (err)
2581                         return err;
2582                 (*p_idx)++;
2583         }
2584         return 0;
2585 }
2586
2587 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
2588                                              struct netlink_callback *cb)
2589 {
2590         struct devlink *devlink;
2591         struct devlink_sb *devlink_sb;
2592         int start = cb->args[0];
2593         unsigned long index;
2594         int idx = 0;
2595         int err = 0;
2596
2597         mutex_lock(&devlink_mutex);
2598         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2599                 if (!devlink_try_get(devlink))
2600                         continue;
2601
2602                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2603                     !devlink->ops->sb_pool_get)
2604                         goto retry;
2605
2606                 devl_lock(devlink);
2607                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2608                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
2609                                                    devlink_sb,
2610                                                    NETLINK_CB(cb->skb).portid,
2611                                                    cb->nlh->nlmsg_seq);
2612                         if (err == -EOPNOTSUPP) {
2613                                 err = 0;
2614                         } else if (err) {
2615                                 devl_unlock(devlink);
2616                                 devlink_put(devlink);
2617                                 goto out;
2618                         }
2619                 }
2620                 devl_unlock(devlink);
2621 retry:
2622                 devlink_put(devlink);
2623         }
2624 out:
2625         mutex_unlock(&devlink_mutex);
2626
2627         if (err != -EMSGSIZE)
2628                 return err;
2629
2630         cb->args[0] = idx;
2631         return msg->len;
2632 }
2633
2634 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
2635                                u16 pool_index, u32 size,
2636                                enum devlink_sb_threshold_type threshold_type,
2637                                struct netlink_ext_ack *extack)
2638
2639 {
2640         const struct devlink_ops *ops = devlink->ops;
2641
2642         if (ops->sb_pool_set)
2643                 return ops->sb_pool_set(devlink, sb_index, pool_index,
2644                                         size, threshold_type, extack);
2645         return -EOPNOTSUPP;
2646 }
2647
2648 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
2649                                            struct genl_info *info)
2650 {
2651         struct devlink *devlink = info->user_ptr[0];
2652         enum devlink_sb_threshold_type threshold_type;
2653         struct devlink_sb *devlink_sb;
2654         u16 pool_index;
2655         u32 size;
2656         int err;
2657
2658         devlink_sb = devlink_sb_get_from_info(devlink, info);
2659         if (IS_ERR(devlink_sb))
2660                 return PTR_ERR(devlink_sb);
2661
2662         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2663                                                   &pool_index);
2664         if (err)
2665                 return err;
2666
2667         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
2668         if (err)
2669                 return err;
2670
2671         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
2672                 return -EINVAL;
2673
2674         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
2675         return devlink_sb_pool_set(devlink, devlink_sb->index,
2676                                    pool_index, size, threshold_type,
2677                                    info->extack);
2678 }
2679
2680 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
2681                                         struct devlink *devlink,
2682                                         struct devlink_port *devlink_port,
2683                                         struct devlink_sb *devlink_sb,
2684                                         u16 pool_index,
2685                                         enum devlink_command cmd,
2686                                         u32 portid, u32 seq, int flags)
2687 {
2688         const struct devlink_ops *ops = devlink->ops;
2689         u32 threshold;
2690         void *hdr;
2691         int err;
2692
2693         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
2694                                     pool_index, &threshold);
2695         if (err)
2696                 return err;
2697
2698         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2699         if (!hdr)
2700                 return -EMSGSIZE;
2701
2702         if (devlink_nl_put_handle(msg, devlink))
2703                 goto nla_put_failure;
2704         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2705                 goto nla_put_failure;
2706         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2707                 goto nla_put_failure;
2708         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2709                 goto nla_put_failure;
2710         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2711                 goto nla_put_failure;
2712
2713         if (ops->sb_occ_port_pool_get) {
2714                 u32 cur;
2715                 u32 max;
2716
2717                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
2718                                                 pool_index, &cur, &max);
2719                 if (err && err != -EOPNOTSUPP)
2720                         goto sb_occ_get_failure;
2721                 if (!err) {
2722                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2723                                 goto nla_put_failure;
2724                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2725                                 goto nla_put_failure;
2726                 }
2727         }
2728
2729         genlmsg_end(msg, hdr);
2730         return 0;
2731
2732 nla_put_failure:
2733         err = -EMSGSIZE;
2734 sb_occ_get_failure:
2735         genlmsg_cancel(msg, hdr);
2736         return err;
2737 }
2738
2739 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
2740                                                 struct genl_info *info)
2741 {
2742         struct devlink_port *devlink_port = info->user_ptr[1];
2743         struct devlink *devlink = devlink_port->devlink;
2744         struct devlink_sb *devlink_sb;
2745         struct sk_buff *msg;
2746         u16 pool_index;
2747         int err;
2748
2749         devlink_sb = devlink_sb_get_from_info(devlink, info);
2750         if (IS_ERR(devlink_sb))
2751                 return PTR_ERR(devlink_sb);
2752
2753         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2754                                                   &pool_index);
2755         if (err)
2756                 return err;
2757
2758         if (!devlink->ops->sb_port_pool_get)
2759                 return -EOPNOTSUPP;
2760
2761         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2762         if (!msg)
2763                 return -ENOMEM;
2764
2765         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
2766                                            devlink_sb, pool_index,
2767                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2768                                            info->snd_portid, info->snd_seq, 0);
2769         if (err) {
2770                 nlmsg_free(msg);
2771                 return err;
2772         }
2773
2774         return genlmsg_reply(msg, info);
2775 }
2776
2777 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2778                                      struct devlink *devlink,
2779                                      struct devlink_sb *devlink_sb,
2780                                      u32 portid, u32 seq)
2781 {
2782         struct devlink_port *devlink_port;
2783         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2784         u16 pool_index;
2785         int err;
2786
2787         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2788                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
2789                         if (*p_idx < start) {
2790                                 (*p_idx)++;
2791                                 continue;
2792                         }
2793                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
2794                                                            devlink_port,
2795                                                            devlink_sb,
2796                                                            pool_index,
2797                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2798                                                            portid, seq,
2799                                                            NLM_F_MULTI);
2800                         if (err)
2801                                 return err;
2802                         (*p_idx)++;
2803                 }
2804         }
2805         return 0;
2806 }
2807
2808 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
2809                                                   struct netlink_callback *cb)
2810 {
2811         struct devlink *devlink;
2812         struct devlink_sb *devlink_sb;
2813         int start = cb->args[0];
2814         unsigned long index;
2815         int idx = 0;
2816         int err = 0;
2817
2818         mutex_lock(&devlink_mutex);
2819         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2820                 if (!devlink_try_get(devlink))
2821                         continue;
2822
2823                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2824                     !devlink->ops->sb_port_pool_get)
2825                         goto retry;
2826
2827                 devl_lock(devlink);
2828                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2829                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
2830                                                         devlink, devlink_sb,
2831                                                         NETLINK_CB(cb->skb).portid,
2832                                                         cb->nlh->nlmsg_seq);
2833                         if (err == -EOPNOTSUPP) {
2834                                 err = 0;
2835                         } else if (err) {
2836                                 devl_unlock(devlink);
2837                                 devlink_put(devlink);
2838                                 goto out;
2839                         }
2840                 }
2841                 devl_unlock(devlink);
2842 retry:
2843                 devlink_put(devlink);
2844         }
2845 out:
2846         mutex_unlock(&devlink_mutex);
2847
2848         if (err != -EMSGSIZE)
2849                 return err;
2850
2851         cb->args[0] = idx;
2852         return msg->len;
2853 }
2854
2855 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
2856                                     unsigned int sb_index, u16 pool_index,
2857                                     u32 threshold,
2858                                     struct netlink_ext_ack *extack)
2859
2860 {
2861         const struct devlink_ops *ops = devlink_port->devlink->ops;
2862
2863         if (ops->sb_port_pool_set)
2864                 return ops->sb_port_pool_set(devlink_port, sb_index,
2865                                              pool_index, threshold, extack);
2866         return -EOPNOTSUPP;
2867 }
2868
2869 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
2870                                                 struct genl_info *info)
2871 {
2872         struct devlink_port *devlink_port = info->user_ptr[1];
2873         struct devlink *devlink = info->user_ptr[0];
2874         struct devlink_sb *devlink_sb;
2875         u16 pool_index;
2876         u32 threshold;
2877         int err;
2878
2879         devlink_sb = devlink_sb_get_from_info(devlink, info);
2880         if (IS_ERR(devlink_sb))
2881                 return PTR_ERR(devlink_sb);
2882
2883         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2884                                                   &pool_index);
2885         if (err)
2886                 return err;
2887
2888         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2889                 return -EINVAL;
2890
2891         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2892         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
2893                                         pool_index, threshold, info->extack);
2894 }
2895
2896 static int
2897 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
2898                                 struct devlink_port *devlink_port,
2899                                 struct devlink_sb *devlink_sb, u16 tc_index,
2900                                 enum devlink_sb_pool_type pool_type,
2901                                 enum devlink_command cmd,
2902                                 u32 portid, u32 seq, int flags)
2903 {
2904         const struct devlink_ops *ops = devlink->ops;
2905         u16 pool_index;
2906         u32 threshold;
2907         void *hdr;
2908         int err;
2909
2910         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
2911                                        tc_index, pool_type,
2912                                        &pool_index, &threshold);
2913         if (err)
2914                 return err;
2915
2916         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2917         if (!hdr)
2918                 return -EMSGSIZE;
2919
2920         if (devlink_nl_put_handle(msg, devlink))
2921                 goto nla_put_failure;
2922         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2923                 goto nla_put_failure;
2924         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2925                 goto nla_put_failure;
2926         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
2927                 goto nla_put_failure;
2928         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
2929                 goto nla_put_failure;
2930         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2931                 goto nla_put_failure;
2932         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2933                 goto nla_put_failure;
2934
2935         if (ops->sb_occ_tc_port_bind_get) {
2936                 u32 cur;
2937                 u32 max;
2938
2939                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
2940                                                    devlink_sb->index,
2941                                                    tc_index, pool_type,
2942                                                    &cur, &max);
2943                 if (err && err != -EOPNOTSUPP)
2944                         return err;
2945                 if (!err) {
2946                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2947                                 goto nla_put_failure;
2948                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2949                                 goto nla_put_failure;
2950                 }
2951         }
2952
2953         genlmsg_end(msg, hdr);
2954         return 0;
2955
2956 nla_put_failure:
2957         genlmsg_cancel(msg, hdr);
2958         return -EMSGSIZE;
2959 }
2960
2961 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
2962                                                    struct genl_info *info)
2963 {
2964         struct devlink_port *devlink_port = info->user_ptr[1];
2965         struct devlink *devlink = devlink_port->devlink;
2966         struct devlink_sb *devlink_sb;
2967         struct sk_buff *msg;
2968         enum devlink_sb_pool_type pool_type;
2969         u16 tc_index;
2970         int err;
2971
2972         devlink_sb = devlink_sb_get_from_info(devlink, info);
2973         if (IS_ERR(devlink_sb))
2974                 return PTR_ERR(devlink_sb);
2975
2976         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2977         if (err)
2978                 return err;
2979
2980         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2981                                                 pool_type, &tc_index);
2982         if (err)
2983                 return err;
2984
2985         if (!devlink->ops->sb_tc_pool_bind_get)
2986                 return -EOPNOTSUPP;
2987
2988         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2989         if (!msg)
2990                 return -ENOMEM;
2991
2992         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
2993                                               devlink_sb, tc_index, pool_type,
2994                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2995                                               info->snd_portid,
2996                                               info->snd_seq, 0);
2997         if (err) {
2998                 nlmsg_free(msg);
2999                 return err;
3000         }
3001
3002         return genlmsg_reply(msg, info);
3003 }
3004
3005 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
3006                                         int start, int *p_idx,
3007                                         struct devlink *devlink,
3008                                         struct devlink_sb *devlink_sb,
3009                                         u32 portid, u32 seq)
3010 {
3011         struct devlink_port *devlink_port;
3012         u16 tc_index;
3013         int err;
3014
3015         list_for_each_entry(devlink_port, &devlink->port_list, list) {
3016                 for (tc_index = 0;
3017                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
3018                         if (*p_idx < start) {
3019                                 (*p_idx)++;
3020                                 continue;
3021                         }
3022                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
3023                                                               devlink_port,
3024                                                               devlink_sb,
3025                                                               tc_index,
3026                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
3027                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
3028                                                               portid, seq,
3029                                                               NLM_F_MULTI);
3030                         if (err)
3031                                 return err;
3032                         (*p_idx)++;
3033                 }
3034                 for (tc_index = 0;
3035                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
3036                         if (*p_idx < start) {
3037                                 (*p_idx)++;
3038                                 continue;
3039                         }
3040                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
3041                                                               devlink_port,
3042                                                               devlink_sb,
3043                                                               tc_index,
3044                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
3045                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
3046                                                               portid, seq,
3047                                                               NLM_F_MULTI);
3048                         if (err)
3049                                 return err;
3050                         (*p_idx)++;
3051                 }
3052         }
3053         return 0;
3054 }
3055
3056 static int
3057 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
3058                                           struct netlink_callback *cb)
3059 {
3060         struct devlink *devlink;
3061         struct devlink_sb *devlink_sb;
3062         int start = cb->args[0];
3063         unsigned long index;
3064         int idx = 0;
3065         int err = 0;
3066
3067         mutex_lock(&devlink_mutex);
3068         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
3069                 if (!devlink_try_get(devlink))
3070                         continue;
3071
3072                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
3073                     !devlink->ops->sb_tc_pool_bind_get)
3074                         goto retry;
3075
3076                 devl_lock(devlink);
3077                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
3078                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
3079                                                            devlink,
3080                                                            devlink_sb,
3081                                                            NETLINK_CB(cb->skb).portid,
3082                                                            cb->nlh->nlmsg_seq);
3083                         if (err == -EOPNOTSUPP) {
3084                                 err = 0;
3085                         } else if (err) {
3086                                 devl_unlock(devlink);
3087                                 devlink_put(devlink);
3088                                 goto out;
3089                         }
3090                 }
3091                 devl_unlock(devlink);
3092 retry:
3093                 devlink_put(devlink);
3094         }
3095 out:
3096         mutex_unlock(&devlink_mutex);
3097
3098         if (err != -EMSGSIZE)
3099                 return err;
3100
3101         cb->args[0] = idx;
3102         return msg->len;
3103 }
3104
3105 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
3106                                        unsigned int sb_index, u16 tc_index,
3107                                        enum devlink_sb_pool_type pool_type,
3108                                        u16 pool_index, u32 threshold,
3109                                        struct netlink_ext_ack *extack)
3110
3111 {
3112         const struct devlink_ops *ops = devlink_port->devlink->ops;
3113
3114         if (ops->sb_tc_pool_bind_set)
3115                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
3116                                                 tc_index, pool_type,
3117                                                 pool_index, threshold, extack);
3118         return -EOPNOTSUPP;
3119 }
3120
3121 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
3122                                                    struct genl_info *info)
3123 {
3124         struct devlink_port *devlink_port = info->user_ptr[1];
3125         struct devlink *devlink = info->user_ptr[0];
3126         enum devlink_sb_pool_type pool_type;
3127         struct devlink_sb *devlink_sb;
3128         u16 tc_index;
3129         u16 pool_index;
3130         u32 threshold;
3131         int err;
3132
3133         devlink_sb = devlink_sb_get_from_info(devlink, info);
3134         if (IS_ERR(devlink_sb))
3135                 return PTR_ERR(devlink_sb);
3136
3137         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
3138         if (err)
3139                 return err;
3140
3141         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
3142                                                 pool_type, &tc_index);
3143         if (err)
3144                 return err;
3145
3146         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
3147                                                   &pool_index);
3148         if (err)
3149                 return err;
3150
3151         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
3152                 return -EINVAL;
3153
3154         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
3155         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
3156                                            tc_index, pool_type,
3157                                            pool_index, threshold, info->extack);
3158 }
3159
3160 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
3161                                                struct genl_info *info)
3162 {
3163         struct devlink *devlink = info->user_ptr[0];
3164         const struct devlink_ops *ops = devlink->ops;
3165         struct devlink_sb *devlink_sb;
3166
3167         devlink_sb = devlink_sb_get_from_info(devlink, info);
3168         if (IS_ERR(devlink_sb))
3169                 return PTR_ERR(devlink_sb);
3170
3171         if (ops->sb_occ_snapshot)
3172                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
3173         return -EOPNOTSUPP;
3174 }
3175
3176 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
3177                                                 struct genl_info *info)
3178 {
3179         struct devlink *devlink = info->user_ptr[0];
3180         const struct devlink_ops *ops = devlink->ops;
3181         struct devlink_sb *devlink_sb;
3182
3183         devlink_sb = devlink_sb_get_from_info(devlink, info);
3184         if (IS_ERR(devlink_sb))
3185                 return PTR_ERR(devlink_sb);
3186
3187         if (ops->sb_occ_max_clear)
3188                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
3189         return -EOPNOTSUPP;
3190 }
3191
3192 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
3193                                    enum devlink_command cmd, u32 portid,
3194                                    u32 seq, int flags)
3195 {
3196         const struct devlink_ops *ops = devlink->ops;
3197         enum devlink_eswitch_encap_mode encap_mode;
3198         u8 inline_mode;
3199         void *hdr;
3200         int err = 0;
3201         u16 mode;
3202
3203         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
3204         if (!hdr)
3205                 return -EMSGSIZE;
3206
3207         err = devlink_nl_put_handle(msg, devlink);
3208         if (err)
3209                 goto nla_put_failure;
3210
3211         if (ops->eswitch_mode_get) {
3212                 err = ops->eswitch_mode_get(devlink, &mode);
3213                 if (err)
3214                         goto nla_put_failure;
3215                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
3216                 if (err)
3217                         goto nla_put_failure;
3218         }
3219
3220         if (ops->eswitch_inline_mode_get) {
3221                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
3222                 if (err)
3223                         goto nla_put_failure;
3224                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
3225                                  inline_mode);
3226                 if (err)
3227                         goto nla_put_failure;
3228         }
3229
3230         if (ops->eswitch_encap_mode_get) {
3231                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
3232                 if (err)
3233                         goto nla_put_failure;
3234                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
3235                 if (err)
3236                         goto nla_put_failure;
3237         }
3238
3239         genlmsg_end(msg, hdr);
3240         return 0;
3241
3242 nla_put_failure:
3243         genlmsg_cancel(msg, hdr);
3244         return err;
3245 }
3246
3247 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
3248                                            struct genl_info *info)
3249 {
3250         struct devlink *devlink = info->user_ptr[0];
3251         struct sk_buff *msg;
3252         int err;
3253
3254         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3255         if (!msg)
3256                 return -ENOMEM;
3257
3258         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
3259                                       info->snd_portid, info->snd_seq, 0);
3260
3261         if (err) {
3262                 nlmsg_free(msg);
3263                 return err;
3264         }
3265
3266         return genlmsg_reply(msg, info);
3267 }
3268
3269 static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
3270                                     struct netlink_ext_ack *extack)
3271 {
3272         struct devlink_rate *devlink_rate;
3273
3274         list_for_each_entry(devlink_rate, &devlink->rate_list, list)
3275                 if (devlink_rate_is_node(devlink_rate)) {
3276                         NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
3277                         return -EBUSY;
3278                 }
3279         return 0;
3280 }
3281
3282 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
3283                                            struct genl_info *info)
3284 {
3285         struct devlink *devlink = info->user_ptr[0];
3286         const struct devlink_ops *ops = devlink->ops;
3287         enum devlink_eswitch_encap_mode encap_mode;
3288         u8 inline_mode;
3289         int err = 0;
3290         u16 mode;
3291
3292         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
3293                 if (!ops->eswitch_mode_set)
3294                         return -EOPNOTSUPP;
3295                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
3296                 err = devlink_rate_nodes_check(devlink, mode, info->extack);
3297                 if (err)
3298                         return err;
3299                 err = ops->eswitch_mode_set(devlink, mode, info->extack);
3300                 if (err)
3301                         return err;
3302         }
3303
3304         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
3305                 if (!ops->eswitch_inline_mode_set)
3306                         return -EOPNOTSUPP;
3307                 inline_mode = nla_get_u8(
3308                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
3309                 err = ops->eswitch_inline_mode_set(devlink, inline_mode,
3310                                                    info->extack);
3311                 if (err)
3312                         return err;
3313         }
3314
3315         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
3316                 if (!ops->eswitch_encap_mode_set)
3317                         return -EOPNOTSUPP;
3318                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
3319                 err = ops->eswitch_encap_mode_set(devlink, encap_mode,
3320                                                   info->extack);
3321                 if (err)
3322                         return err;
3323         }
3324
3325         return 0;
3326 }
3327
3328 int devlink_dpipe_match_put(struct sk_buff *skb,
3329                             struct devlink_dpipe_match *match)
3330 {
3331         struct devlink_dpipe_header *header = match->header;
3332         struct devlink_dpipe_field *field = &header->fields[match->field_id];
3333         struct nlattr *match_attr;
3334
3335         match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
3336         if (!match_attr)
3337                 return -EMSGSIZE;
3338
3339         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
3340             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
3341             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3342             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3343             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3344                 goto nla_put_failure;
3345
3346         nla_nest_end(skb, match_attr);
3347         return 0;
3348
3349 nla_put_failure:
3350         nla_nest_cancel(skb, match_attr);
3351         return -EMSGSIZE;
3352 }
3353 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
3354
3355 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
3356                                      struct sk_buff *skb)
3357 {
3358         struct nlattr *matches_attr;
3359
3360         matches_attr = nla_nest_start_noflag(skb,
3361                                              DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
3362         if (!matches_attr)
3363                 return -EMSGSIZE;
3364
3365         if (table->table_ops->matches_dump(table->priv, skb))
3366                 goto nla_put_failure;
3367
3368         nla_nest_end(skb, matches_attr);
3369         return 0;
3370
3371 nla_put_failure:
3372         nla_nest_cancel(skb, matches_attr);
3373         return -EMSGSIZE;
3374 }
3375
3376 int devlink_dpipe_action_put(struct sk_buff *skb,
3377                              struct devlink_dpipe_action *action)
3378 {
3379         struct devlink_dpipe_header *header = action->header;
3380         struct devlink_dpipe_field *field = &header->fields[action->field_id];
3381         struct nlattr *action_attr;
3382
3383         action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
3384         if (!action_attr)
3385                 return -EMSGSIZE;
3386
3387         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
3388             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
3389             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3390             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3391             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3392                 goto nla_put_failure;
3393
3394         nla_nest_end(skb, action_attr);
3395         return 0;
3396
3397 nla_put_failure:
3398         nla_nest_cancel(skb, action_attr);
3399         return -EMSGSIZE;
3400 }
3401 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
3402
3403 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
3404                                      struct sk_buff *skb)
3405 {
3406         struct nlattr *actions_attr;
3407
3408         actions_attr = nla_nest_start_noflag(skb,
3409                                              DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
3410         if (!actions_attr)
3411                 return -EMSGSIZE;
3412
3413         if (table->table_ops->actions_dump(table->priv, skb))
3414                 goto nla_put_failure;
3415
3416         nla_nest_end(skb, actions_attr);
3417         return 0;
3418
3419 nla_put_failure:
3420         nla_nest_cancel(skb, actions_attr);
3421         return -EMSGSIZE;
3422 }
3423
3424 static int devlink_dpipe_table_put(struct sk_buff *skb,
3425                                    struct devlink_dpipe_table *table)
3426 {
3427         struct nlattr *table_attr;
3428         u64 table_size;
3429
3430         table_size = table->table_ops->size_get(table->priv);
3431         table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
3432         if (!table_attr)
3433                 return -EMSGSIZE;
3434
3435         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
3436             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
3437                               DEVLINK_ATTR_PAD))
3438                 goto nla_put_failure;
3439         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
3440                        table->counters_enabled))
3441                 goto nla_put_failure;
3442
3443         if (table->resource_valid) {
3444                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
3445                                       table->resource_id, DEVLINK_ATTR_PAD) ||
3446                     nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
3447                                       table->resource_units, DEVLINK_ATTR_PAD))
3448                         goto nla_put_failure;
3449         }
3450         if (devlink_dpipe_matches_put(table, skb))
3451                 goto nla_put_failure;
3452
3453         if (devlink_dpipe_actions_put(table, skb))
3454                 goto nla_put_failure;
3455
3456         nla_nest_end(skb, table_attr);
3457         return 0;
3458
3459 nla_put_failure:
3460         nla_nest_cancel(skb, table_attr);
3461         return -EMSGSIZE;
3462 }
3463
3464 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
3465                                             struct genl_info *info)
3466 {
3467         int err;
3468
3469         if (*pskb) {
3470                 err = genlmsg_reply(*pskb, info);
3471                 if (err)
3472                         return err;
3473         }
3474         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
3475         if (!*pskb)
3476                 return -ENOMEM;
3477         return 0;
3478 }
3479
3480 static int devlink_dpipe_tables_fill(struct genl_info *info,
3481                                      enum devlink_command cmd, int flags,
3482                                      struct list_head *dpipe_tables,
3483                                      const char *table_name)
3484 {
3485         struct devlink *devlink = info->user_ptr[0];
3486         struct devlink_dpipe_table *table;
3487         struct nlattr *tables_attr;
3488         struct sk_buff *skb = NULL;
3489         struct nlmsghdr *nlh;
3490         bool incomplete;
3491         void *hdr;
3492         int i;
3493         int err;
3494
3495         table = list_first_entry(dpipe_tables,
3496                                  struct devlink_dpipe_table, list);
3497 start_again:
3498         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3499         if (err)
3500                 return err;
3501
3502         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3503                           &devlink_nl_family, NLM_F_MULTI, cmd);
3504         if (!hdr) {
3505                 nlmsg_free(skb);
3506                 return -EMSGSIZE;
3507         }
3508
3509         if (devlink_nl_put_handle(skb, devlink))
3510                 goto nla_put_failure;
3511         tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
3512         if (!tables_attr)
3513                 goto nla_put_failure;
3514
3515         i = 0;
3516         incomplete = false;
3517         list_for_each_entry_from(table, dpipe_tables, list) {
3518                 if (!table_name) {
3519                         err = devlink_dpipe_table_put(skb, table);
3520                         if (err) {
3521                                 if (!i)
3522                                         goto err_table_put;
3523                                 incomplete = true;
3524                                 break;
3525                         }
3526                 } else {
3527                         if (!strcmp(table->name, table_name)) {
3528                                 err = devlink_dpipe_table_put(skb, table);
3529                                 if (err)
3530                                         break;
3531                         }
3532                 }
3533                 i++;
3534         }
3535
3536         nla_nest_end(skb, tables_attr);
3537         genlmsg_end(skb, hdr);
3538         if (incomplete)
3539                 goto start_again;
3540
3541 send_done:
3542         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3543                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3544         if (!nlh) {
3545                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3546                 if (err)
3547                         return err;
3548                 goto send_done;
3549         }
3550
3551         return genlmsg_reply(skb, info);
3552
3553 nla_put_failure:
3554         err = -EMSGSIZE;
3555 err_table_put:
3556         nlmsg_free(skb);
3557         return err;
3558 }
3559
3560 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
3561                                           struct genl_info *info)
3562 {
3563         struct devlink *devlink = info->user_ptr[0];
3564         const char *table_name =  NULL;
3565
3566         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3567                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3568
3569         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
3570                                          &devlink->dpipe_table_list,
3571                                          table_name);
3572 }
3573
3574 static int devlink_dpipe_value_put(struct sk_buff *skb,
3575                                    struct devlink_dpipe_value *value)
3576 {
3577         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
3578                     value->value_size, value->value))
3579                 return -EMSGSIZE;
3580         if (value->mask)
3581                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
3582                             value->value_size, value->mask))
3583                         return -EMSGSIZE;
3584         if (value->mapping_valid)
3585                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
3586                                 value->mapping_value))
3587                         return -EMSGSIZE;
3588         return 0;
3589 }
3590
3591 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
3592                                           struct devlink_dpipe_value *value)
3593 {
3594         if (!value->action)
3595                 return -EINVAL;
3596         if (devlink_dpipe_action_put(skb, value->action))
3597                 return -EMSGSIZE;
3598         if (devlink_dpipe_value_put(skb, value))
3599                 return -EMSGSIZE;
3600         return 0;
3601 }
3602
3603 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
3604                                            struct devlink_dpipe_value *values,
3605                                            unsigned int values_count)
3606 {
3607         struct nlattr *action_attr;
3608         int i;
3609         int err;
3610
3611         for (i = 0; i < values_count; i++) {
3612                 action_attr = nla_nest_start_noflag(skb,
3613                                                     DEVLINK_ATTR_DPIPE_ACTION_VALUE);
3614                 if (!action_attr)
3615                         return -EMSGSIZE;
3616                 err = devlink_dpipe_action_value_put(skb, &values[i]);
3617                 if (err)
3618                         goto err_action_value_put;
3619                 nla_nest_end(skb, action_attr);
3620         }
3621         return 0;
3622
3623 err_action_value_put:
3624         nla_nest_cancel(skb, action_attr);
3625         return err;
3626 }
3627
3628 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
3629                                          struct devlink_dpipe_value *value)
3630 {
3631         if (!value->match)
3632                 return -EINVAL;
3633         if (devlink_dpipe_match_put(skb, value->match))
3634                 return -EMSGSIZE;
3635         if (devlink_dpipe_value_put(skb, value))
3636                 return -EMSGSIZE;
3637         return 0;
3638 }
3639
3640 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
3641                                           struct devlink_dpipe_value *values,
3642                                           unsigned int values_count)
3643 {
3644         struct nlattr *match_attr;
3645         int i;
3646         int err;
3647
3648         for (i = 0; i < values_count; i++) {
3649                 match_attr = nla_nest_start_noflag(skb,
3650                                                    DEVLINK_ATTR_DPIPE_MATCH_VALUE);
3651                 if (!match_attr)
3652                         return -EMSGSIZE;
3653                 err = devlink_dpipe_match_value_put(skb, &values[i]);
3654                 if (err)
3655                         goto err_match_value_put;
3656                 nla_nest_end(skb, match_attr);
3657         }
3658         return 0;
3659
3660 err_match_value_put:
3661         nla_nest_cancel(skb, match_attr);
3662         return err;
3663 }
3664
3665 static int devlink_dpipe_entry_put(struct sk_buff *skb,
3666                                    struct devlink_dpipe_entry *entry)
3667 {
3668         struct nlattr *entry_attr, *matches_attr, *actions_attr;
3669         int err;
3670
3671         entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
3672         if (!entry_attr)
3673                 return  -EMSGSIZE;
3674
3675         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
3676                               DEVLINK_ATTR_PAD))
3677                 goto nla_put_failure;
3678         if (entry->counter_valid)
3679                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
3680                                       entry->counter, DEVLINK_ATTR_PAD))
3681                         goto nla_put_failure;
3682
3683         matches_attr = nla_nest_start_noflag(skb,
3684                                              DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
3685         if (!matches_attr)
3686                 goto nla_put_failure;
3687
3688         err = devlink_dpipe_match_values_put(skb, entry->match_values,
3689                                              entry->match_values_count);
3690         if (err) {
3691                 nla_nest_cancel(skb, matches_attr);
3692                 goto err_match_values_put;
3693         }
3694         nla_nest_end(skb, matches_attr);
3695
3696         actions_attr = nla_nest_start_noflag(skb,
3697                                              DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
3698         if (!actions_attr)
3699                 goto nla_put_failure;
3700
3701         err = devlink_dpipe_action_values_put(skb, entry->action_values,
3702                                               entry->action_values_count);
3703         if (err) {
3704                 nla_nest_cancel(skb, actions_attr);
3705                 goto err_action_values_put;
3706         }
3707         nla_nest_end(skb, actions_attr);
3708
3709         nla_nest_end(skb, entry_attr);
3710         return 0;
3711
3712 nla_put_failure:
3713         err = -EMSGSIZE;
3714 err_match_values_put:
3715 err_action_values_put:
3716         nla_nest_cancel(skb, entry_attr);
3717         return err;
3718 }
3719
3720 static struct devlink_dpipe_table *
3721 devlink_dpipe_table_find(struct list_head *dpipe_tables,
3722                          const char *table_name, struct devlink *devlink)
3723 {
3724         struct devlink_dpipe_table *table;
3725         list_for_each_entry_rcu(table, dpipe_tables, list,
3726                                 lockdep_is_held(&devlink->lock)) {
3727                 if (!strcmp(table->name, table_name))
3728                         return table;
3729         }
3730         return NULL;
3731 }
3732
3733 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
3734 {
3735         struct devlink *devlink;
3736         int err;
3737
3738         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
3739                                                dump_ctx->info);
3740         if (err)
3741                 return err;
3742
3743         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
3744                                     dump_ctx->info->snd_portid,
3745                                     dump_ctx->info->snd_seq,
3746                                     &devlink_nl_family, NLM_F_MULTI,
3747                                     dump_ctx->cmd);
3748         if (!dump_ctx->hdr)
3749                 goto nla_put_failure;
3750
3751         devlink = dump_ctx->info->user_ptr[0];
3752         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
3753                 goto nla_put_failure;
3754         dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
3755                                                DEVLINK_ATTR_DPIPE_ENTRIES);
3756         if (!dump_ctx->nest)
3757                 goto nla_put_failure;
3758         return 0;
3759
3760 nla_put_failure:
3761         nlmsg_free(dump_ctx->skb);
3762         return -EMSGSIZE;
3763 }
3764 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
3765
3766 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
3767                                    struct devlink_dpipe_entry *entry)
3768 {
3769         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
3770 }
3771 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
3772
3773 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
3774 {
3775         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
3776         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
3777         return 0;
3778 }
3779 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
3780
3781 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
3782
3783 {
3784         unsigned int value_count, value_index;
3785         struct devlink_dpipe_value *value;
3786
3787         value = entry->action_values;
3788         value_count = entry->action_values_count;
3789         for (value_index = 0; value_index < value_count; value_index++) {
3790                 kfree(value[value_index].value);
3791                 kfree(value[value_index].mask);
3792         }
3793
3794         value = entry->match_values;
3795         value_count = entry->match_values_count;
3796         for (value_index = 0; value_index < value_count; value_index++) {
3797                 kfree(value[value_index].value);
3798                 kfree(value[value_index].mask);
3799         }
3800 }
3801 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
3802
3803 static int devlink_dpipe_entries_fill(struct genl_info *info,
3804                                       enum devlink_command cmd, int flags,
3805                                       struct devlink_dpipe_table *table)
3806 {
3807         struct devlink_dpipe_dump_ctx dump_ctx;
3808         struct nlmsghdr *nlh;
3809         int err;
3810
3811         dump_ctx.skb = NULL;
3812         dump_ctx.cmd = cmd;
3813         dump_ctx.info = info;
3814
3815         err = table->table_ops->entries_dump(table->priv,
3816                                              table->counters_enabled,
3817                                              &dump_ctx);
3818         if (err)
3819                 return err;
3820
3821 send_done:
3822         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
3823                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3824         if (!nlh) {
3825                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
3826                 if (err)
3827                         return err;
3828                 goto send_done;
3829         }
3830         return genlmsg_reply(dump_ctx.skb, info);
3831 }
3832
3833 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
3834                                             struct genl_info *info)
3835 {
3836         struct devlink *devlink = info->user_ptr[0];
3837         struct devlink_dpipe_table *table;
3838         const char *table_name;
3839
3840         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3841                 return -EINVAL;
3842
3843         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3844         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3845                                          table_name, devlink);
3846         if (!table)
3847                 return -EINVAL;
3848
3849         if (!table->table_ops->entries_dump)
3850                 return -EINVAL;
3851
3852         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
3853                                           0, table);
3854 }
3855
3856 static int devlink_dpipe_fields_put(struct sk_buff *skb,
3857                                     const struct devlink_dpipe_header *header)
3858 {
3859         struct devlink_dpipe_field *field;
3860         struct nlattr *field_attr;
3861         int i;
3862
3863         for (i = 0; i < header->fields_count; i++) {
3864                 field = &header->fields[i];
3865                 field_attr = nla_nest_start_noflag(skb,
3866                                                    DEVLINK_ATTR_DPIPE_FIELD);
3867                 if (!field_attr)
3868                         return -EMSGSIZE;
3869                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
3870                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3871                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
3872                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
3873                         goto nla_put_failure;
3874                 nla_nest_end(skb, field_attr);
3875         }
3876         return 0;
3877
3878 nla_put_failure:
3879         nla_nest_cancel(skb, field_attr);
3880         return -EMSGSIZE;
3881 }
3882
3883 static int devlink_dpipe_header_put(struct sk_buff *skb,
3884                                     struct devlink_dpipe_header *header)
3885 {
3886         struct nlattr *fields_attr, *header_attr;
3887         int err;
3888
3889         header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
3890         if (!header_attr)
3891                 return -EMSGSIZE;
3892
3893         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
3894             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3895             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3896                 goto nla_put_failure;
3897
3898         fields_attr = nla_nest_start_noflag(skb,
3899                                             DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
3900         if (!fields_attr)
3901                 goto nla_put_failure;
3902
3903         err = devlink_dpipe_fields_put(skb, header);
3904         if (err) {
3905                 nla_nest_cancel(skb, fields_attr);
3906                 goto nla_put_failure;
3907         }
3908         nla_nest_end(skb, fields_attr);
3909         nla_nest_end(skb, header_attr);
3910         return 0;
3911
3912 nla_put_failure:
3913         err = -EMSGSIZE;
3914         nla_nest_cancel(skb, header_attr);
3915         return err;
3916 }
3917
3918 static int devlink_dpipe_headers_fill(struct genl_info *info,
3919                                       enum devlink_command cmd, int flags,
3920                                       struct devlink_dpipe_headers *
3921                                       dpipe_headers)
3922 {
3923         struct devlink *devlink = info->user_ptr[0];
3924         struct nlattr *headers_attr;
3925         struct sk_buff *skb = NULL;
3926         struct nlmsghdr *nlh;
3927         void *hdr;
3928         int i, j;
3929         int err;
3930
3931         i = 0;
3932 start_again:
3933         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3934         if (err)
3935                 return err;
3936
3937         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3938                           &devlink_nl_family, NLM_F_MULTI, cmd);
3939         if (!hdr) {
3940                 nlmsg_free(skb);
3941                 return -EMSGSIZE;
3942         }
3943
3944         if (devlink_nl_put_handle(skb, devlink))
3945                 goto nla_put_failure;
3946         headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
3947         if (!headers_attr)
3948                 goto nla_put_failure;
3949
3950         j = 0;
3951         for (; i < dpipe_headers->headers_count; i++) {
3952                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
3953                 if (err) {
3954                         if (!j)
3955                                 goto err_table_put;
3956                         break;
3957                 }
3958                 j++;
3959         }
3960         nla_nest_end(skb, headers_attr);
3961         genlmsg_end(skb, hdr);
3962         if (i != dpipe_headers->headers_count)
3963                 goto start_again;
3964
3965 send_done:
3966         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3967                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3968         if (!nlh) {
3969                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3970                 if (err)
3971                         return err;
3972                 goto send_done;
3973         }
3974         return genlmsg_reply(skb, info);
3975
3976 nla_put_failure:
3977         err = -EMSGSIZE;
3978 err_table_put:
3979         nlmsg_free(skb);
3980         return err;
3981 }
3982
3983 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
3984                                             struct genl_info *info)
3985 {
3986         struct devlink *devlink = info->user_ptr[0];
3987
3988         if (!devlink->dpipe_headers)
3989                 return -EOPNOTSUPP;
3990         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
3991                                           0, devlink->dpipe_headers);
3992 }
3993
3994 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
3995                                             const char *table_name,
3996                                             bool enable)
3997 {
3998         struct devlink_dpipe_table *table;
3999
4000         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
4001                                          table_name, devlink);
4002         if (!table)
4003                 return -EINVAL;
4004
4005         if (table->counter_control_extern)
4006                 return -EOPNOTSUPP;
4007
4008         if (!(table->counters_enabled ^ enable))
4009                 return 0;
4010
4011         table->counters_enabled = enable;
4012         if (table->table_ops->counters_set_update)
4013                 table->table_ops->counters_set_update(table->priv, enable);
4014         return 0;
4015 }
4016
4017 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
4018                                                    struct genl_info *info)
4019 {
4020         struct devlink *devlink = info->user_ptr[0];
4021         const char *table_name;
4022         bool counters_enable;
4023
4024         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
4025             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
4026                 return -EINVAL;
4027
4028         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
4029         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
4030
4031         return devlink_dpipe_table_counters_set(devlink, table_name,
4032                                                 counters_enable);
4033 }
4034
4035 static struct devlink_resource *
4036 devlink_resource_find(struct devlink *devlink,
4037                       struct devlink_resource *resource, u64 resource_id)
4038 {
4039         struct list_head *resource_list;
4040
4041         if (resource)
4042                 resource_list = &resource->resource_list;
4043         else
4044                 resource_list = &devlink->resource_list;
4045
4046         list_for_each_entry(resource, resource_list, list) {
4047                 struct devlink_resource *child_resource;
4048
4049                 if (resource->id == resource_id)
4050                         return resource;
4051
4052                 child_resource = devlink_resource_find(devlink, resource,
4053                                                        resource_id);
4054                 if (child_resource)
4055                         return child_resource;
4056         }
4057         return NULL;
4058 }
4059
4060 static void
4061 devlink_resource_validate_children(struct devlink_resource *resource)
4062 {
4063         struct devlink_resource *child_resource;
4064         bool size_valid = true;
4065         u64 parts_size = 0;
4066
4067         if (list_empty(&resource->resource_list))
4068                 goto out;
4069
4070         list_for_each_entry(child_resource, &resource->resource_list, list)
4071                 parts_size += child_resource->size_new;
4072
4073         if (parts_size > resource->size_new)
4074                 size_valid = false;
4075 out:
4076         resource->size_valid = size_valid;
4077 }
4078
4079 static int
4080 devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
4081                                struct netlink_ext_ack *extack)
4082 {
4083         u64 reminder;
4084         int err = 0;
4085
4086         if (size > resource->size_params.size_max) {
4087                 NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
4088                 err = -EINVAL;
4089         }
4090
4091         if (size < resource->size_params.size_min) {
4092                 NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
4093                 err = -EINVAL;
4094         }
4095
4096         div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
4097         if (reminder) {
4098                 NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
4099                 err = -EINVAL;
4100         }
4101
4102         return err;
4103 }
4104
4105 static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
4106                                        struct genl_info *info)
4107 {
4108         struct devlink *devlink = info->user_ptr[0];
4109         struct devlink_resource *resource;
4110         u64 resource_id;
4111         u64 size;
4112         int err;
4113
4114         if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
4115             !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
4116                 return -EINVAL;
4117         resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
4118
4119         resource = devlink_resource_find(devlink, NULL, resource_id);
4120         if (!resource)
4121                 return -EINVAL;
4122
4123         size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
4124         err = devlink_resource_validate_size(resource, size, info->extack);
4125         if (err)
4126                 return err;
4127
4128         resource->size_new = size;
4129         devlink_resource_validate_children(resource);
4130         if (resource->parent)
4131                 devlink_resource_validate_children(resource->parent);
4132         return 0;
4133 }
4134
4135 static int
4136 devlink_resource_size_params_put(struct devlink_resource *resource,
4137                                  struct sk_buff *skb)
4138 {
4139         struct devlink_resource_size_params *size_params;
4140
4141         size_params = &resource->size_params;
4142         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
4143                               size_params->size_granularity, DEVLINK_ATTR_PAD) ||
4144             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
4145                               size_params->size_max, DEVLINK_ATTR_PAD) ||
4146             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
4147                               size_params->size_min, DEVLINK_ATTR_PAD) ||
4148             nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
4149                 return -EMSGSIZE;
4150         return 0;
4151 }
4152
4153 static int devlink_resource_occ_put(struct devlink_resource *resource,
4154                                     struct sk_buff *skb)
4155 {
4156         if (!resource->occ_get)
4157                 return 0;
4158         return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
4159                                  resource->occ_get(resource->occ_get_priv),
4160                                  DEVLINK_ATTR_PAD);
4161 }
4162
4163 static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
4164                                 struct devlink_resource *resource)
4165 {
4166         struct devlink_resource *child_resource;
4167         struct nlattr *child_resource_attr;
4168         struct nlattr *resource_attr;
4169
4170         resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
4171         if (!resource_attr)
4172                 return -EMSGSIZE;
4173
4174         if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
4175             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
4176                               DEVLINK_ATTR_PAD) ||
4177             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
4178                               DEVLINK_ATTR_PAD))
4179                 goto nla_put_failure;
4180         if (resource->size != resource->size_new)
4181                 nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
4182                                   resource->size_new, DEVLINK_ATTR_PAD);
4183         if (devlink_resource_occ_put(resource, skb))
4184                 goto nla_put_failure;
4185         if (devlink_resource_size_params_put(resource, skb))
4186                 goto nla_put_failure;
4187         if (list_empty(&resource->resource_list))
4188                 goto out;
4189
4190         if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
4191                        resource->size_valid))
4192                 goto nla_put_failure;
4193
4194         child_resource_attr = nla_nest_start_noflag(skb,
4195                                                     DEVLINK_ATTR_RESOURCE_LIST);
4196         if (!child_resource_attr)
4197                 goto nla_put_failure;
4198
4199         list_for_each_entry(child_resource, &resource->resource_list, list) {
4200                 if (devlink_resource_put(devlink, skb, child_resource))
4201                         goto resource_put_failure;
4202         }
4203
4204         nla_nest_end(skb, child_resource_attr);
4205 out:
4206         nla_nest_end(skb, resource_attr);
4207         return 0;
4208
4209 resource_put_failure:
4210         nla_nest_cancel(skb, child_resource_attr);
4211 nla_put_failure:
4212         nla_nest_cancel(skb, resource_attr);
4213         return -EMSGSIZE;
4214 }
4215
4216 static int devlink_resource_fill(struct genl_info *info,
4217                                  enum devlink_command cmd, int flags)
4218 {
4219         struct devlink *devlink = info->user_ptr[0];
4220         struct devlink_resource *resource;
4221         struct nlattr *resources_attr;
4222         struct sk_buff *skb = NULL;
4223         struct nlmsghdr *nlh;
4224         bool incomplete;
4225         void *hdr;
4226         int i;
4227         int err;
4228
4229         resource = list_first_entry(&devlink->resource_list,
4230                                     struct devlink_resource, list);
4231 start_again:
4232         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
4233         if (err)
4234                 return err;
4235
4236         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
4237                           &devlink_nl_family, NLM_F_MULTI, cmd);
4238         if (!hdr) {
4239                 nlmsg_free(skb);
4240                 return -EMSGSIZE;
4241         }
4242
4243         if (devlink_nl_put_handle(skb, devlink))
4244                 goto nla_put_failure;
4245
4246         resources_attr = nla_nest_start_noflag(skb,
4247                                                DEVLINK_ATTR_RESOURCE_LIST);
4248         if (!resources_attr)
4249                 goto nla_put_failure;
4250
4251         incomplete = false;
4252         i = 0;
4253         list_for_each_entry_from(resource, &devlink->resource_list, list) {
4254                 err = devlink_resource_put(devlink, skb, resource);
4255                 if (err) {
4256                         if (!i)
4257                                 goto err_resource_put;
4258                         incomplete = true;
4259                         break;
4260                 }
4261                 i++;
4262         }
4263         nla_nest_end(skb, resources_attr);
4264         genlmsg_end(skb, hdr);
4265         if (incomplete)
4266                 goto start_again;
4267 send_done:
4268         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
4269                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
4270         if (!nlh) {
4271                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
4272                 if (err)
4273                         return err;
4274                 goto send_done;
4275         }
4276         return genlmsg_reply(skb, info);
4277
4278 nla_put_failure:
4279         err = -EMSGSIZE;
4280 err_resource_put:
4281         nlmsg_free(skb);
4282         return err;
4283 }
4284
4285 static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
4286                                         struct genl_info *info)
4287 {
4288         struct devlink *devlink = info->user_ptr[0];
4289
4290         if (list_empty(&devlink->resource_list))
4291                 return -EOPNOTSUPP;
4292
4293         return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
4294 }
4295
4296 static int
4297 devlink_resources_validate(struct devlink *devlink,
4298                            struct devlink_resource *resource,
4299                            struct genl_info *info)
4300 {
4301         struct list_head *resource_list;
4302         int err = 0;
4303
4304         if (resource)
4305                 resource_list = &resource->resource_list;
4306         else
4307                 resource_list = &devlink->resource_list;
4308
4309         list_for_each_entry(resource, resource_list, list) {
4310                 if (!resource->size_valid)
4311                         return -EINVAL;
4312                 err = devlink_resources_validate(devlink, resource, info);
4313                 if (err)
4314                         return err;
4315         }
4316         return err;
4317 }
4318
4319 static struct net *devlink_netns_get(struct sk_buff *skb,
4320                                      struct genl_info *info)
4321 {
4322         struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
4323         struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
4324         struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
4325         struct net *net;
4326
4327         if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
4328                 NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
4329                 return ERR_PTR(-EINVAL);
4330         }
4331
4332         if (netns_pid_attr) {
4333                 net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
4334         } else if (netns_fd_attr) {
4335                 net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
4336         } else if (netns_id_attr) {
4337                 net = get_net_ns_by_id(sock_net(skb->sk),
4338                                        nla_get_u32(netns_id_attr));
4339                 if (!net)
4340                         net = ERR_PTR(-EINVAL);
4341         } else {
4342                 WARN_ON(1);
4343                 net = ERR_PTR(-EINVAL);
4344         }
4345         if (IS_ERR(net)) {
4346                 NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
4347                 return ERR_PTR(-EINVAL);
4348         }
4349         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
4350                 put_net(net);
4351                 return ERR_PTR(-EPERM);
4352         }
4353         return net;
4354 }
4355
4356 static void devlink_param_notify(struct devlink *devlink,
4357                                  unsigned int port_index,
4358                                  struct devlink_param_item *param_item,
4359                                  enum devlink_command cmd);
4360
4361 static void devlink_ns_change_notify(struct devlink *devlink,
4362                                      struct net *dest_net, struct net *curr_net,
4363                                      bool new)
4364 {
4365         struct devlink_param_item *param_item;
4366         enum devlink_command cmd;
4367
4368         /* Userspace needs to be notified about devlink objects
4369          * removed from original and entering new network namespace.
4370          * The rest of the devlink objects are re-created during
4371          * reload process so the notifications are generated separatelly.
4372          */
4373
4374         if (!dest_net || net_eq(dest_net, curr_net))
4375                 return;
4376
4377         if (new)
4378                 devlink_notify(devlink, DEVLINK_CMD_NEW);
4379
4380         cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
4381         list_for_each_entry(param_item, &devlink->param_list, list)
4382                 devlink_param_notify(devlink, 0, param_item, cmd);
4383
4384         if (!new)
4385                 devlink_notify(devlink, DEVLINK_CMD_DEL);
4386 }
4387
4388 static bool devlink_reload_supported(const struct devlink_ops *ops)
4389 {
4390         return ops->reload_down && ops->reload_up;
4391 }
4392
4393 static void devlink_reload_failed_set(struct devlink *devlink,
4394                                       bool reload_failed)
4395 {
4396         if (devlink->reload_failed == reload_failed)
4397                 return;
4398         devlink->reload_failed = reload_failed;
4399         devlink_notify(devlink, DEVLINK_CMD_NEW);
4400 }
4401
4402 bool devlink_is_reload_failed(const struct devlink *devlink)
4403 {
4404         return devlink->reload_failed;
4405 }
4406 EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
4407
4408 static void
4409 __devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
4410                               enum devlink_reload_limit limit, u32 actions_performed)
4411 {
4412         unsigned long actions = actions_performed;
4413         int stat_idx;
4414         int action;
4415
4416         for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
4417                 stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
4418                 reload_stats[stat_idx]++;
4419         }
4420         devlink_notify(devlink, DEVLINK_CMD_NEW);
4421 }
4422
4423 static void
4424 devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
4425                             u32 actions_performed)
4426 {
4427         __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
4428                                       actions_performed);
4429 }
4430
4431 /**
4432  *      devlink_remote_reload_actions_performed - Update devlink on reload actions
4433  *        performed which are not a direct result of devlink reload call.
4434  *
4435  *      This should be called by a driver after performing reload actions in case it was not
4436  *      a result of devlink reload call. For example fw_activate was performed as a result
4437  *      of devlink reload triggered fw_activate on another host.
4438  *      The motivation for this function is to keep data on reload actions performed on this
4439  *      function whether it was done due to direct devlink reload call or not.
4440  *
4441  *      @devlink: devlink
4442  *      @limit: reload limit
4443  *      @actions_performed: bitmask of actions performed
4444  */
4445 void devlink_remote_reload_actions_performed(struct devlink *devlink,
4446                                              enum devlink_reload_limit limit,
4447                                              u32 actions_performed)
4448 {
4449         if (WARN_ON(!actions_performed ||
4450                     actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
4451                     actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
4452                     limit > DEVLINK_RELOAD_LIMIT_MAX))
4453                 return;
4454
4455         __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
4456                                       actions_performed);
4457 }
4458 EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
4459
4460 static int devlink_reload(struct devlink *devlink, struct net *dest_net,
4461                           enum devlink_reload_action action, enum devlink_reload_limit limit,
4462                           u32 *actions_performed, struct netlink_ext_ack *extack)
4463 {
4464         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
4465         struct net *curr_net;
4466         int err;
4467
4468         memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
4469                sizeof(remote_reload_stats));
4470
4471         curr_net = devlink_net(devlink);
4472         devlink_ns_change_notify(devlink, dest_net, curr_net, false);
4473         err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
4474         if (err)
4475                 return err;
4476
4477         if (dest_net && !net_eq(dest_net, curr_net))
4478                 write_pnet(&devlink->_net, dest_net);
4479
4480         err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
4481         devlink_reload_failed_set(devlink, !!err);
4482         if (err)
4483                 return err;
4484
4485         devlink_ns_change_notify(devlink, dest_net, curr_net, true);
4486         WARN_ON(!(*actions_performed & BIT(action)));
4487         /* Catch driver on updating the remote action within devlink reload */
4488         WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
4489                        sizeof(remote_reload_stats)));
4490         devlink_reload_stats_update(devlink, limit, *actions_performed);
4491         return 0;
4492 }
4493
4494 static int
4495 devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
4496                                         enum devlink_command cmd, struct genl_info *info)
4497 {
4498         struct sk_buff *msg;
4499         void *hdr;
4500
4501         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4502         if (!msg)
4503                 return -ENOMEM;
4504
4505         hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
4506         if (!hdr)
4507                 goto free_msg;
4508
4509         if (devlink_nl_put_handle(msg, devlink))
4510                 goto nla_put_failure;
4511
4512         if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
4513                                actions_performed))
4514                 goto nla_put_failure;
4515         genlmsg_end(msg, hdr);
4516
4517         return genlmsg_reply(msg, info);
4518
4519 nla_put_failure:
4520         genlmsg_cancel(msg, hdr);
4521 free_msg:
4522         nlmsg_free(msg);
4523         return -EMSGSIZE;
4524 }
4525
4526 static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
4527 {
4528         struct devlink *devlink = info->user_ptr[0];
4529         enum devlink_reload_action action;
4530         enum devlink_reload_limit limit;
4531         struct net *dest_net = NULL;
4532         u32 actions_performed;
4533         int err;
4534
4535         if (!(devlink->features & DEVLINK_F_RELOAD))
4536                 return -EOPNOTSUPP;
4537
4538         err = devlink_resources_validate(devlink, NULL, info);
4539         if (err) {
4540                 NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
4541                 return err;
4542         }
4543
4544         if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
4545                 action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
4546         else
4547                 action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
4548
4549         if (!devlink_reload_action_is_supported(devlink, action)) {
4550                 NL_SET_ERR_MSG_MOD(info->extack,
4551                                    "Requested reload action is not supported by the driver");
4552                 return -EOPNOTSUPP;
4553         }
4554
4555         limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
4556         if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
4557                 struct nla_bitfield32 limits;
4558                 u32 limits_selected;
4559
4560                 limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
4561                 limits_selected = limits.value & limits.selector;
4562                 if (!limits_selected) {
4563                         NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
4564                         return -EINVAL;
4565                 }
4566                 for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
4567                         if (limits_selected & BIT(limit))
4568                                 break;
4569                 /* UAPI enables multiselection, but currently it is not used */
4570                 if (limits_selected != BIT(limit)) {
4571                         NL_SET_ERR_MSG_MOD(info->extack,
4572                                            "Multiselection of limit is not supported");
4573                         return -EOPNOTSUPP;
4574                 }
4575                 if (!devlink_reload_limit_is_supported(devlink, limit)) {
4576                         NL_SET_ERR_MSG_MOD(info->extack,
4577                                            "Requested limit is not supported by the driver");
4578                         return -EOPNOTSUPP;
4579                 }
4580                 if (devlink_reload_combination_is_invalid(action, limit)) {
4581                         NL_SET_ERR_MSG_MOD(info->extack,
4582                                            "Requested limit is invalid for this action");
4583                         return -EINVAL;
4584                 }
4585         }
4586         if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
4587             info->attrs[DEVLINK_ATTR_NETNS_FD] ||
4588             info->attrs[DEVLINK_ATTR_NETNS_ID]) {
4589                 dest_net = devlink_netns_get(skb, info);
4590                 if (IS_ERR(dest_net))
4591                         return PTR_ERR(dest_net);
4592         }
4593
4594         err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
4595
4596         if (dest_net)
4597                 put_net(dest_net);
4598
4599         if (err)
4600                 return err;
4601         /* For backward compatibility generate reply only if attributes used by user */
4602         if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
4603                 return 0;
4604
4605         return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
4606                                                        DEVLINK_CMD_RELOAD, info);
4607 }
4608
4609 static int devlink_nl_flash_update_fill(struct sk_buff *msg,
4610                                         struct devlink *devlink,
4611                                         enum devlink_command cmd,
4612                                         struct devlink_flash_notify *params)
4613 {
4614         void *hdr;
4615
4616         hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
4617         if (!hdr)
4618                 return -EMSGSIZE;
4619
4620         if (devlink_nl_put_handle(msg, devlink))
4621                 goto nla_put_failure;
4622
4623         if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
4624                 goto out;
4625
4626         if (params->status_msg &&
4627             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
4628                            params->status_msg))
4629                 goto nla_put_failure;
4630         if (params->component &&
4631             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
4632                            params->component))
4633                 goto nla_put_failure;
4634         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
4635                               params->done, DEVLINK_ATTR_PAD))
4636                 goto nla_put_failure;
4637         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
4638                               params->total, DEVLINK_ATTR_PAD))
4639                 goto nla_put_failure;
4640         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
4641                               params->timeout, DEVLINK_ATTR_PAD))
4642                 goto nla_put_failure;
4643
4644 out:
4645         genlmsg_end(msg, hdr);
4646         return 0;
4647
4648 nla_put_failure:
4649         genlmsg_cancel(msg, hdr);
4650         return -EMSGSIZE;
4651 }
4652
4653 static void __devlink_flash_update_notify(struct devlink *devlink,
4654                                           enum devlink_command cmd,
4655                                           struct devlink_flash_notify *params)
4656 {
4657         struct sk_buff *msg;
4658         int err;
4659
4660         WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
4661                 cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
4662                 cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
4663
4664         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
4665                 return;
4666
4667         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4668         if (!msg)
4669                 return;
4670
4671         err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
4672         if (err)
4673                 goto out_free_msg;
4674
4675         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4676                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4677         return;
4678
4679 out_free_msg:
4680         nlmsg_free(msg);
4681 }
4682
4683 static void devlink_flash_update_begin_notify(struct devlink *devlink)
4684 {
4685         struct devlink_flash_notify params = {};
4686
4687         __devlink_flash_update_notify(devlink,
4688                                       DEVLINK_CMD_FLASH_UPDATE,
4689                                       &params);
4690 }
4691
4692 static void devlink_flash_update_end_notify(struct devlink *devlink)
4693 {
4694         struct devlink_flash_notify params = {};
4695
4696         __devlink_flash_update_notify(devlink,
4697                                       DEVLINK_CMD_FLASH_UPDATE_END,
4698                                       &params);
4699 }
4700
4701 void devlink_flash_update_status_notify(struct devlink *devlink,
4702                                         const char *status_msg,
4703                                         const char *component,
4704                                         unsigned long done,
4705                                         unsigned long total)
4706 {
4707         struct devlink_flash_notify params = {
4708                 .status_msg = status_msg,
4709                 .component = component,
4710                 .done = done,
4711                 .total = total,
4712         };
4713
4714         __devlink_flash_update_notify(devlink,
4715                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4716                                       &params);
4717 }
4718 EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
4719
4720 void devlink_flash_update_timeout_notify(struct devlink *devlink,
4721                                          const char *status_msg,
4722                                          const char *component,
4723                                          unsigned long timeout)
4724 {
4725         struct devlink_flash_notify params = {
4726                 .status_msg = status_msg,
4727                 .component = component,
4728                 .timeout = timeout,
4729         };
4730
4731         __devlink_flash_update_notify(devlink,
4732                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4733                                       &params);
4734 }
4735 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
4736
4737 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
4738                                        struct genl_info *info)
4739 {
4740         struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name;
4741         struct devlink_flash_update_params params = {};
4742         struct devlink *devlink = info->user_ptr[0];
4743         const char *file_name;
4744         u32 supported_params;
4745         int ret;
4746
4747         if (!devlink->ops->flash_update)
4748                 return -EOPNOTSUPP;
4749
4750         if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
4751                 return -EINVAL;
4752
4753         supported_params = devlink->ops->supported_flash_update_params;
4754
4755         nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
4756         if (nla_component) {
4757                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
4758                         NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
4759                                             "component update is not supported by this device");
4760                         return -EOPNOTSUPP;
4761                 }
4762                 params.component = nla_data(nla_component);
4763         }
4764
4765         nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
4766         if (nla_overwrite_mask) {
4767                 struct nla_bitfield32 sections;
4768
4769                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
4770                         NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
4771                                             "overwrite settings are not supported by this device");
4772                         return -EOPNOTSUPP;
4773                 }
4774                 sections = nla_get_bitfield32(nla_overwrite_mask);
4775                 params.overwrite_mask = sections.value & sections.selector;
4776         }
4777
4778         nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
4779         file_name = nla_data(nla_file_name);
4780         ret = request_firmware(&params.fw, file_name, devlink->dev);
4781         if (ret) {
4782                 NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
4783                 return ret;
4784         }
4785
4786         devlink_flash_update_begin_notify(devlink);
4787         ret = devlink->ops->flash_update(devlink, &params, info->extack);
4788         devlink_flash_update_end_notify(devlink);
4789
4790         release_firmware(params.fw);
4791
4792         return ret;
4793 }
4794
4795 static const struct devlink_param devlink_param_generic[] = {
4796         {
4797                 .id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
4798                 .name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
4799                 .type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
4800         },
4801         {
4802                 .id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
4803                 .name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
4804                 .type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
4805         },
4806         {
4807                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
4808                 .name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
4809                 .type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
4810         },
4811         {
4812                 .id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
4813                 .name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
4814                 .type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
4815         },
4816         {
4817                 .id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
4818                 .name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
4819                 .type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
4820         },
4821         {
4822                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
4823                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
4824                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
4825         },
4826         {
4827                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
4828                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
4829                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
4830         },
4831         {
4832                 .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
4833                 .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
4834                 .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
4835         },
4836         {
4837                 .id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
4838                 .name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
4839                 .type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
4840         },
4841         {
4842                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
4843                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
4844                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
4845         },
4846         {
4847                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
4848                 .name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
4849                 .type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
4850         },
4851         {
4852                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
4853                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
4854                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
4855         },
4856         {
4857                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
4858                 .name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
4859                 .type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
4860         },
4861         {
4862                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
4863                 .name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
4864                 .type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
4865         },
4866         {
4867                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
4868                 .name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME,
4869                 .type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE,
4870         },
4871         {
4872                 .id = DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
4873                 .name = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_NAME,
4874                 .type = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_TYPE,
4875         },
4876         {
4877                 .id = DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
4878                 .name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
4879                 .type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
4880         },
4881 };
4882
4883 static int devlink_param_generic_verify(const struct devlink_param *param)
4884 {
4885         /* verify it match generic parameter by id and name */
4886         if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
4887                 return -EINVAL;
4888         if (strcmp(param->name, devlink_param_generic[param->id].name))
4889                 return -ENOENT;
4890
4891         WARN_ON(param->type != devlink_param_generic[param->id].type);
4892
4893         return 0;
4894 }
4895
4896 static int devlink_param_driver_verify(const struct devlink_param *param)
4897 {
4898         int i;
4899
4900         if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
4901                 return -EINVAL;
4902         /* verify no such name in generic params */
4903         for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
4904                 if (!strcmp(param->name, devlink_param_generic[i].name))
4905                         return -EEXIST;
4906
4907         return 0;
4908 }
4909
4910 static struct devlink_param_item *
4911 devlink_param_find_by_name(struct list_head *param_list,
4912                            const char *param_name)
4913 {
4914         struct devlink_param_item *param_item;
4915
4916         list_for_each_entry(param_item, param_list, list)
4917                 if (!strcmp(param_item->param->name, param_name))
4918                         return param_item;
4919         return NULL;
4920 }
4921
4922 static struct devlink_param_item *
4923 devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
4924 {
4925         struct devlink_param_item *param_item;
4926
4927         list_for_each_entry(param_item, param_list, list)
4928                 if (param_item->param->id == param_id)
4929                         return param_item;
4930         return NULL;
4931 }
4932
4933 static bool
4934 devlink_param_cmode_is_supported(const struct devlink_param *param,
4935                                  enum devlink_param_cmode cmode)
4936 {
4937         return test_bit(cmode, &param->supported_cmodes);
4938 }
4939
4940 static int devlink_param_get(struct devlink *devlink,
4941                              const struct devlink_param *param,
4942                              struct devlink_param_gset_ctx *ctx)
4943 {
4944         if (!param->get)
4945                 return -EOPNOTSUPP;
4946         return param->get(devlink, param->id, ctx);
4947 }
4948
4949 static int devlink_param_set(struct devlink *devlink,
4950                              const struct devlink_param *param,
4951                              struct devlink_param_gset_ctx *ctx)
4952 {
4953         if (!param->set)
4954                 return -EOPNOTSUPP;
4955         return param->set(devlink, param->id, ctx);
4956 }
4957
4958 static int
4959 devlink_param_type_to_nla_type(enum devlink_param_type param_type)
4960 {
4961         switch (param_type) {
4962         case DEVLINK_PARAM_TYPE_U8:
4963                 return NLA_U8;
4964         case DEVLINK_PARAM_TYPE_U16:
4965                 return NLA_U16;
4966         case DEVLINK_PARAM_TYPE_U32:
4967                 return NLA_U32;
4968         case DEVLINK_PARAM_TYPE_STRING:
4969                 return NLA_STRING;
4970         case DEVLINK_PARAM_TYPE_BOOL:
4971                 return NLA_FLAG;
4972         default:
4973                 return -EINVAL;
4974         }
4975 }
4976
4977 static int
4978 devlink_nl_param_value_fill_one(struct sk_buff *msg,
4979                                 enum devlink_param_type type,
4980                                 enum devlink_param_cmode cmode,
4981                                 union devlink_param_value val)
4982 {
4983         struct nlattr *param_value_attr;
4984
4985         param_value_attr = nla_nest_start_noflag(msg,
4986                                                  DEVLINK_ATTR_PARAM_VALUE);
4987         if (!param_value_attr)
4988                 goto nla_put_failure;
4989
4990         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
4991                 goto value_nest_cancel;
4992
4993         switch (type) {
4994         case DEVLINK_PARAM_TYPE_U8:
4995                 if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
4996                         goto value_nest_cancel;
4997                 break;
4998         case DEVLINK_PARAM_TYPE_U16:
4999                 if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
5000                         goto value_nest_cancel;
5001                 break;
5002         case DEVLINK_PARAM_TYPE_U32:
5003                 if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
5004                         goto value_nest_cancel;
5005                 break;
5006         case DEVLINK_PARAM_TYPE_STRING:
5007                 if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
5008                                    val.vstr))
5009                         goto value_nest_cancel;
5010                 break;
5011         case DEVLINK_PARAM_TYPE_BOOL:
5012                 if (val.vbool &&
5013                     nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
5014                         goto value_nest_cancel;
5015                 break;
5016         }
5017
5018         nla_nest_end(msg, param_value_attr);
5019         return 0;
5020
5021 value_nest_cancel:
5022         nla_nest_cancel(msg, param_value_attr);
5023 nla_put_failure:
5024         return -EMSGSIZE;
5025 }
5026
5027 static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
5028                                  unsigned int port_index,
5029                                  struct devlink_param_item *param_item,
5030                                  enum devlink_command cmd,
5031                                  u32 portid, u32 seq, int flags)
5032 {
5033         union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
5034         bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
5035         const struct devlink_param *param = param_item->param;
5036         struct devlink_param_gset_ctx ctx;
5037         struct nlattr *param_values_list;
5038         struct nlattr *param_attr;
5039         int nla_type;
5040         void *hdr;
5041         int err;
5042         int i;
5043
5044         /* Get value from driver part to driverinit configuration mode */
5045         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
5046                 if (!devlink_param_cmode_is_supported(param, i))
5047                         continue;
5048                 if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
5049                         if (!param_item->driverinit_value_valid)
5050                                 return -EOPNOTSUPP;
5051                         param_value[i] = param_item->driverinit_value;
5052                 } else {
5053                         ctx.cmode = i;
5054                         err = devlink_param_get(devlink, param, &ctx);
5055                         if (err)
5056                                 return err;
5057                         param_value[i] = ctx.val;
5058                 }
5059                 param_value_set[i] = true;
5060         }
5061
5062         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5063         if (!hdr)
5064                 return -EMSGSIZE;
5065
5066         if (devlink_nl_put_handle(msg, devlink))
5067                 goto genlmsg_cancel;
5068
5069         if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
5070             cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
5071             cmd == DEVLINK_CMD_PORT_PARAM_DEL)
5072                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
5073                         goto genlmsg_cancel;
5074
5075         param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
5076         if (!param_attr)
5077                 goto genlmsg_cancel;
5078         if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
5079                 goto param_nest_cancel;
5080         if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
5081                 goto param_nest_cancel;
5082
5083         nla_type = devlink_param_type_to_nla_type(param->type);
5084         if (nla_type < 0)
5085                 goto param_nest_cancel;
5086         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
5087                 goto param_nest_cancel;
5088
5089         param_values_list = nla_nest_start_noflag(msg,
5090                                                   DEVLINK_ATTR_PARAM_VALUES_LIST);
5091         if (!param_values_list)
5092                 goto param_nest_cancel;
5093
5094         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
5095                 if (!param_value_set[i])
5096                         continue;
5097                 err = devlink_nl_param_value_fill_one(msg, param->type,
5098                                                       i, param_value[i]);
5099                 if (err)
5100                         goto values_list_nest_cancel;
5101         }
5102
5103         nla_nest_end(msg, param_values_list);
5104         nla_nest_end(msg, param_attr);
5105         genlmsg_end(msg, hdr);
5106         return 0;
5107
5108 values_list_nest_cancel:
5109         nla_nest_end(msg, param_values_list);
5110 param_nest_cancel:
5111         nla_nest_cancel(msg, param_attr);
5112 genlmsg_cancel:
5113         genlmsg_cancel(msg, hdr);
5114         return -EMSGSIZE;
5115 }
5116
5117 static void devlink_param_notify(struct devlink *devlink,
5118                                  unsigned int port_index,
5119                                  struct devlink_param_item *param_item,
5120                                  enum devlink_command cmd)
5121 {
5122         struct sk_buff *msg;
5123         int err;
5124
5125         WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
5126                 cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
5127                 cmd != DEVLINK_CMD_PORT_PARAM_DEL);
5128         ASSERT_DEVLINK_REGISTERED(devlink);
5129
5130         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5131         if (!msg)
5132                 return;
5133         err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
5134                                     0, 0, 0);
5135         if (err) {
5136                 nlmsg_free(msg);
5137                 return;
5138         }
5139
5140         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
5141                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5142 }
5143
5144 static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
5145                                            struct netlink_callback *cb)
5146 {
5147         struct devlink_param_item *param_item;
5148         struct devlink *devlink;
5149         int start = cb->args[0];
5150         unsigned long index;
5151         int idx = 0;
5152         int err = 0;
5153
5154         mutex_lock(&devlink_mutex);
5155         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5156                 if (!devlink_try_get(devlink))
5157                         continue;
5158
5159                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5160                         goto retry;
5161
5162                 devl_lock(devlink);
5163                 list_for_each_entry(param_item, &devlink->param_list, list) {
5164                         if (idx < start) {
5165                                 idx++;
5166                                 continue;
5167                         }
5168                         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
5169                                                     DEVLINK_CMD_PARAM_GET,
5170                                                     NETLINK_CB(cb->skb).portid,
5171                                                     cb->nlh->nlmsg_seq,
5172                                                     NLM_F_MULTI);
5173                         if (err == -EOPNOTSUPP) {
5174                                 err = 0;
5175                         } else if (err) {
5176                                 devl_unlock(devlink);
5177                                 devlink_put(devlink);
5178                                 goto out;
5179                         }
5180                         idx++;
5181                 }
5182                 devl_unlock(devlink);
5183 retry:
5184                 devlink_put(devlink);
5185         }
5186 out:
5187         mutex_unlock(&devlink_mutex);
5188
5189         if (err != -EMSGSIZE)
5190                 return err;
5191
5192         cb->args[0] = idx;
5193         return msg->len;
5194 }
5195
5196 static int
5197 devlink_param_type_get_from_info(struct genl_info *info,
5198                                  enum devlink_param_type *param_type)
5199 {
5200         if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
5201                 return -EINVAL;
5202
5203         switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
5204         case NLA_U8:
5205                 *param_type = DEVLINK_PARAM_TYPE_U8;
5206                 break;
5207         case NLA_U16:
5208                 *param_type = DEVLINK_PARAM_TYPE_U16;
5209                 break;
5210         case NLA_U32:
5211                 *param_type = DEVLINK_PARAM_TYPE_U32;
5212                 break;
5213         case NLA_STRING:
5214                 *param_type = DEVLINK_PARAM_TYPE_STRING;
5215                 break;
5216         case NLA_FLAG:
5217                 *param_type = DEVLINK_PARAM_TYPE_BOOL;
5218                 break;
5219         default:
5220                 return -EINVAL;
5221         }
5222
5223         return 0;
5224 }
5225
5226 static int
5227 devlink_param_value_get_from_info(const struct devlink_param *param,
5228                                   struct genl_info *info,
5229                                   union devlink_param_value *value)
5230 {
5231         struct nlattr *param_data;
5232         int len;
5233
5234         param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
5235
5236         if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
5237                 return -EINVAL;
5238
5239         switch (param->type) {
5240         case DEVLINK_PARAM_TYPE_U8:
5241                 if (nla_len(param_data) != sizeof(u8))
5242                         return -EINVAL;
5243                 value->vu8 = nla_get_u8(param_data);
5244                 break;
5245         case DEVLINK_PARAM_TYPE_U16:
5246                 if (nla_len(param_data) != sizeof(u16))
5247                         return -EINVAL;
5248                 value->vu16 = nla_get_u16(param_data);
5249                 break;
5250         case DEVLINK_PARAM_TYPE_U32:
5251                 if (nla_len(param_data) != sizeof(u32))
5252                         return -EINVAL;
5253                 value->vu32 = nla_get_u32(param_data);
5254                 break;
5255         case DEVLINK_PARAM_TYPE_STRING:
5256                 len = strnlen(nla_data(param_data), nla_len(param_data));
5257                 if (len == nla_len(param_data) ||
5258                     len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
5259                         return -EINVAL;
5260                 strcpy(value->vstr, nla_data(param_data));
5261                 break;
5262         case DEVLINK_PARAM_TYPE_BOOL:
5263                 if (param_data && nla_len(param_data))
5264                         return -EINVAL;
5265                 value->vbool = nla_get_flag(param_data);
5266                 break;
5267         }
5268         return 0;
5269 }
5270
5271 static struct devlink_param_item *
5272 devlink_param_get_from_info(struct list_head *param_list,
5273                             struct genl_info *info)
5274 {
5275         char *param_name;
5276
5277         if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
5278                 return NULL;
5279
5280         param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
5281         return devlink_param_find_by_name(param_list, param_name);
5282 }
5283
5284 static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
5285                                          struct genl_info *info)
5286 {
5287         struct devlink *devlink = info->user_ptr[0];
5288         struct devlink_param_item *param_item;
5289         struct sk_buff *msg;
5290         int err;
5291
5292         param_item = devlink_param_get_from_info(&devlink->param_list, info);
5293         if (!param_item)
5294                 return -EINVAL;
5295
5296         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5297         if (!msg)
5298                 return -ENOMEM;
5299
5300         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
5301                                     DEVLINK_CMD_PARAM_GET,
5302                                     info->snd_portid, info->snd_seq, 0);
5303         if (err) {
5304                 nlmsg_free(msg);
5305                 return err;
5306         }
5307
5308         return genlmsg_reply(msg, info);
5309 }
5310
5311 static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
5312                                            unsigned int port_index,
5313                                            struct list_head *param_list,
5314                                            struct genl_info *info,
5315                                            enum devlink_command cmd)
5316 {
5317         enum devlink_param_type param_type;
5318         struct devlink_param_gset_ctx ctx;
5319         enum devlink_param_cmode cmode;
5320         struct devlink_param_item *param_item;
5321         const struct devlink_param *param;
5322         union devlink_param_value value;
5323         int err = 0;
5324
5325         param_item = devlink_param_get_from_info(param_list, info);
5326         if (!param_item)
5327                 return -EINVAL;
5328         param = param_item->param;
5329         err = devlink_param_type_get_from_info(info, &param_type);
5330         if (err)
5331                 return err;
5332         if (param_type != param->type)
5333                 return -EINVAL;
5334         err = devlink_param_value_get_from_info(param, info, &value);
5335         if (err)
5336                 return err;
5337         if (param->validate) {
5338                 err = param->validate(devlink, param->id, value, info->extack);
5339                 if (err)
5340                         return err;
5341         }
5342
5343         if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
5344                 return -EINVAL;
5345         cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
5346         if (!devlink_param_cmode_is_supported(param, cmode))
5347                 return -EOPNOTSUPP;
5348
5349         if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
5350                 if (param->type == DEVLINK_PARAM_TYPE_STRING)
5351                         strcpy(param_item->driverinit_value.vstr, value.vstr);
5352                 else
5353                         param_item->driverinit_value = value;
5354                 param_item->driverinit_value_valid = true;
5355         } else {
5356                 if (!param->set)
5357                         return -EOPNOTSUPP;
5358                 ctx.val = value;
5359                 ctx.cmode = cmode;
5360                 err = devlink_param_set(devlink, param, &ctx);
5361                 if (err)
5362                         return err;
5363         }
5364
5365         devlink_param_notify(devlink, port_index, param_item, cmd);
5366         return 0;
5367 }
5368
5369 static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
5370                                          struct genl_info *info)
5371 {
5372         struct devlink *devlink = info->user_ptr[0];
5373
5374         return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
5375                                                info, DEVLINK_CMD_PARAM_NEW);
5376 }
5377
5378 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
5379                                                 struct netlink_callback *cb)
5380 {
5381         struct devlink_param_item *param_item;
5382         struct devlink_port *devlink_port;
5383         struct devlink *devlink;
5384         int start = cb->args[0];
5385         unsigned long index;
5386         int idx = 0;
5387         int err = 0;
5388
5389         mutex_lock(&devlink_mutex);
5390         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5391                 if (!devlink_try_get(devlink))
5392                         continue;
5393
5394                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5395                         goto retry;
5396
5397                 devl_lock(devlink);
5398                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
5399                         list_for_each_entry(param_item,
5400                                             &devlink_port->param_list, list) {
5401                                 if (idx < start) {
5402                                         idx++;
5403                                         continue;
5404                                 }
5405                                 err = devlink_nl_param_fill(msg,
5406                                                 devlink_port->devlink,
5407                                                 devlink_port->index, param_item,
5408                                                 DEVLINK_CMD_PORT_PARAM_GET,
5409                                                 NETLINK_CB(cb->skb).portid,
5410                                                 cb->nlh->nlmsg_seq,
5411                                                 NLM_F_MULTI);
5412                                 if (err == -EOPNOTSUPP) {
5413                                         err = 0;
5414                                 } else if (err) {
5415                                         devl_unlock(devlink);
5416                                         devlink_put(devlink);
5417                                         goto out;
5418                                 }
5419                                 idx++;
5420                         }
5421                 }
5422                 devl_unlock(devlink);
5423 retry:
5424                 devlink_put(devlink);
5425         }
5426 out:
5427         mutex_unlock(&devlink_mutex);
5428
5429         if (err != -EMSGSIZE)
5430                 return err;
5431
5432         cb->args[0] = idx;
5433         return msg->len;
5434 }
5435
5436 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
5437                                               struct genl_info *info)
5438 {
5439         struct devlink_port *devlink_port = info->user_ptr[1];
5440         struct devlink_param_item *param_item;
5441         struct sk_buff *msg;
5442         int err;
5443
5444         param_item = devlink_param_get_from_info(&devlink_port->param_list,
5445                                                  info);
5446         if (!param_item)
5447                 return -EINVAL;
5448
5449         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5450         if (!msg)
5451                 return -ENOMEM;
5452
5453         err = devlink_nl_param_fill(msg, devlink_port->devlink,
5454                                     devlink_port->index, param_item,
5455                                     DEVLINK_CMD_PORT_PARAM_GET,
5456                                     info->snd_portid, info->snd_seq, 0);
5457         if (err) {
5458                 nlmsg_free(msg);
5459                 return err;
5460         }
5461
5462         return genlmsg_reply(msg, info);
5463 }
5464
5465 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
5466                                               struct genl_info *info)
5467 {
5468         struct devlink_port *devlink_port = info->user_ptr[1];
5469
5470         return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
5471                                                devlink_port->index,
5472                                                &devlink_port->param_list, info,
5473                                                DEVLINK_CMD_PORT_PARAM_NEW);
5474 }
5475
5476 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
5477                                              struct devlink *devlink,
5478                                              struct devlink_snapshot *snapshot)
5479 {
5480         struct nlattr *snap_attr;
5481         int err;
5482
5483         snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
5484         if (!snap_attr)
5485                 return -EINVAL;
5486
5487         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
5488         if (err)
5489                 goto nla_put_failure;
5490
5491         nla_nest_end(msg, snap_attr);
5492         return 0;
5493
5494 nla_put_failure:
5495         nla_nest_cancel(msg, snap_attr);
5496         return err;
5497 }
5498
5499 static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
5500                                               struct devlink *devlink,
5501                                               struct devlink_region *region)
5502 {
5503         struct devlink_snapshot *snapshot;
5504         struct nlattr *snapshots_attr;
5505         int err;
5506
5507         snapshots_attr = nla_nest_start_noflag(msg,
5508                                                DEVLINK_ATTR_REGION_SNAPSHOTS);
5509         if (!snapshots_attr)
5510                 return -EINVAL;
5511
5512         list_for_each_entry(snapshot, &region->snapshot_list, list) {
5513                 err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
5514                 if (err)
5515                         goto nla_put_failure;
5516         }
5517
5518         nla_nest_end(msg, snapshots_attr);
5519         return 0;
5520
5521 nla_put_failure:
5522         nla_nest_cancel(msg, snapshots_attr);
5523         return err;
5524 }
5525
5526 static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
5527                                   enum devlink_command cmd, u32 portid,
5528                                   u32 seq, int flags,
5529                                   struct devlink_region *region)
5530 {
5531         void *hdr;
5532         int err;
5533
5534         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5535         if (!hdr)
5536                 return -EMSGSIZE;
5537
5538         err = devlink_nl_put_handle(msg, devlink);
5539         if (err)
5540                 goto nla_put_failure;
5541
5542         if (region->port) {
5543                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5544                                   region->port->index);
5545                 if (err)
5546                         goto nla_put_failure;
5547         }
5548
5549         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
5550         if (err)
5551                 goto nla_put_failure;
5552
5553         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5554                                 region->size,
5555                                 DEVLINK_ATTR_PAD);
5556         if (err)
5557                 goto nla_put_failure;
5558
5559         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
5560                           region->max_snapshots);
5561         if (err)
5562                 goto nla_put_failure;
5563
5564         err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
5565         if (err)
5566                 goto nla_put_failure;
5567
5568         genlmsg_end(msg, hdr);
5569         return 0;
5570
5571 nla_put_failure:
5572         genlmsg_cancel(msg, hdr);
5573         return err;
5574 }
5575
5576 static struct sk_buff *
5577 devlink_nl_region_notify_build(struct devlink_region *region,
5578                                struct devlink_snapshot *snapshot,
5579                                enum devlink_command cmd, u32 portid, u32 seq)
5580 {
5581         struct devlink *devlink = region->devlink;
5582         struct sk_buff *msg;
5583         void *hdr;
5584         int err;
5585
5586
5587         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5588         if (!msg)
5589                 return ERR_PTR(-ENOMEM);
5590
5591         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
5592         if (!hdr) {
5593                 err = -EMSGSIZE;
5594                 goto out_free_msg;
5595         }
5596
5597         err = devlink_nl_put_handle(msg, devlink);
5598         if (err)
5599                 goto out_cancel_msg;
5600
5601         if (region->port) {
5602                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5603                                   region->port->index);
5604                 if (err)
5605                         goto out_cancel_msg;
5606         }
5607
5608         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
5609                              region->ops->name);
5610         if (err)
5611                 goto out_cancel_msg;
5612
5613         if (snapshot) {
5614                 err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
5615                                   snapshot->id);
5616                 if (err)
5617                         goto out_cancel_msg;
5618         } else {
5619                 err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5620                                         region->size, DEVLINK_ATTR_PAD);
5621                 if (err)
5622                         goto out_cancel_msg;
5623         }
5624         genlmsg_end(msg, hdr);
5625
5626         return msg;
5627
5628 out_cancel_msg:
5629         genlmsg_cancel(msg, hdr);
5630 out_free_msg:
5631         nlmsg_free(msg);
5632         return ERR_PTR(err);
5633 }
5634
5635 static void devlink_nl_region_notify(struct devlink_region *region,
5636                                      struct devlink_snapshot *snapshot,
5637                                      enum devlink_command cmd)
5638 {
5639         struct devlink *devlink = region->devlink;
5640         struct sk_buff *msg;
5641
5642         WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
5643         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
5644                 return;
5645
5646         msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
5647         if (IS_ERR(msg))
5648                 return;
5649
5650         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
5651                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5652 }
5653
5654 /**
5655  * __devlink_snapshot_id_increment - Increment number of snapshots using an id
5656  *      @devlink: devlink instance
5657  *      @id: the snapshot id
5658  *
5659  *      Track when a new snapshot begins using an id. Load the count for the
5660  *      given id from the snapshot xarray, increment it, and store it back.
5661  *
5662  *      Called when a new snapshot is created with the given id.
5663  *
5664  *      The id *must* have been previously allocated by
5665  *      devlink_region_snapshot_id_get().
5666  *
5667  *      Returns 0 on success, or an error on failure.
5668  */
5669 static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
5670 {
5671         unsigned long count;
5672         void *p;
5673
5674         devl_assert_locked(devlink);
5675
5676         p = xa_load(&devlink->snapshot_ids, id);
5677         if (WARN_ON(!p))
5678                 return -EINVAL;
5679
5680         if (WARN_ON(!xa_is_value(p)))
5681                 return -EINVAL;
5682
5683         count = xa_to_value(p);
5684         count++;
5685
5686         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5687                                GFP_KERNEL));
5688 }
5689
5690 /**
5691  * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
5692  *      @devlink: devlink instance
5693  *      @id: the snapshot id
5694  *
5695  *      Track when a snapshot is deleted and stops using an id. Load the count
5696  *      for the given id from the snapshot xarray, decrement it, and store it
5697  *      back.
5698  *
5699  *      If the count reaches zero, erase this id from the xarray, freeing it
5700  *      up for future re-use by devlink_region_snapshot_id_get().
5701  *
5702  *      Called when a snapshot using the given id is deleted, and when the
5703  *      initial allocator of the id is finished using it.
5704  */
5705 static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
5706 {
5707         unsigned long count;
5708         void *p;
5709
5710         devl_assert_locked(devlink);
5711
5712         p = xa_load(&devlink->snapshot_ids, id);
5713         if (WARN_ON(!p))
5714                 return;
5715
5716         if (WARN_ON(!xa_is_value(p)))
5717                 return;
5718
5719         count = xa_to_value(p);
5720
5721         if (count > 1) {
5722                 count--;
5723                 xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5724                          GFP_KERNEL);
5725         } else {
5726                 /* If this was the last user, we can erase this id */
5727                 xa_erase(&devlink->snapshot_ids, id);
5728         }
5729 }
5730
5731 /**
5732  *      __devlink_snapshot_id_insert - Insert a specific snapshot ID
5733  *      @devlink: devlink instance
5734  *      @id: the snapshot id
5735  *
5736  *      Mark the given snapshot id as used by inserting a zero value into the
5737  *      snapshot xarray.
5738  *
5739  *      This must be called while holding the devlink instance lock. Unlike
5740  *      devlink_snapshot_id_get, the initial reference count is zero, not one.
5741  *      It is expected that the id will immediately be used before
5742  *      releasing the devlink instance lock.
5743  *
5744  *      Returns zero on success, or an error code if the snapshot id could not
5745  *      be inserted.
5746  */
5747 static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
5748 {
5749         devl_assert_locked(devlink);
5750
5751         if (xa_load(&devlink->snapshot_ids, id))
5752                 return -EEXIST;
5753
5754         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
5755                                GFP_KERNEL));
5756 }
5757
5758 /**
5759  *      __devlink_region_snapshot_id_get - get snapshot ID
5760  *      @devlink: devlink instance
5761  *      @id: storage to return snapshot id
5762  *
5763  *      Allocates a new snapshot id. Returns zero on success, or a negative
5764  *      error on failure. Must be called while holding the devlink instance
5765  *      lock.
5766  *
5767  *      Snapshot IDs are tracked using an xarray which stores the number of
5768  *      users of the snapshot id.
5769  *
5770  *      Note that the caller of this function counts as a 'user', in order to
5771  *      avoid race conditions. The caller must release its hold on the
5772  *      snapshot by using devlink_region_snapshot_id_put.
5773  */
5774 static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
5775 {
5776         devl_assert_locked(devlink);
5777
5778         return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
5779                         xa_limit_32b, GFP_KERNEL);
5780 }
5781
5782 /**
5783  *      __devlink_region_snapshot_create - create a new snapshot
5784  *      This will add a new snapshot of a region. The snapshot
5785  *      will be stored on the region struct and can be accessed
5786  *      from devlink. This is useful for future analyses of snapshots.
5787  *      Multiple snapshots can be created on a region.
5788  *      The @snapshot_id should be obtained using the getter function.
5789  *
5790  *      Must be called only while holding the devlink instance lock.
5791  *
5792  *      @region: devlink region of the snapshot
5793  *      @data: snapshot data
5794  *      @snapshot_id: snapshot id to be created
5795  */
5796 static int
5797 __devlink_region_snapshot_create(struct devlink_region *region,
5798                                  u8 *data, u32 snapshot_id)
5799 {
5800         struct devlink *devlink = region->devlink;
5801         struct devlink_snapshot *snapshot;
5802         int err;
5803
5804         devl_assert_locked(devlink);
5805
5806         /* check if region can hold one more snapshot */
5807         if (region->cur_snapshots == region->max_snapshots)
5808                 return -ENOSPC;
5809
5810         if (devlink_region_snapshot_get_by_id(region, snapshot_id))
5811                 return -EEXIST;
5812
5813         snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
5814         if (!snapshot)
5815                 return -ENOMEM;
5816
5817         err = __devlink_snapshot_id_increment(devlink, snapshot_id);
5818         if (err)
5819                 goto err_snapshot_id_increment;
5820
5821         snapshot->id = snapshot_id;
5822         snapshot->region = region;
5823         snapshot->data = data;
5824
5825         list_add_tail(&snapshot->list, &region->snapshot_list);
5826
5827         region->cur_snapshots++;
5828
5829         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
5830         return 0;
5831
5832 err_snapshot_id_increment:
5833         kfree(snapshot);
5834         return err;
5835 }
5836
5837 static void devlink_region_snapshot_del(struct devlink_region *region,
5838                                         struct devlink_snapshot *snapshot)
5839 {
5840         struct devlink *devlink = region->devlink;
5841
5842         devl_assert_locked(devlink);
5843
5844         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
5845         region->cur_snapshots--;
5846         list_del(&snapshot->list);
5847         region->ops->destructor(snapshot->data);
5848         __devlink_snapshot_id_decrement(devlink, snapshot->id);
5849         kfree(snapshot);
5850 }
5851
5852 static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
5853                                           struct genl_info *info)
5854 {
5855         struct devlink *devlink = info->user_ptr[0];
5856         struct devlink_port *port = NULL;
5857         struct devlink_region *region;
5858         const char *region_name;
5859         struct sk_buff *msg;
5860         unsigned int index;
5861         int err;
5862
5863         if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
5864                 return -EINVAL;
5865
5866         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5867                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5868
5869                 port = devlink_port_get_by_index(devlink, index);
5870                 if (!port)
5871                         return -ENODEV;
5872         }
5873
5874         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5875         if (port)
5876                 region = devlink_port_region_get_by_name(port, region_name);
5877         else
5878                 region = devlink_region_get_by_name(devlink, region_name);
5879
5880         if (!region)
5881                 return -EINVAL;
5882
5883         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5884         if (!msg)
5885                 return -ENOMEM;
5886
5887         err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
5888                                      info->snd_portid, info->snd_seq, 0,
5889                                      region);
5890         if (err) {
5891                 nlmsg_free(msg);
5892                 return err;
5893         }
5894
5895         return genlmsg_reply(msg, info);
5896 }
5897
5898 static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
5899                                                  struct netlink_callback *cb,
5900                                                  struct devlink_port *port,
5901                                                  int *idx,
5902                                                  int start)
5903 {
5904         struct devlink_region *region;
5905         int err = 0;
5906
5907         list_for_each_entry(region, &port->region_list, list) {
5908                 if (*idx < start) {
5909                         (*idx)++;
5910                         continue;
5911                 }
5912                 err = devlink_nl_region_fill(msg, port->devlink,
5913                                              DEVLINK_CMD_REGION_GET,
5914                                              NETLINK_CB(cb->skb).portid,
5915                                              cb->nlh->nlmsg_seq,
5916                                              NLM_F_MULTI, region);
5917                 if (err)
5918                         goto out;
5919                 (*idx)++;
5920         }
5921
5922 out:
5923         return err;
5924 }
5925
5926 static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
5927                                                     struct netlink_callback *cb,
5928                                                     struct devlink *devlink,
5929                                                     int *idx,
5930                                                     int start)
5931 {
5932         struct devlink_region *region;
5933         struct devlink_port *port;
5934         int err = 0;
5935
5936         devl_lock(devlink);
5937         list_for_each_entry(region, &devlink->region_list, list) {
5938                 if (*idx < start) {
5939                         (*idx)++;
5940                         continue;
5941                 }
5942                 err = devlink_nl_region_fill(msg, devlink,
5943                                              DEVLINK_CMD_REGION_GET,
5944                                              NETLINK_CB(cb->skb).portid,
5945                                              cb->nlh->nlmsg_seq,
5946                                              NLM_F_MULTI, region);
5947                 if (err)
5948                         goto out;
5949                 (*idx)++;
5950         }
5951
5952         list_for_each_entry(port, &devlink->port_list, list) {
5953                 err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
5954                                                             start);
5955                 if (err)
5956                         goto out;
5957         }
5958
5959 out:
5960         devl_unlock(devlink);
5961         return err;
5962 }
5963
5964 static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
5965                                             struct netlink_callback *cb)
5966 {
5967         struct devlink *devlink;
5968         int start = cb->args[0];
5969         unsigned long index;
5970         int idx = 0;
5971         int err = 0;
5972
5973         mutex_lock(&devlink_mutex);
5974         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5975                 if (!devlink_try_get(devlink))
5976                         continue;
5977
5978                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5979                         goto retry;
5980
5981                 err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
5982                                                                &idx, start);
5983 retry:
5984                 devlink_put(devlink);
5985                 if (err)
5986                         goto out;
5987         }
5988 out:
5989         mutex_unlock(&devlink_mutex);
5990         cb->args[0] = idx;
5991         return msg->len;
5992 }
5993
5994 static int devlink_nl_cmd_region_del(struct sk_buff *skb,
5995                                      struct genl_info *info)
5996 {
5997         struct devlink *devlink = info->user_ptr[0];
5998         struct devlink_snapshot *snapshot;
5999         struct devlink_port *port = NULL;
6000         struct devlink_region *region;
6001         const char *region_name;
6002         unsigned int index;
6003         u32 snapshot_id;
6004
6005         if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
6006             !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
6007                 return -EINVAL;
6008
6009         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
6010         snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
6011
6012         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6013                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6014
6015                 port = devlink_port_get_by_index(devlink, index);
6016                 if (!port)
6017                         return -ENODEV;
6018         }
6019
6020         if (port)
6021                 region = devlink_port_region_get_by_name(port, region_name);
6022         else
6023                 region = devlink_region_get_by_name(devlink, region_name);
6024
6025         if (!region)
6026                 return -EINVAL;
6027
6028         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
6029         if (!snapshot)
6030                 return -EINVAL;
6031
6032         devlink_region_snapshot_del(region, snapshot);
6033         return 0;
6034 }
6035
6036 static int
6037 devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
6038 {
6039         struct devlink *devlink = info->user_ptr[0];
6040         struct devlink_snapshot *snapshot;
6041         struct devlink_port *port = NULL;
6042         struct nlattr *snapshot_id_attr;
6043         struct devlink_region *region;
6044         const char *region_name;
6045         unsigned int index;
6046         u32 snapshot_id;
6047         u8 *data;
6048         int err;
6049
6050         if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
6051                 NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
6052                 return -EINVAL;
6053         }
6054
6055         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
6056
6057         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6058                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6059
6060                 port = devlink_port_get_by_index(devlink, index);
6061                 if (!port)
6062                         return -ENODEV;
6063         }
6064
6065         if (port)
6066                 region = devlink_port_region_get_by_name(port, region_name);
6067         else
6068                 region = devlink_region_get_by_name(devlink, region_name);
6069
6070         if (!region) {
6071                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
6072                 return -EINVAL;
6073         }
6074
6075         if (!region->ops->snapshot) {
6076                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
6077                 return -EOPNOTSUPP;
6078         }
6079
6080         if (region->cur_snapshots == region->max_snapshots) {
6081                 NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
6082                 return -ENOSPC;
6083         }
6084
6085         snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
6086         if (snapshot_id_attr) {
6087                 snapshot_id = nla_get_u32(snapshot_id_attr);
6088
6089                 if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
6090                         NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
6091                         return -EEXIST;
6092                 }
6093
6094                 err = __devlink_snapshot_id_insert(devlink, snapshot_id);
6095                 if (err)
6096                         return err;
6097         } else {
6098                 err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
6099                 if (err) {
6100                         NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
6101                         return err;
6102                 }
6103         }
6104
6105         if (port)
6106                 err = region->port_ops->snapshot(port, region->port_ops,
6107                                                  info->extack, &data);
6108         else
6109                 err = region->ops->snapshot(devlink, region->ops,
6110                                             info->extack, &data);
6111         if (err)
6112                 goto err_snapshot_capture;
6113
6114         err = __devlink_region_snapshot_create(region, data, snapshot_id);
6115         if (err)
6116                 goto err_snapshot_create;
6117
6118         if (!snapshot_id_attr) {
6119                 struct sk_buff *msg;
6120
6121                 snapshot = devlink_region_snapshot_get_by_id(region,
6122                                                              snapshot_id);
6123                 if (WARN_ON(!snapshot))
6124                         return -EINVAL;
6125
6126                 msg = devlink_nl_region_notify_build(region, snapshot,
6127                                                      DEVLINK_CMD_REGION_NEW,
6128                                                      info->snd_portid,
6129                                                      info->snd_seq);
6130                 err = PTR_ERR_OR_ZERO(msg);
6131                 if (err)
6132                         goto err_notify;
6133
6134                 err = genlmsg_reply(msg, info);
6135                 if (err)
6136                         goto err_notify;
6137         }
6138
6139         return 0;
6140
6141 err_snapshot_create:
6142         region->ops->destructor(data);
6143 err_snapshot_capture:
6144         __devlink_snapshot_id_decrement(devlink, snapshot_id);
6145         return err;
6146
6147 err_notify:
6148         devlink_region_snapshot_del(region, snapshot);
6149         return err;
6150 }
6151
6152 static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
6153                                                  struct devlink *devlink,
6154                                                  u8 *chunk, u32 chunk_size,
6155                                                  u64 addr)
6156 {
6157         struct nlattr *chunk_attr;
6158         int err;
6159
6160         chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
6161         if (!chunk_attr)
6162                 return -EINVAL;
6163
6164         err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
6165         if (err)
6166                 goto nla_put_failure;
6167
6168         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
6169                                 DEVLINK_ATTR_PAD);
6170         if (err)
6171                 goto nla_put_failure;
6172
6173         nla_nest_end(msg, chunk_attr);
6174         return 0;
6175
6176 nla_put_failure:
6177         nla_nest_cancel(msg, chunk_attr);
6178         return err;
6179 }
6180
6181 #define DEVLINK_REGION_READ_CHUNK_SIZE 256
6182
6183 static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
6184                                                 struct devlink *devlink,
6185                                                 struct devlink_region *region,
6186                                                 struct nlattr **attrs,
6187                                                 u64 start_offset,
6188                                                 u64 end_offset,
6189                                                 u64 *new_offset)
6190 {
6191         struct devlink_snapshot *snapshot;
6192         u64 curr_offset = start_offset;
6193         u32 snapshot_id;
6194         int err = 0;
6195
6196         *new_offset = start_offset;
6197
6198         snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
6199         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
6200         if (!snapshot)
6201                 return -EINVAL;
6202
6203         while (curr_offset < end_offset) {
6204                 u32 data_size;
6205                 u8 *data;
6206
6207                 if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
6208                         data_size = end_offset - curr_offset;
6209                 else
6210                         data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
6211
6212                 data = &snapshot->data[curr_offset];
6213                 err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
6214                                                             data, data_size,
6215                                                             curr_offset);
6216                 if (err)
6217                         break;
6218
6219                 curr_offset += data_size;
6220         }
6221         *new_offset = curr_offset;
6222
6223         return err;
6224 }
6225
6226 static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
6227                                              struct netlink_callback *cb)
6228 {
6229         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
6230         u64 ret_offset, start_offset, end_offset = U64_MAX;
6231         struct nlattr **attrs = info->attrs;
6232         struct devlink_port *port = NULL;
6233         struct devlink_region *region;
6234         struct nlattr *chunks_attr;
6235         const char *region_name;
6236         struct devlink *devlink;
6237         unsigned int index;
6238         void *hdr;
6239         int err;
6240
6241         start_offset = *((u64 *)&cb->args[0]);
6242
6243         mutex_lock(&devlink_mutex);
6244         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
6245         if (IS_ERR(devlink)) {
6246                 err = PTR_ERR(devlink);
6247                 goto out_dev;
6248         }
6249
6250         devl_lock(devlink);
6251
6252         if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
6253             !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
6254                 err = -EINVAL;
6255                 goto out_unlock;
6256         }
6257
6258         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
6259                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
6260
6261                 port = devlink_port_get_by_index(devlink, index);
6262                 if (!port) {
6263                         err = -ENODEV;
6264                         goto out_unlock;
6265                 }
6266         }
6267
6268         region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
6269
6270         if (port)
6271                 region = devlink_port_region_get_by_name(port, region_name);
6272         else
6273                 region = devlink_region_get_by_name(devlink, region_name);
6274
6275         if (!region) {
6276                 err = -EINVAL;
6277                 goto out_unlock;
6278         }
6279
6280         if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
6281             attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
6282                 if (!start_offset)
6283                         start_offset =
6284                                 nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
6285
6286                 end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
6287                 end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
6288         }
6289
6290         if (end_offset > region->size)
6291                 end_offset = region->size;
6292
6293         /* return 0 if there is no further data to read */
6294         if (start_offset == end_offset) {
6295                 err = 0;
6296                 goto out_unlock;
6297         }
6298
6299         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
6300                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
6301                           DEVLINK_CMD_REGION_READ);
6302         if (!hdr) {
6303                 err = -EMSGSIZE;
6304                 goto out_unlock;
6305         }
6306
6307         err = devlink_nl_put_handle(skb, devlink);
6308         if (err)
6309                 goto nla_put_failure;
6310
6311         if (region->port) {
6312                 err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
6313                                   region->port->index);
6314                 if (err)
6315                         goto nla_put_failure;
6316         }
6317
6318         err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
6319         if (err)
6320                 goto nla_put_failure;
6321
6322         chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
6323         if (!chunks_attr) {
6324                 err = -EMSGSIZE;
6325                 goto nla_put_failure;
6326         }
6327
6328         err = devlink_nl_region_read_snapshot_fill(skb, devlink,
6329                                                    region, attrs,
6330                                                    start_offset,
6331                                                    end_offset, &ret_offset);
6332
6333         if (err && err != -EMSGSIZE)
6334                 goto nla_put_failure;
6335
6336         /* Check if there was any progress done to prevent infinite loop */
6337         if (ret_offset == start_offset) {
6338                 err = -EINVAL;
6339                 goto nla_put_failure;
6340         }
6341
6342         *((u64 *)&cb->args[0]) = ret_offset;
6343
6344         nla_nest_end(skb, chunks_attr);
6345         genlmsg_end(skb, hdr);
6346         devl_unlock(devlink);
6347         devlink_put(devlink);
6348         mutex_unlock(&devlink_mutex);
6349
6350         return skb->len;
6351
6352 nla_put_failure:
6353         genlmsg_cancel(skb, hdr);
6354 out_unlock:
6355         devl_unlock(devlink);
6356         devlink_put(devlink);
6357 out_dev:
6358         mutex_unlock(&devlink_mutex);
6359         return err;
6360 }
6361
6362 struct devlink_info_req {
6363         struct sk_buff *msg;
6364 };
6365
6366 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
6367 {
6368         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
6369 }
6370 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
6371
6372 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
6373 {
6374         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
6375 }
6376 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
6377
6378 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
6379                                          const char *bsn)
6380 {
6381         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
6382                               bsn);
6383 }
6384 EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
6385
6386 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
6387                                     const char *version_name,
6388                                     const char *version_value)
6389 {
6390         struct nlattr *nest;
6391         int err;
6392
6393         nest = nla_nest_start_noflag(req->msg, attr);
6394         if (!nest)
6395                 return -EMSGSIZE;
6396
6397         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
6398                              version_name);
6399         if (err)
6400                 goto nla_put_failure;
6401
6402         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
6403                              version_value);
6404         if (err)
6405                 goto nla_put_failure;
6406
6407         nla_nest_end(req->msg, nest);
6408
6409         return 0;
6410
6411 nla_put_failure:
6412         nla_nest_cancel(req->msg, nest);
6413         return err;
6414 }
6415
6416 int devlink_info_version_fixed_put(struct devlink_info_req *req,
6417                                    const char *version_name,
6418                                    const char *version_value)
6419 {
6420         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
6421                                         version_name, version_value);
6422 }
6423 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
6424
6425 int devlink_info_version_stored_put(struct devlink_info_req *req,
6426                                     const char *version_name,
6427                                     const char *version_value)
6428 {
6429         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
6430                                         version_name, version_value);
6431 }
6432 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
6433
6434 int devlink_info_version_running_put(struct devlink_info_req *req,
6435                                      const char *version_name,
6436                                      const char *version_value)
6437 {
6438         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
6439                                         version_name, version_value);
6440 }
6441 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
6442
6443 static int
6444 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
6445                      enum devlink_command cmd, u32 portid,
6446                      u32 seq, int flags, struct netlink_ext_ack *extack)
6447 {
6448         struct devlink_info_req req;
6449         void *hdr;
6450         int err;
6451
6452         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6453         if (!hdr)
6454                 return -EMSGSIZE;
6455
6456         err = -EMSGSIZE;
6457         if (devlink_nl_put_handle(msg, devlink))
6458                 goto err_cancel_msg;
6459
6460         req.msg = msg;
6461         err = devlink->ops->info_get(devlink, &req, extack);
6462         if (err)
6463                 goto err_cancel_msg;
6464
6465         genlmsg_end(msg, hdr);
6466         return 0;
6467
6468 err_cancel_msg:
6469         genlmsg_cancel(msg, hdr);
6470         return err;
6471 }
6472
6473 static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
6474                                         struct genl_info *info)
6475 {
6476         struct devlink *devlink = info->user_ptr[0];
6477         struct sk_buff *msg;
6478         int err;
6479
6480         if (!devlink->ops->info_get)
6481                 return -EOPNOTSUPP;
6482
6483         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6484         if (!msg)
6485                 return -ENOMEM;
6486
6487         err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6488                                    info->snd_portid, info->snd_seq, 0,
6489                                    info->extack);
6490         if (err) {
6491                 nlmsg_free(msg);
6492                 return err;
6493         }
6494
6495         return genlmsg_reply(msg, info);
6496 }
6497
6498 static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
6499                                           struct netlink_callback *cb)
6500 {
6501         struct devlink *devlink;
6502         int start = cb->args[0];
6503         unsigned long index;
6504         int idx = 0;
6505         int err = 0;
6506
6507         mutex_lock(&devlink_mutex);
6508         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
6509                 if (!devlink_try_get(devlink))
6510                         continue;
6511
6512                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6513                         goto retry;
6514
6515                 if (idx < start || !devlink->ops->info_get)
6516                         goto inc;
6517
6518                 devl_lock(devlink);
6519                 err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6520                                            NETLINK_CB(cb->skb).portid,
6521                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
6522                                            cb->extack);
6523                 devl_unlock(devlink);
6524                 if (err == -EOPNOTSUPP)
6525                         err = 0;
6526                 else if (err) {
6527                         devlink_put(devlink);
6528                         break;
6529                 }
6530 inc:
6531                 idx++;
6532 retry:
6533                 devlink_put(devlink);
6534         }
6535         mutex_unlock(&devlink_mutex);
6536
6537         if (err != -EMSGSIZE)
6538                 return err;
6539
6540         cb->args[0] = idx;
6541         return msg->len;
6542 }
6543
6544 struct devlink_fmsg_item {
6545         struct list_head list;
6546         int attrtype;
6547         u8 nla_type;
6548         u16 len;
6549         int value[];
6550 };
6551
6552 struct devlink_fmsg {
6553         struct list_head item_list;
6554         bool putting_binary; /* This flag forces enclosing of binary data
6555                               * in an array brackets. It forces using
6556                               * of designated API:
6557                               * devlink_fmsg_binary_pair_nest_start()
6558                               * devlink_fmsg_binary_pair_nest_end()
6559                               */
6560 };
6561
6562 static struct devlink_fmsg *devlink_fmsg_alloc(void)
6563 {
6564         struct devlink_fmsg *fmsg;
6565
6566         fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
6567         if (!fmsg)
6568                 return NULL;
6569
6570         INIT_LIST_HEAD(&fmsg->item_list);
6571
6572         return fmsg;
6573 }
6574
6575 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
6576 {
6577         struct devlink_fmsg_item *item, *tmp;
6578
6579         list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
6580                 list_del(&item->list);
6581                 kfree(item);
6582         }
6583         kfree(fmsg);
6584 }
6585
6586 static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
6587                                     int attrtype)
6588 {
6589         struct devlink_fmsg_item *item;
6590
6591         item = kzalloc(sizeof(*item), GFP_KERNEL);
6592         if (!item)
6593                 return -ENOMEM;
6594
6595         item->attrtype = attrtype;
6596         list_add_tail(&item->list, &fmsg->item_list);
6597
6598         return 0;
6599 }
6600
6601 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
6602 {
6603         if (fmsg->putting_binary)
6604                 return -EINVAL;
6605
6606         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
6607 }
6608 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
6609
6610 static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
6611 {
6612         if (fmsg->putting_binary)
6613                 return -EINVAL;
6614
6615         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
6616 }
6617
6618 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
6619 {
6620         if (fmsg->putting_binary)
6621                 return -EINVAL;
6622
6623         return devlink_fmsg_nest_end(fmsg);
6624 }
6625 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
6626
6627 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
6628
6629 static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
6630 {
6631         struct devlink_fmsg_item *item;
6632
6633         if (fmsg->putting_binary)
6634                 return -EINVAL;
6635
6636         if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
6637                 return -EMSGSIZE;
6638
6639         item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
6640         if (!item)
6641                 return -ENOMEM;
6642
6643         item->nla_type = NLA_NUL_STRING;
6644         item->len = strlen(name) + 1;
6645         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
6646         memcpy(&item->value, name, item->len);
6647         list_add_tail(&item->list, &fmsg->item_list);
6648
6649         return 0;
6650 }
6651
6652 int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
6653 {
6654         int err;
6655
6656         if (fmsg->putting_binary)
6657                 return -EINVAL;
6658
6659         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
6660         if (err)
6661                 return err;
6662
6663         err = devlink_fmsg_put_name(fmsg, name);
6664         if (err)
6665                 return err;
6666
6667         return 0;
6668 }
6669 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
6670
6671 int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
6672 {
6673         if (fmsg->putting_binary)
6674                 return -EINVAL;
6675
6676         return devlink_fmsg_nest_end(fmsg);
6677 }
6678 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
6679
6680 int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
6681                                      const char *name)
6682 {
6683         int err;
6684
6685         if (fmsg->putting_binary)
6686                 return -EINVAL;
6687
6688         err = devlink_fmsg_pair_nest_start(fmsg, name);
6689         if (err)
6690                 return err;
6691
6692         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
6693         if (err)
6694                 return err;
6695
6696         return 0;
6697 }
6698 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
6699
6700 int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
6701 {
6702         int err;
6703
6704         if (fmsg->putting_binary)
6705                 return -EINVAL;
6706
6707         err = devlink_fmsg_nest_end(fmsg);
6708         if (err)
6709                 return err;
6710
6711         err = devlink_fmsg_nest_end(fmsg);
6712         if (err)
6713                 return err;
6714
6715         return 0;
6716 }
6717 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
6718
6719 int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
6720                                         const char *name)
6721 {
6722         int err;
6723
6724         err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
6725         if (err)
6726                 return err;
6727
6728         fmsg->putting_binary = true;
6729         return err;
6730 }
6731 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
6732
6733 int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
6734 {
6735         if (!fmsg->putting_binary)
6736                 return -EINVAL;
6737
6738         fmsg->putting_binary = false;
6739         return devlink_fmsg_arr_pair_nest_end(fmsg);
6740 }
6741 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
6742
6743 static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
6744                                   const void *value, u16 value_len,
6745                                   u8 value_nla_type)
6746 {
6747         struct devlink_fmsg_item *item;
6748
6749         if (value_len > DEVLINK_FMSG_MAX_SIZE)
6750                 return -EMSGSIZE;
6751
6752         item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
6753         if (!item)
6754                 return -ENOMEM;
6755
6756         item->nla_type = value_nla_type;
6757         item->len = value_len;
6758         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6759         memcpy(&item->value, value, item->len);
6760         list_add_tail(&item->list, &fmsg->item_list);
6761
6762         return 0;
6763 }
6764
6765 static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
6766 {
6767         if (fmsg->putting_binary)
6768                 return -EINVAL;
6769
6770         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
6771 }
6772
6773 static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
6774 {
6775         if (fmsg->putting_binary)
6776                 return -EINVAL;
6777
6778         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
6779 }
6780
6781 int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
6782 {
6783         if (fmsg->putting_binary)
6784                 return -EINVAL;
6785
6786         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
6787 }
6788 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
6789
6790 static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
6791 {
6792         if (fmsg->putting_binary)
6793                 return -EINVAL;
6794
6795         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
6796 }
6797
6798 int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
6799 {
6800         if (fmsg->putting_binary)
6801                 return -EINVAL;
6802
6803         return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
6804                                       NLA_NUL_STRING);
6805 }
6806 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
6807
6808 int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
6809                             u16 value_len)
6810 {
6811         if (!fmsg->putting_binary)
6812                 return -EINVAL;
6813
6814         return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
6815 }
6816 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
6817
6818 int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
6819                                bool value)
6820 {
6821         int err;
6822
6823         err = devlink_fmsg_pair_nest_start(fmsg, name);
6824         if (err)
6825                 return err;
6826
6827         err = devlink_fmsg_bool_put(fmsg, value);
6828         if (err)
6829                 return err;
6830
6831         err = devlink_fmsg_pair_nest_end(fmsg);
6832         if (err)
6833                 return err;
6834
6835         return 0;
6836 }
6837 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
6838
6839 int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
6840                              u8 value)
6841 {
6842         int err;
6843
6844         err = devlink_fmsg_pair_nest_start(fmsg, name);
6845         if (err)
6846                 return err;
6847
6848         err = devlink_fmsg_u8_put(fmsg, value);
6849         if (err)
6850                 return err;
6851
6852         err = devlink_fmsg_pair_nest_end(fmsg);
6853         if (err)
6854                 return err;
6855
6856         return 0;
6857 }
6858 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
6859
6860 int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
6861                               u32 value)
6862 {
6863         int err;
6864
6865         err = devlink_fmsg_pair_nest_start(fmsg, name);
6866         if (err)
6867                 return err;
6868
6869         err = devlink_fmsg_u32_put(fmsg, value);
6870         if (err)
6871                 return err;
6872
6873         err = devlink_fmsg_pair_nest_end(fmsg);
6874         if (err)
6875                 return err;
6876
6877         return 0;
6878 }
6879 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
6880
6881 int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
6882                               u64 value)
6883 {
6884         int err;
6885
6886         err = devlink_fmsg_pair_nest_start(fmsg, name);
6887         if (err)
6888                 return err;
6889
6890         err = devlink_fmsg_u64_put(fmsg, value);
6891         if (err)
6892                 return err;
6893
6894         err = devlink_fmsg_pair_nest_end(fmsg);
6895         if (err)
6896                 return err;
6897
6898         return 0;
6899 }
6900 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
6901
6902 int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
6903                                  const char *value)
6904 {
6905         int err;
6906
6907         err = devlink_fmsg_pair_nest_start(fmsg, name);
6908         if (err)
6909                 return err;
6910
6911         err = devlink_fmsg_string_put(fmsg, value);
6912         if (err)
6913                 return err;
6914
6915         err = devlink_fmsg_pair_nest_end(fmsg);
6916         if (err)
6917                 return err;
6918
6919         return 0;
6920 }
6921 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
6922
6923 int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
6924                                  const void *value, u32 value_len)
6925 {
6926         u32 data_size;
6927         int end_err;
6928         u32 offset;
6929         int err;
6930
6931         err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
6932         if (err)
6933                 return err;
6934
6935         for (offset = 0; offset < value_len; offset += data_size) {
6936                 data_size = value_len - offset;
6937                 if (data_size > DEVLINK_FMSG_MAX_SIZE)
6938                         data_size = DEVLINK_FMSG_MAX_SIZE;
6939                 err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
6940                 if (err)
6941                         break;
6942                 /* Exit from loop with a break (instead of
6943                  * return) to make sure putting_binary is turned off in
6944                  * devlink_fmsg_binary_pair_nest_end
6945                  */
6946         }
6947
6948         end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
6949         if (end_err)
6950                 err = end_err;
6951
6952         return err;
6953 }
6954 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
6955
6956 static int
6957 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6958 {
6959         switch (msg->nla_type) {
6960         case NLA_FLAG:
6961         case NLA_U8:
6962         case NLA_U32:
6963         case NLA_U64:
6964         case NLA_NUL_STRING:
6965         case NLA_BINARY:
6966                 return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
6967                                   msg->nla_type);
6968         default:
6969                 return -EINVAL;
6970         }
6971 }
6972
6973 static int
6974 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6975 {
6976         int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6977         u8 tmp;
6978
6979         switch (msg->nla_type) {
6980         case NLA_FLAG:
6981                 /* Always provide flag data, regardless of its value */
6982                 tmp = *(bool *) msg->value;
6983
6984                 return nla_put_u8(skb, attrtype, tmp);
6985         case NLA_U8:
6986                 return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
6987         case NLA_U32:
6988                 return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
6989         case NLA_U64:
6990                 return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
6991                                          DEVLINK_ATTR_PAD);
6992         case NLA_NUL_STRING:
6993                 return nla_put_string(skb, attrtype, (char *) &msg->value);
6994         case NLA_BINARY:
6995                 return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
6996         default:
6997                 return -EINVAL;
6998         }
6999 }
7000
7001 static int
7002 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
7003                          int *start)
7004 {
7005         struct devlink_fmsg_item *item;
7006         struct nlattr *fmsg_nlattr;
7007         int i = 0;
7008         int err;
7009
7010         fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
7011         if (!fmsg_nlattr)
7012                 return -EMSGSIZE;
7013
7014         list_for_each_entry(item, &fmsg->item_list, list) {
7015                 if (i < *start) {
7016                         i++;
7017                         continue;
7018                 }
7019
7020                 switch (item->attrtype) {
7021                 case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
7022                 case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
7023                 case DEVLINK_ATTR_FMSG_ARR_NEST_START:
7024                 case DEVLINK_ATTR_FMSG_NEST_END:
7025                         err = nla_put_flag(skb, item->attrtype);
7026                         break;
7027                 case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
7028                         err = devlink_fmsg_item_fill_type(item, skb);
7029                         if (err)
7030                                 break;
7031                         err = devlink_fmsg_item_fill_data(item, skb);
7032                         break;
7033                 case DEVLINK_ATTR_FMSG_OBJ_NAME:
7034                         err = nla_put_string(skb, item->attrtype,
7035                                              (char *) &item->value);
7036                         break;
7037                 default:
7038                         err = -EINVAL;
7039                         break;
7040                 }
7041                 if (!err)
7042                         *start = ++i;
7043                 else
7044                         break;
7045         }
7046
7047         nla_nest_end(skb, fmsg_nlattr);
7048         return err;
7049 }
7050
7051 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
7052                             struct genl_info *info,
7053                             enum devlink_command cmd, int flags)
7054 {
7055         struct nlmsghdr *nlh;
7056         struct sk_buff *skb;
7057         bool last = false;
7058         int index = 0;
7059         void *hdr;
7060         int err;
7061
7062         while (!last) {
7063                 int tmp_index = index;
7064
7065                 skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
7066                 if (!skb)
7067                         return -ENOMEM;
7068
7069                 hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
7070                                   &devlink_nl_family, flags | NLM_F_MULTI, cmd);
7071                 if (!hdr) {
7072                         err = -EMSGSIZE;
7073                         goto nla_put_failure;
7074                 }
7075
7076                 err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
7077                 if (!err)
7078                         last = true;
7079                 else if (err != -EMSGSIZE || tmp_index == index)
7080                         goto nla_put_failure;
7081
7082                 genlmsg_end(skb, hdr);
7083                 err = genlmsg_reply(skb, info);
7084                 if (err)
7085                         return err;
7086         }
7087
7088         skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
7089         if (!skb)
7090                 return -ENOMEM;
7091         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
7092                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
7093         if (!nlh) {
7094                 err = -EMSGSIZE;
7095                 goto nla_put_failure;
7096         }
7097
7098         return genlmsg_reply(skb, info);
7099
7100 nla_put_failure:
7101         nlmsg_free(skb);
7102         return err;
7103 }
7104
7105 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
7106                                struct netlink_callback *cb,
7107                                enum devlink_command cmd)
7108 {
7109         int index = cb->args[0];
7110         int tmp_index = index;
7111         void *hdr;
7112         int err;
7113
7114         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7115                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
7116         if (!hdr) {
7117                 err = -EMSGSIZE;
7118                 goto nla_put_failure;
7119         }
7120
7121         err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
7122         if ((err && err != -EMSGSIZE) || tmp_index == index)
7123                 goto nla_put_failure;
7124
7125         cb->args[0] = index;
7126         genlmsg_end(skb, hdr);
7127         return skb->len;
7128
7129 nla_put_failure:
7130         genlmsg_cancel(skb, hdr);
7131         return err;
7132 }
7133
7134 struct devlink_health_reporter {
7135         struct list_head list;
7136         void *priv;
7137         const struct devlink_health_reporter_ops *ops;
7138         struct devlink *devlink;
7139         struct devlink_port *devlink_port;
7140         struct devlink_fmsg *dump_fmsg;
7141         struct mutex dump_lock; /* lock parallel read/write from dump buffers */
7142         u64 graceful_period;
7143         bool auto_recover;
7144         bool auto_dump;
7145         u8 health_state;
7146         u64 dump_ts;
7147         u64 dump_real_ts;
7148         u64 error_count;
7149         u64 recovery_count;
7150         u64 last_recovery_ts;
7151         refcount_t refcount;
7152 };
7153
7154 void *
7155 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
7156 {
7157         return reporter->priv;
7158 }
7159 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
7160
7161 static struct devlink_health_reporter *
7162 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
7163                                        struct mutex *list_lock,
7164                                        const char *reporter_name)
7165 {
7166         struct devlink_health_reporter *reporter;
7167
7168         lockdep_assert_held(list_lock);
7169         list_for_each_entry(reporter, reporter_list, list)
7170                 if (!strcmp(reporter->ops->name, reporter_name))
7171                         return reporter;
7172         return NULL;
7173 }
7174
7175 static struct devlink_health_reporter *
7176 devlink_health_reporter_find_by_name(struct devlink *devlink,
7177                                      const char *reporter_name)
7178 {
7179         return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
7180                                                       &devlink->reporters_lock,
7181                                                       reporter_name);
7182 }
7183
7184 static struct devlink_health_reporter *
7185 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
7186                                           const char *reporter_name)
7187 {
7188         return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
7189                                                       &devlink_port->reporters_lock,
7190                                                       reporter_name);
7191 }
7192
7193 static struct devlink_health_reporter *
7194 __devlink_health_reporter_create(struct devlink *devlink,
7195                                  const struct devlink_health_reporter_ops *ops,
7196                                  u64 graceful_period, void *priv)
7197 {
7198         struct devlink_health_reporter *reporter;
7199
7200         if (WARN_ON(graceful_period && !ops->recover))
7201                 return ERR_PTR(-EINVAL);
7202
7203         reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
7204         if (!reporter)
7205                 return ERR_PTR(-ENOMEM);
7206
7207         reporter->priv = priv;
7208         reporter->ops = ops;
7209         reporter->devlink = devlink;
7210         reporter->graceful_period = graceful_period;
7211         reporter->auto_recover = !!ops->recover;
7212         reporter->auto_dump = !!ops->dump;
7213         mutex_init(&reporter->dump_lock);
7214         refcount_set(&reporter->refcount, 1);
7215         return reporter;
7216 }
7217
7218 /**
7219  *      devlink_port_health_reporter_create - create devlink health reporter for
7220  *                                            specified port instance
7221  *
7222  *      @port: devlink_port which should contain the new reporter
7223  *      @ops: ops
7224  *      @graceful_period: to avoid recovery loops, in msecs
7225  *      @priv: priv
7226  */
7227 struct devlink_health_reporter *
7228 devlink_port_health_reporter_create(struct devlink_port *port,
7229                                     const struct devlink_health_reporter_ops *ops,
7230                                     u64 graceful_period, void *priv)
7231 {
7232         struct devlink_health_reporter *reporter;
7233
7234         mutex_lock(&port->reporters_lock);
7235         if (__devlink_health_reporter_find_by_name(&port->reporter_list,
7236                                                    &port->reporters_lock, ops->name)) {
7237                 reporter = ERR_PTR(-EEXIST);
7238                 goto unlock;
7239         }
7240
7241         reporter = __devlink_health_reporter_create(port->devlink, ops,
7242                                                     graceful_period, priv);
7243         if (IS_ERR(reporter))
7244                 goto unlock;
7245
7246         reporter->devlink_port = port;
7247         list_add_tail(&reporter->list, &port->reporter_list);
7248 unlock:
7249         mutex_unlock(&port->reporters_lock);
7250         return reporter;
7251 }
7252 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
7253
7254 /**
7255  *      devlink_health_reporter_create - create devlink health reporter
7256  *
7257  *      @devlink: devlink
7258  *      @ops: ops
7259  *      @graceful_period: to avoid recovery loops, in msecs
7260  *      @priv: priv
7261  */
7262 struct devlink_health_reporter *
7263 devlink_health_reporter_create(struct devlink *devlink,
7264                                const struct devlink_health_reporter_ops *ops,
7265                                u64 graceful_period, void *priv)
7266 {
7267         struct devlink_health_reporter *reporter;
7268
7269         mutex_lock(&devlink->reporters_lock);
7270         if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
7271                 reporter = ERR_PTR(-EEXIST);
7272                 goto unlock;
7273         }
7274
7275         reporter = __devlink_health_reporter_create(devlink, ops,
7276                                                     graceful_period, priv);
7277         if (IS_ERR(reporter))
7278                 goto unlock;
7279
7280         list_add_tail(&reporter->list, &devlink->reporter_list);
7281 unlock:
7282         mutex_unlock(&devlink->reporters_lock);
7283         return reporter;
7284 }
7285 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
7286
7287 static void
7288 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
7289 {
7290         mutex_destroy(&reporter->dump_lock);
7291         if (reporter->dump_fmsg)
7292                 devlink_fmsg_free(reporter->dump_fmsg);
7293         kfree(reporter);
7294 }
7295
7296 static void
7297 devlink_health_reporter_put(struct devlink_health_reporter *reporter)
7298 {
7299         if (refcount_dec_and_test(&reporter->refcount))
7300                 devlink_health_reporter_free(reporter);
7301 }
7302
7303 static void
7304 __devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
7305 {
7306         list_del(&reporter->list);
7307         devlink_health_reporter_put(reporter);
7308 }
7309
7310 /**
7311  *      devlink_health_reporter_destroy - destroy devlink health reporter
7312  *
7313  *      @reporter: devlink health reporter to destroy
7314  */
7315 void
7316 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
7317 {
7318         struct mutex *lock = &reporter->devlink->reporters_lock;
7319
7320         mutex_lock(lock);
7321         __devlink_health_reporter_destroy(reporter);
7322         mutex_unlock(lock);
7323 }
7324 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
7325
7326 /**
7327  *      devlink_port_health_reporter_destroy - destroy devlink port health reporter
7328  *
7329  *      @reporter: devlink health reporter to destroy
7330  */
7331 void
7332 devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
7333 {
7334         struct mutex *lock = &reporter->devlink_port->reporters_lock;
7335
7336         mutex_lock(lock);
7337         __devlink_health_reporter_destroy(reporter);
7338         mutex_unlock(lock);
7339 }
7340 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
7341
7342 static int
7343 devlink_nl_health_reporter_fill(struct sk_buff *msg,
7344                                 struct devlink_health_reporter *reporter,
7345                                 enum devlink_command cmd, u32 portid,
7346                                 u32 seq, int flags)
7347 {
7348         struct devlink *devlink = reporter->devlink;
7349         struct nlattr *reporter_attr;
7350         void *hdr;
7351
7352         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7353         if (!hdr)
7354                 return -EMSGSIZE;
7355
7356         if (devlink_nl_put_handle(msg, devlink))
7357                 goto genlmsg_cancel;
7358
7359         if (reporter->devlink_port) {
7360                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
7361                         goto genlmsg_cancel;
7362         }
7363         reporter_attr = nla_nest_start_noflag(msg,
7364                                               DEVLINK_ATTR_HEALTH_REPORTER);
7365         if (!reporter_attr)
7366                 goto genlmsg_cancel;
7367         if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
7368                            reporter->ops->name))
7369                 goto reporter_nest_cancel;
7370         if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
7371                        reporter->health_state))
7372                 goto reporter_nest_cancel;
7373         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
7374                               reporter->error_count, DEVLINK_ATTR_PAD))
7375                 goto reporter_nest_cancel;
7376         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
7377                               reporter->recovery_count, DEVLINK_ATTR_PAD))
7378                 goto reporter_nest_cancel;
7379         if (reporter->ops->recover &&
7380             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
7381                               reporter->graceful_period,
7382                               DEVLINK_ATTR_PAD))
7383                 goto reporter_nest_cancel;
7384         if (reporter->ops->recover &&
7385             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
7386                        reporter->auto_recover))
7387                 goto reporter_nest_cancel;
7388         if (reporter->dump_fmsg &&
7389             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
7390                               jiffies_to_msecs(reporter->dump_ts),
7391                               DEVLINK_ATTR_PAD))
7392                 goto reporter_nest_cancel;
7393         if (reporter->dump_fmsg &&
7394             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
7395                               reporter->dump_real_ts, DEVLINK_ATTR_PAD))
7396                 goto reporter_nest_cancel;
7397         if (reporter->ops->dump &&
7398             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
7399                        reporter->auto_dump))
7400                 goto reporter_nest_cancel;
7401
7402         nla_nest_end(msg, reporter_attr);
7403         genlmsg_end(msg, hdr);
7404         return 0;
7405
7406 reporter_nest_cancel:
7407         nla_nest_end(msg, reporter_attr);
7408 genlmsg_cancel:
7409         genlmsg_cancel(msg, hdr);
7410         return -EMSGSIZE;
7411 }
7412
7413 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
7414                                    enum devlink_command cmd)
7415 {
7416         struct devlink *devlink = reporter->devlink;
7417         struct sk_buff *msg;
7418         int err;
7419
7420         WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7421         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
7422
7423         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7424         if (!msg)
7425                 return;
7426
7427         err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
7428         if (err) {
7429                 nlmsg_free(msg);
7430                 return;
7431         }
7432
7433         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
7434                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
7435 }
7436
7437 void
7438 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
7439 {
7440         reporter->recovery_count++;
7441         reporter->last_recovery_ts = jiffies;
7442 }
7443 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
7444
7445 static int
7446 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
7447                                 void *priv_ctx, struct netlink_ext_ack *extack)
7448 {
7449         int err;
7450
7451         if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
7452                 return 0;
7453
7454         if (!reporter->ops->recover)
7455                 return -EOPNOTSUPP;
7456
7457         err = reporter->ops->recover(reporter, priv_ctx, extack);
7458         if (err)
7459                 return err;
7460
7461         devlink_health_reporter_recovery_done(reporter);
7462         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
7463         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7464
7465         return 0;
7466 }
7467
7468 static void
7469 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
7470 {
7471         if (!reporter->dump_fmsg)
7472                 return;
7473         devlink_fmsg_free(reporter->dump_fmsg);
7474         reporter->dump_fmsg = NULL;
7475 }
7476
7477 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
7478                                   void *priv_ctx,
7479                                   struct netlink_ext_ack *extack)
7480 {
7481         int err;
7482
7483         if (!reporter->ops->dump)
7484                 return 0;
7485
7486         if (reporter->dump_fmsg)
7487                 return 0;
7488
7489         reporter->dump_fmsg = devlink_fmsg_alloc();
7490         if (!reporter->dump_fmsg) {
7491                 err = -ENOMEM;
7492                 return err;
7493         }
7494
7495         err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
7496         if (err)
7497                 goto dump_err;
7498
7499         err = reporter->ops->dump(reporter, reporter->dump_fmsg,
7500                                   priv_ctx, extack);
7501         if (err)
7502                 goto dump_err;
7503
7504         err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
7505         if (err)
7506                 goto dump_err;
7507
7508         reporter->dump_ts = jiffies;
7509         reporter->dump_real_ts = ktime_get_real_ns();
7510
7511         return 0;
7512
7513 dump_err:
7514         devlink_health_dump_clear(reporter);
7515         return err;
7516 }
7517
7518 int devlink_health_report(struct devlink_health_reporter *reporter,
7519                           const char *msg, void *priv_ctx)
7520 {
7521         enum devlink_health_reporter_state prev_health_state;
7522         struct devlink *devlink = reporter->devlink;
7523         unsigned long recover_ts_threshold;
7524
7525         /* write a log message of the current error */
7526         WARN_ON(!msg);
7527         trace_devlink_health_report(devlink, reporter->ops->name, msg);
7528         reporter->error_count++;
7529         prev_health_state = reporter->health_state;
7530         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7531         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7532
7533         /* abort if the previous error wasn't recovered */
7534         recover_ts_threshold = reporter->last_recovery_ts +
7535                                msecs_to_jiffies(reporter->graceful_period);
7536         if (reporter->auto_recover &&
7537             (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
7538              (reporter->last_recovery_ts && reporter->recovery_count &&
7539               time_is_after_jiffies(recover_ts_threshold)))) {
7540                 trace_devlink_health_recover_aborted(devlink,
7541                                                      reporter->ops->name,
7542                                                      reporter->health_state,
7543                                                      jiffies -
7544                                                      reporter->last_recovery_ts);
7545                 return -ECANCELED;
7546         }
7547
7548         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7549
7550         if (reporter->auto_dump) {
7551                 mutex_lock(&reporter->dump_lock);
7552                 /* store current dump of current error, for later analysis */
7553                 devlink_health_do_dump(reporter, priv_ctx, NULL);
7554                 mutex_unlock(&reporter->dump_lock);
7555         }
7556
7557         if (reporter->auto_recover)
7558                 return devlink_health_reporter_recover(reporter,
7559                                                        priv_ctx, NULL);
7560
7561         return 0;
7562 }
7563 EXPORT_SYMBOL_GPL(devlink_health_report);
7564
7565 static struct devlink_health_reporter *
7566 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
7567                                        struct nlattr **attrs)
7568 {
7569         struct devlink_health_reporter *reporter;
7570         struct devlink_port *devlink_port;
7571         char *reporter_name;
7572
7573         if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
7574                 return NULL;
7575
7576         reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
7577         devlink_port = devlink_port_get_from_attrs(devlink, attrs);
7578         if (IS_ERR(devlink_port)) {
7579                 mutex_lock(&devlink->reporters_lock);
7580                 reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
7581                 if (reporter)
7582                         refcount_inc(&reporter->refcount);
7583                 mutex_unlock(&devlink->reporters_lock);
7584         } else {
7585                 mutex_lock(&devlink_port->reporters_lock);
7586                 reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
7587                 if (reporter)
7588                         refcount_inc(&reporter->refcount);
7589                 mutex_unlock(&devlink_port->reporters_lock);
7590         }
7591
7592         return reporter;
7593 }
7594
7595 static struct devlink_health_reporter *
7596 devlink_health_reporter_get_from_info(struct devlink *devlink,
7597                                       struct genl_info *info)
7598 {
7599         return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
7600 }
7601
7602 static struct devlink_health_reporter *
7603 devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
7604 {
7605         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
7606         struct devlink_health_reporter *reporter;
7607         struct nlattr **attrs = info->attrs;
7608         struct devlink *devlink;
7609
7610         mutex_lock(&devlink_mutex);
7611         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
7612         if (IS_ERR(devlink))
7613                 goto unlock;
7614
7615         reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
7616         devlink_put(devlink);
7617         mutex_unlock(&devlink_mutex);
7618         return reporter;
7619 unlock:
7620         mutex_unlock(&devlink_mutex);
7621         return NULL;
7622 }
7623
7624 void
7625 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
7626                                      enum devlink_health_reporter_state state)
7627 {
7628         if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
7629                     state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
7630                 return;
7631
7632         if (reporter->health_state == state)
7633                 return;
7634
7635         reporter->health_state = state;
7636         trace_devlink_health_reporter_state_update(reporter->devlink,
7637                                                    reporter->ops->name, state);
7638         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7639 }
7640 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
7641
7642 static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
7643                                                    struct genl_info *info)
7644 {
7645         struct devlink *devlink = info->user_ptr[0];
7646         struct devlink_health_reporter *reporter;
7647         struct sk_buff *msg;
7648         int err;
7649
7650         reporter = devlink_health_reporter_get_from_info(devlink, info);
7651         if (!reporter)
7652                 return -EINVAL;
7653
7654         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7655         if (!msg) {
7656                 err = -ENOMEM;
7657                 goto out;
7658         }
7659
7660         err = devlink_nl_health_reporter_fill(msg, reporter,
7661                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
7662                                               info->snd_portid, info->snd_seq,
7663                                               0);
7664         if (err) {
7665                 nlmsg_free(msg);
7666                 goto out;
7667         }
7668
7669         err = genlmsg_reply(msg, info);
7670 out:
7671         devlink_health_reporter_put(reporter);
7672         return err;
7673 }
7674
7675 static int
7676 devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
7677                                           struct netlink_callback *cb)
7678 {
7679         struct devlink_health_reporter *reporter;
7680         struct devlink_port *port;
7681         struct devlink *devlink;
7682         int start = cb->args[0];
7683         unsigned long index;
7684         int idx = 0;
7685         int err;
7686
7687         mutex_lock(&devlink_mutex);
7688         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7689                 if (!devlink_try_get(devlink))
7690                         continue;
7691
7692                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7693                         goto retry_rep;
7694
7695                 mutex_lock(&devlink->reporters_lock);
7696                 list_for_each_entry(reporter, &devlink->reporter_list,
7697                                     list) {
7698                         if (idx < start) {
7699                                 idx++;
7700                                 continue;
7701                         }
7702                         err = devlink_nl_health_reporter_fill(
7703                                 msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
7704                                 NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7705                                 NLM_F_MULTI);
7706                         if (err) {
7707                                 mutex_unlock(&devlink->reporters_lock);
7708                                 devlink_put(devlink);
7709                                 goto out;
7710                         }
7711                         idx++;
7712                 }
7713                 mutex_unlock(&devlink->reporters_lock);
7714 retry_rep:
7715                 devlink_put(devlink);
7716         }
7717
7718         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7719                 if (!devlink_try_get(devlink))
7720                         continue;
7721
7722                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7723                         goto retry_port;
7724
7725                 devl_lock(devlink);
7726                 list_for_each_entry(port, &devlink->port_list, list) {
7727                         mutex_lock(&port->reporters_lock);
7728                         list_for_each_entry(reporter, &port->reporter_list, list) {
7729                                 if (idx < start) {
7730                                         idx++;
7731                                         continue;
7732                                 }
7733                                 err = devlink_nl_health_reporter_fill(
7734                                         msg, reporter,
7735                                         DEVLINK_CMD_HEALTH_REPORTER_GET,
7736                                         NETLINK_CB(cb->skb).portid,
7737                                         cb->nlh->nlmsg_seq, NLM_F_MULTI);
7738                                 if (err) {
7739                                         mutex_unlock(&port->reporters_lock);
7740                                         devl_unlock(devlink);
7741                                         devlink_put(devlink);
7742                                         goto out;
7743                                 }
7744                                 idx++;
7745                         }
7746                         mutex_unlock(&port->reporters_lock);
7747                 }
7748                 devl_unlock(devlink);
7749 retry_port:
7750                 devlink_put(devlink);
7751         }
7752 out:
7753         mutex_unlock(&devlink_mutex);
7754
7755         cb->args[0] = idx;
7756         return msg->len;
7757 }
7758
7759 static int
7760 devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
7761                                         struct genl_info *info)
7762 {
7763         struct devlink *devlink = info->user_ptr[0];
7764         struct devlink_health_reporter *reporter;
7765         int err;
7766
7767         reporter = devlink_health_reporter_get_from_info(devlink, info);
7768         if (!reporter)
7769                 return -EINVAL;
7770
7771         if (!reporter->ops->recover &&
7772             (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
7773              info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
7774                 err = -EOPNOTSUPP;
7775                 goto out;
7776         }
7777         if (!reporter->ops->dump &&
7778             info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
7779                 err = -EOPNOTSUPP;
7780                 goto out;
7781         }
7782
7783         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
7784                 reporter->graceful_period =
7785                         nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
7786
7787         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
7788                 reporter->auto_recover =
7789                         nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
7790
7791         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
7792                 reporter->auto_dump =
7793                 nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
7794
7795         devlink_health_reporter_put(reporter);
7796         return 0;
7797 out:
7798         devlink_health_reporter_put(reporter);
7799         return err;
7800 }
7801
7802 static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
7803                                                        struct genl_info *info)
7804 {
7805         struct devlink *devlink = info->user_ptr[0];
7806         struct devlink_health_reporter *reporter;
7807         int err;
7808
7809         reporter = devlink_health_reporter_get_from_info(devlink, info);
7810         if (!reporter)
7811                 return -EINVAL;
7812
7813         err = devlink_health_reporter_recover(reporter, NULL, info->extack);
7814
7815         devlink_health_reporter_put(reporter);
7816         return err;
7817 }
7818
7819 static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
7820                                                         struct genl_info *info)
7821 {
7822         struct devlink *devlink = info->user_ptr[0];
7823         struct devlink_health_reporter *reporter;
7824         struct devlink_fmsg *fmsg;
7825         int err;
7826
7827         reporter = devlink_health_reporter_get_from_info(devlink, info);
7828         if (!reporter)
7829                 return -EINVAL;
7830
7831         if (!reporter->ops->diagnose) {
7832                 devlink_health_reporter_put(reporter);
7833                 return -EOPNOTSUPP;
7834         }
7835
7836         fmsg = devlink_fmsg_alloc();
7837         if (!fmsg) {
7838                 devlink_health_reporter_put(reporter);
7839                 return -ENOMEM;
7840         }
7841
7842         err = devlink_fmsg_obj_nest_start(fmsg);
7843         if (err)
7844                 goto out;
7845
7846         err = reporter->ops->diagnose(reporter, fmsg, info->extack);
7847         if (err)
7848                 goto out;
7849
7850         err = devlink_fmsg_obj_nest_end(fmsg);
7851         if (err)
7852                 goto out;
7853
7854         err = devlink_fmsg_snd(fmsg, info,
7855                                DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
7856
7857 out:
7858         devlink_fmsg_free(fmsg);
7859         devlink_health_reporter_put(reporter);
7860         return err;
7861 }
7862
7863 static int
7864 devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
7865                                                struct netlink_callback *cb)
7866 {
7867         struct devlink_health_reporter *reporter;
7868         u64 start = cb->args[0];
7869         int err;
7870
7871         reporter = devlink_health_reporter_get_from_cb(cb);
7872         if (!reporter)
7873                 return -EINVAL;
7874
7875         if (!reporter->ops->dump) {
7876                 err = -EOPNOTSUPP;
7877                 goto out;
7878         }
7879         mutex_lock(&reporter->dump_lock);
7880         if (!start) {
7881                 err = devlink_health_do_dump(reporter, NULL, cb->extack);
7882                 if (err)
7883                         goto unlock;
7884                 cb->args[1] = reporter->dump_ts;
7885         }
7886         if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
7887                 NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
7888                 err = -EAGAIN;
7889                 goto unlock;
7890         }
7891
7892         err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
7893                                   DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
7894 unlock:
7895         mutex_unlock(&reporter->dump_lock);
7896 out:
7897         devlink_health_reporter_put(reporter);
7898         return err;
7899 }
7900
7901 static int
7902 devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
7903                                                struct genl_info *info)
7904 {
7905         struct devlink *devlink = info->user_ptr[0];
7906         struct devlink_health_reporter *reporter;
7907
7908         reporter = devlink_health_reporter_get_from_info(devlink, info);
7909         if (!reporter)
7910                 return -EINVAL;
7911
7912         if (!reporter->ops->dump) {
7913                 devlink_health_reporter_put(reporter);
7914                 return -EOPNOTSUPP;
7915         }
7916
7917         mutex_lock(&reporter->dump_lock);
7918         devlink_health_dump_clear(reporter);
7919         mutex_unlock(&reporter->dump_lock);
7920         devlink_health_reporter_put(reporter);
7921         return 0;
7922 }
7923
7924 static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
7925                                                     struct genl_info *info)
7926 {
7927         struct devlink *devlink = info->user_ptr[0];
7928         struct devlink_health_reporter *reporter;
7929         int err;
7930
7931         reporter = devlink_health_reporter_get_from_info(devlink, info);
7932         if (!reporter)
7933                 return -EINVAL;
7934
7935         if (!reporter->ops->test) {
7936                 devlink_health_reporter_put(reporter);
7937                 return -EOPNOTSUPP;
7938         }
7939
7940         err = reporter->ops->test(reporter, info->extack);
7941
7942         devlink_health_reporter_put(reporter);
7943         return err;
7944 }
7945
7946 struct devlink_stats {
7947         u64_stats_t rx_bytes;
7948         u64_stats_t rx_packets;
7949         struct u64_stats_sync syncp;
7950 };
7951
7952 /**
7953  * struct devlink_trap_policer_item - Packet trap policer attributes.
7954  * @policer: Immutable packet trap policer attributes.
7955  * @rate: Rate in packets / sec.
7956  * @burst: Burst size in packets.
7957  * @list: trap_policer_list member.
7958  *
7959  * Describes packet trap policer attributes. Created by devlink during trap
7960  * policer registration.
7961  */
7962 struct devlink_trap_policer_item {
7963         const struct devlink_trap_policer *policer;
7964         u64 rate;
7965         u64 burst;
7966         struct list_head list;
7967 };
7968
7969 /**
7970  * struct devlink_trap_group_item - Packet trap group attributes.
7971  * @group: Immutable packet trap group attributes.
7972  * @policer_item: Associated policer item. Can be NULL.
7973  * @list: trap_group_list member.
7974  * @stats: Trap group statistics.
7975  *
7976  * Describes packet trap group attributes. Created by devlink during trap
7977  * group registration.
7978  */
7979 struct devlink_trap_group_item {
7980         const struct devlink_trap_group *group;
7981         struct devlink_trap_policer_item *policer_item;
7982         struct list_head list;
7983         struct devlink_stats __percpu *stats;
7984 };
7985
7986 /**
7987  * struct devlink_trap_item - Packet trap attributes.
7988  * @trap: Immutable packet trap attributes.
7989  * @group_item: Associated group item.
7990  * @list: trap_list member.
7991  * @action: Trap action.
7992  * @stats: Trap statistics.
7993  * @priv: Driver private information.
7994  *
7995  * Describes both mutable and immutable packet trap attributes. Created by
7996  * devlink during trap registration and used for all trap related operations.
7997  */
7998 struct devlink_trap_item {
7999         const struct devlink_trap *trap;
8000         struct devlink_trap_group_item *group_item;
8001         struct list_head list;
8002         enum devlink_trap_action action;
8003         struct devlink_stats __percpu *stats;
8004         void *priv;
8005 };
8006
8007 static struct devlink_trap_policer_item *
8008 devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
8009 {
8010         struct devlink_trap_policer_item *policer_item;
8011
8012         list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
8013                 if (policer_item->policer->id == id)
8014                         return policer_item;
8015         }
8016
8017         return NULL;
8018 }
8019
8020 static struct devlink_trap_item *
8021 devlink_trap_item_lookup(struct devlink *devlink, const char *name)
8022 {
8023         struct devlink_trap_item *trap_item;
8024
8025         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8026                 if (!strcmp(trap_item->trap->name, name))
8027                         return trap_item;
8028         }
8029
8030         return NULL;
8031 }
8032
8033 static struct devlink_trap_item *
8034 devlink_trap_item_get_from_info(struct devlink *devlink,
8035                                 struct genl_info *info)
8036 {
8037         struct nlattr *attr;
8038
8039         if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
8040                 return NULL;
8041         attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
8042
8043         return devlink_trap_item_lookup(devlink, nla_data(attr));
8044 }
8045
8046 static int
8047 devlink_trap_action_get_from_info(struct genl_info *info,
8048                                   enum devlink_trap_action *p_trap_action)
8049 {
8050         u8 val;
8051
8052         val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
8053         switch (val) {
8054         case DEVLINK_TRAP_ACTION_DROP:
8055         case DEVLINK_TRAP_ACTION_TRAP:
8056         case DEVLINK_TRAP_ACTION_MIRROR:
8057                 *p_trap_action = val;
8058                 break;
8059         default:
8060                 return -EINVAL;
8061         }
8062
8063         return 0;
8064 }
8065
8066 static int devlink_trap_metadata_put(struct sk_buff *msg,
8067                                      const struct devlink_trap *trap)
8068 {
8069         struct nlattr *attr;
8070
8071         attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
8072         if (!attr)
8073                 return -EMSGSIZE;
8074
8075         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
8076             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
8077                 goto nla_put_failure;
8078         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
8079             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
8080                 goto nla_put_failure;
8081
8082         nla_nest_end(msg, attr);
8083
8084         return 0;
8085
8086 nla_put_failure:
8087         nla_nest_cancel(msg, attr);
8088         return -EMSGSIZE;
8089 }
8090
8091 static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
8092                                     struct devlink_stats *stats)
8093 {
8094         int i;
8095
8096         memset(stats, 0, sizeof(*stats));
8097         for_each_possible_cpu(i) {
8098                 struct devlink_stats *cpu_stats;
8099                 u64 rx_packets, rx_bytes;
8100                 unsigned int start;
8101
8102                 cpu_stats = per_cpu_ptr(trap_stats, i);
8103                 do {
8104                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
8105                         rx_packets = u64_stats_read(&cpu_stats->rx_packets);
8106                         rx_bytes = u64_stats_read(&cpu_stats->rx_bytes);
8107                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
8108
8109                 u64_stats_add(&stats->rx_packets, rx_packets);
8110                 u64_stats_add(&stats->rx_bytes, rx_bytes);
8111         }
8112 }
8113
8114 static int
8115 devlink_trap_group_stats_put(struct sk_buff *msg,
8116                              struct devlink_stats __percpu *trap_stats)
8117 {
8118         struct devlink_stats stats;
8119         struct nlattr *attr;
8120
8121         devlink_trap_stats_read(trap_stats, &stats);
8122
8123         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8124         if (!attr)
8125                 return -EMSGSIZE;
8126
8127         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
8128                               u64_stats_read(&stats.rx_packets),
8129                               DEVLINK_ATTR_PAD))
8130                 goto nla_put_failure;
8131
8132         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
8133                               u64_stats_read(&stats.rx_bytes),
8134                               DEVLINK_ATTR_PAD))
8135                 goto nla_put_failure;
8136
8137         nla_nest_end(msg, attr);
8138
8139         return 0;
8140
8141 nla_put_failure:
8142         nla_nest_cancel(msg, attr);
8143         return -EMSGSIZE;
8144 }
8145
8146 static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
8147                                   const struct devlink_trap_item *trap_item)
8148 {
8149         struct devlink_stats stats;
8150         struct nlattr *attr;
8151         u64 drops = 0;
8152         int err;
8153
8154         if (devlink->ops->trap_drop_counter_get) {
8155                 err = devlink->ops->trap_drop_counter_get(devlink,
8156                                                           trap_item->trap,
8157                                                           &drops);
8158                 if (err)
8159                         return err;
8160         }
8161
8162         devlink_trap_stats_read(trap_item->stats, &stats);
8163
8164         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8165         if (!attr)
8166                 return -EMSGSIZE;
8167
8168         if (devlink->ops->trap_drop_counter_get &&
8169             nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8170                               DEVLINK_ATTR_PAD))
8171                 goto nla_put_failure;
8172
8173         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
8174                               u64_stats_read(&stats.rx_packets),
8175                               DEVLINK_ATTR_PAD))
8176                 goto nla_put_failure;
8177
8178         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
8179                               u64_stats_read(&stats.rx_bytes),
8180                               DEVLINK_ATTR_PAD))
8181                 goto nla_put_failure;
8182
8183         nla_nest_end(msg, attr);
8184
8185         return 0;
8186
8187 nla_put_failure:
8188         nla_nest_cancel(msg, attr);
8189         return -EMSGSIZE;
8190 }
8191
8192 static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
8193                                 const struct devlink_trap_item *trap_item,
8194                                 enum devlink_command cmd, u32 portid, u32 seq,
8195                                 int flags)
8196 {
8197         struct devlink_trap_group_item *group_item = trap_item->group_item;
8198         void *hdr;
8199         int err;
8200
8201         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8202         if (!hdr)
8203                 return -EMSGSIZE;
8204
8205         if (devlink_nl_put_handle(msg, devlink))
8206                 goto nla_put_failure;
8207
8208         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
8209                            group_item->group->name))
8210                 goto nla_put_failure;
8211
8212         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
8213                 goto nla_put_failure;
8214
8215         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
8216                 goto nla_put_failure;
8217
8218         if (trap_item->trap->generic &&
8219             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
8220                 goto nla_put_failure;
8221
8222         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
8223                 goto nla_put_failure;
8224
8225         err = devlink_trap_metadata_put(msg, trap_item->trap);
8226         if (err)
8227                 goto nla_put_failure;
8228
8229         err = devlink_trap_stats_put(msg, devlink, trap_item);
8230         if (err)
8231                 goto nla_put_failure;
8232
8233         genlmsg_end(msg, hdr);
8234
8235         return 0;
8236
8237 nla_put_failure:
8238         genlmsg_cancel(msg, hdr);
8239         return -EMSGSIZE;
8240 }
8241
8242 static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
8243                                         struct genl_info *info)
8244 {
8245         struct netlink_ext_ack *extack = info->extack;
8246         struct devlink *devlink = info->user_ptr[0];
8247         struct devlink_trap_item *trap_item;
8248         struct sk_buff *msg;
8249         int err;
8250
8251         if (list_empty(&devlink->trap_list))
8252                 return -EOPNOTSUPP;
8253
8254         trap_item = devlink_trap_item_get_from_info(devlink, info);
8255         if (!trap_item) {
8256                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
8257                 return -ENOENT;
8258         }
8259
8260         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8261         if (!msg)
8262                 return -ENOMEM;
8263
8264         err = devlink_nl_trap_fill(msg, devlink, trap_item,
8265                                    DEVLINK_CMD_TRAP_NEW, info->snd_portid,
8266                                    info->snd_seq, 0);
8267         if (err)
8268                 goto err_trap_fill;
8269
8270         return genlmsg_reply(msg, info);
8271
8272 err_trap_fill:
8273         nlmsg_free(msg);
8274         return err;
8275 }
8276
8277 static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
8278                                           struct netlink_callback *cb)
8279 {
8280         struct devlink_trap_item *trap_item;
8281         struct devlink *devlink;
8282         int start = cb->args[0];
8283         unsigned long index;
8284         int idx = 0;
8285         int err;
8286
8287         mutex_lock(&devlink_mutex);
8288         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8289                 if (!devlink_try_get(devlink))
8290                         continue;
8291
8292                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8293                         goto retry;
8294
8295                 devl_lock(devlink);
8296                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8297                         if (idx < start) {
8298                                 idx++;
8299                                 continue;
8300                         }
8301                         err = devlink_nl_trap_fill(msg, devlink, trap_item,
8302                                                    DEVLINK_CMD_TRAP_NEW,
8303                                                    NETLINK_CB(cb->skb).portid,
8304                                                    cb->nlh->nlmsg_seq,
8305                                                    NLM_F_MULTI);
8306                         if (err) {
8307                                 devl_unlock(devlink);
8308                                 devlink_put(devlink);
8309                                 goto out;
8310                         }
8311                         idx++;
8312                 }
8313                 devl_unlock(devlink);
8314 retry:
8315                 devlink_put(devlink);
8316         }
8317 out:
8318         mutex_unlock(&devlink_mutex);
8319
8320         cb->args[0] = idx;
8321         return msg->len;
8322 }
8323
8324 static int __devlink_trap_action_set(struct devlink *devlink,
8325                                      struct devlink_trap_item *trap_item,
8326                                      enum devlink_trap_action trap_action,
8327                                      struct netlink_ext_ack *extack)
8328 {
8329         int err;
8330
8331         if (trap_item->action != trap_action &&
8332             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
8333                 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
8334                 return 0;
8335         }
8336
8337         err = devlink->ops->trap_action_set(devlink, trap_item->trap,
8338                                             trap_action, extack);
8339         if (err)
8340                 return err;
8341
8342         trap_item->action = trap_action;
8343
8344         return 0;
8345 }
8346
8347 static int devlink_trap_action_set(struct devlink *devlink,
8348                                    struct devlink_trap_item *trap_item,
8349                                    struct genl_info *info)
8350 {
8351         enum devlink_trap_action trap_action;
8352         int err;
8353
8354         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8355                 return 0;
8356
8357         err = devlink_trap_action_get_from_info(info, &trap_action);
8358         if (err) {
8359                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8360                 return -EINVAL;
8361         }
8362
8363         return __devlink_trap_action_set(devlink, trap_item, trap_action,
8364                                          info->extack);
8365 }
8366
8367 static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
8368                                         struct genl_info *info)
8369 {
8370         struct netlink_ext_ack *extack = info->extack;
8371         struct devlink *devlink = info->user_ptr[0];
8372         struct devlink_trap_item *trap_item;
8373
8374         if (list_empty(&devlink->trap_list))
8375                 return -EOPNOTSUPP;
8376
8377         trap_item = devlink_trap_item_get_from_info(devlink, info);
8378         if (!trap_item) {
8379                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
8380                 return -ENOENT;
8381         }
8382
8383         return devlink_trap_action_set(devlink, trap_item, info);
8384 }
8385
8386 static struct devlink_trap_group_item *
8387 devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
8388 {
8389         struct devlink_trap_group_item *group_item;
8390
8391         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
8392                 if (!strcmp(group_item->group->name, name))
8393                         return group_item;
8394         }
8395
8396         return NULL;
8397 }
8398
8399 static struct devlink_trap_group_item *
8400 devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
8401 {
8402         struct devlink_trap_group_item *group_item;
8403
8404         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
8405                 if (group_item->group->id == id)
8406                         return group_item;
8407         }
8408
8409         return NULL;
8410 }
8411
8412 static struct devlink_trap_group_item *
8413 devlink_trap_group_item_get_from_info(struct devlink *devlink,
8414                                       struct genl_info *info)
8415 {
8416         char *name;
8417
8418         if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
8419                 return NULL;
8420         name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
8421
8422         return devlink_trap_group_item_lookup(devlink, name);
8423 }
8424
8425 static int
8426 devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
8427                            const struct devlink_trap_group_item *group_item,
8428                            enum devlink_command cmd, u32 portid, u32 seq,
8429                            int flags)
8430 {
8431         void *hdr;
8432         int err;
8433
8434         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8435         if (!hdr)
8436                 return -EMSGSIZE;
8437
8438         if (devlink_nl_put_handle(msg, devlink))
8439                 goto nla_put_failure;
8440
8441         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
8442                            group_item->group->name))
8443                 goto nla_put_failure;
8444
8445         if (group_item->group->generic &&
8446             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
8447                 goto nla_put_failure;
8448
8449         if (group_item->policer_item &&
8450             nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8451                         group_item->policer_item->policer->id))
8452                 goto nla_put_failure;
8453
8454         err = devlink_trap_group_stats_put(msg, group_item->stats);
8455         if (err)
8456                 goto nla_put_failure;
8457
8458         genlmsg_end(msg, hdr);
8459
8460         return 0;
8461
8462 nla_put_failure:
8463         genlmsg_cancel(msg, hdr);
8464         return -EMSGSIZE;
8465 }
8466
8467 static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
8468                                               struct genl_info *info)
8469 {
8470         struct netlink_ext_ack *extack = info->extack;
8471         struct devlink *devlink = info->user_ptr[0];
8472         struct devlink_trap_group_item *group_item;
8473         struct sk_buff *msg;
8474         int err;
8475
8476         if (list_empty(&devlink->trap_group_list))
8477                 return -EOPNOTSUPP;
8478
8479         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8480         if (!group_item) {
8481                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8482                 return -ENOENT;
8483         }
8484
8485         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8486         if (!msg)
8487                 return -ENOMEM;
8488
8489         err = devlink_nl_trap_group_fill(msg, devlink, group_item,
8490                                          DEVLINK_CMD_TRAP_GROUP_NEW,
8491                                          info->snd_portid, info->snd_seq, 0);
8492         if (err)
8493                 goto err_trap_group_fill;
8494
8495         return genlmsg_reply(msg, info);
8496
8497 err_trap_group_fill:
8498         nlmsg_free(msg);
8499         return err;
8500 }
8501
8502 static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
8503                                                 struct netlink_callback *cb)
8504 {
8505         enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
8506         struct devlink_trap_group_item *group_item;
8507         u32 portid = NETLINK_CB(cb->skb).portid;
8508         struct devlink *devlink;
8509         int start = cb->args[0];
8510         unsigned long index;
8511         int idx = 0;
8512         int err;
8513
8514         mutex_lock(&devlink_mutex);
8515         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8516                 if (!devlink_try_get(devlink))
8517                         continue;
8518
8519                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8520                         goto retry;
8521
8522                 devl_lock(devlink);
8523                 list_for_each_entry(group_item, &devlink->trap_group_list,
8524                                     list) {
8525                         if (idx < start) {
8526                                 idx++;
8527                                 continue;
8528                         }
8529                         err = devlink_nl_trap_group_fill(msg, devlink,
8530                                                          group_item, cmd,
8531                                                          portid,
8532                                                          cb->nlh->nlmsg_seq,
8533                                                          NLM_F_MULTI);
8534                         if (err) {
8535                                 devl_unlock(devlink);
8536                                 devlink_put(devlink);
8537                                 goto out;
8538                         }
8539                         idx++;
8540                 }
8541                 devl_unlock(devlink);
8542 retry:
8543                 devlink_put(devlink);
8544         }
8545 out:
8546         mutex_unlock(&devlink_mutex);
8547
8548         cb->args[0] = idx;
8549         return msg->len;
8550 }
8551
8552 static int
8553 __devlink_trap_group_action_set(struct devlink *devlink,
8554                                 struct devlink_trap_group_item *group_item,
8555                                 enum devlink_trap_action trap_action,
8556                                 struct netlink_ext_ack *extack)
8557 {
8558         const char *group_name = group_item->group->name;
8559         struct devlink_trap_item *trap_item;
8560         int err;
8561
8562         if (devlink->ops->trap_group_action_set) {
8563                 err = devlink->ops->trap_group_action_set(devlink, group_item->group,
8564                                                           trap_action, extack);
8565                 if (err)
8566                         return err;
8567
8568                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8569                         if (strcmp(trap_item->group_item->group->name, group_name))
8570                                 continue;
8571                         if (trap_item->action != trap_action &&
8572                             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
8573                                 continue;
8574                         trap_item->action = trap_action;
8575                 }
8576
8577                 return 0;
8578         }
8579
8580         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8581                 if (strcmp(trap_item->group_item->group->name, group_name))
8582                         continue;
8583                 err = __devlink_trap_action_set(devlink, trap_item,
8584                                                 trap_action, extack);
8585                 if (err)
8586                         return err;
8587         }
8588
8589         return 0;
8590 }
8591
8592 static int
8593 devlink_trap_group_action_set(struct devlink *devlink,
8594                               struct devlink_trap_group_item *group_item,
8595                               struct genl_info *info, bool *p_modified)
8596 {
8597         enum devlink_trap_action trap_action;
8598         int err;
8599
8600         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8601                 return 0;
8602
8603         err = devlink_trap_action_get_from_info(info, &trap_action);
8604         if (err) {
8605                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8606                 return -EINVAL;
8607         }
8608
8609         err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
8610                                               info->extack);
8611         if (err)
8612                 return err;
8613
8614         *p_modified = true;
8615
8616         return 0;
8617 }
8618
8619 static int devlink_trap_group_set(struct devlink *devlink,
8620                                   struct devlink_trap_group_item *group_item,
8621                                   struct genl_info *info)
8622 {
8623         struct devlink_trap_policer_item *policer_item;
8624         struct netlink_ext_ack *extack = info->extack;
8625         const struct devlink_trap_policer *policer;
8626         struct nlattr **attrs = info->attrs;
8627         int err;
8628
8629         if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8630                 return 0;
8631
8632         if (!devlink->ops->trap_group_set)
8633                 return -EOPNOTSUPP;
8634
8635         policer_item = group_item->policer_item;
8636         if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
8637                 u32 policer_id;
8638
8639                 policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8640                 policer_item = devlink_trap_policer_item_lookup(devlink,
8641                                                                 policer_id);
8642                 if (policer_id && !policer_item) {
8643                         NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8644                         return -ENOENT;
8645                 }
8646         }
8647         policer = policer_item ? policer_item->policer : NULL;
8648
8649         err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
8650                                            extack);
8651         if (err)
8652                 return err;
8653
8654         group_item->policer_item = policer_item;
8655
8656         return 0;
8657 }
8658
8659 static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
8660                                               struct genl_info *info)
8661 {
8662         struct netlink_ext_ack *extack = info->extack;
8663         struct devlink *devlink = info->user_ptr[0];
8664         struct devlink_trap_group_item *group_item;
8665         bool modified = false;
8666         int err;
8667
8668         if (list_empty(&devlink->trap_group_list))
8669                 return -EOPNOTSUPP;
8670
8671         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8672         if (!group_item) {
8673                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8674                 return -ENOENT;
8675         }
8676
8677         err = devlink_trap_group_action_set(devlink, group_item, info,
8678                                             &modified);
8679         if (err)
8680                 return err;
8681
8682         err = devlink_trap_group_set(devlink, group_item, info);
8683         if (err)
8684                 goto err_trap_group_set;
8685
8686         return 0;
8687
8688 err_trap_group_set:
8689         if (modified)
8690                 NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
8691         return err;
8692 }
8693
8694 static struct devlink_trap_policer_item *
8695 devlink_trap_policer_item_get_from_info(struct devlink *devlink,
8696                                         struct genl_info *info)
8697 {
8698         u32 id;
8699
8700         if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8701                 return NULL;
8702         id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8703
8704         return devlink_trap_policer_item_lookup(devlink, id);
8705 }
8706
8707 static int
8708 devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
8709                                const struct devlink_trap_policer *policer)
8710 {
8711         struct nlattr *attr;
8712         u64 drops;
8713         int err;
8714
8715         if (!devlink->ops->trap_policer_counter_get)
8716                 return 0;
8717
8718         err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
8719         if (err)
8720                 return err;
8721
8722         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8723         if (!attr)
8724                 return -EMSGSIZE;
8725
8726         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8727                               DEVLINK_ATTR_PAD))
8728                 goto nla_put_failure;
8729
8730         nla_nest_end(msg, attr);
8731
8732         return 0;
8733
8734 nla_put_failure:
8735         nla_nest_cancel(msg, attr);
8736         return -EMSGSIZE;
8737 }
8738
8739 static int
8740 devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
8741                              const struct devlink_trap_policer_item *policer_item,
8742                              enum devlink_command cmd, u32 portid, u32 seq,
8743                              int flags)
8744 {
8745         void *hdr;
8746         int err;
8747
8748         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8749         if (!hdr)
8750                 return -EMSGSIZE;
8751
8752         if (devlink_nl_put_handle(msg, devlink))
8753                 goto nla_put_failure;
8754
8755         if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8756                         policer_item->policer->id))
8757                 goto nla_put_failure;
8758
8759         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
8760                               policer_item->rate, DEVLINK_ATTR_PAD))
8761                 goto nla_put_failure;
8762
8763         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
8764                               policer_item->burst, DEVLINK_ATTR_PAD))
8765                 goto nla_put_failure;
8766
8767         err = devlink_trap_policer_stats_put(msg, devlink,
8768                                              policer_item->policer);
8769         if (err)
8770                 goto nla_put_failure;
8771
8772         genlmsg_end(msg, hdr);
8773
8774         return 0;
8775
8776 nla_put_failure:
8777         genlmsg_cancel(msg, hdr);
8778         return -EMSGSIZE;
8779 }
8780
8781 static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
8782                                                 struct genl_info *info)
8783 {
8784         struct devlink_trap_policer_item *policer_item;
8785         struct netlink_ext_ack *extack = info->extack;
8786         struct devlink *devlink = info->user_ptr[0];
8787         struct sk_buff *msg;
8788         int err;
8789
8790         if (list_empty(&devlink->trap_policer_list))
8791                 return -EOPNOTSUPP;
8792
8793         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8794         if (!policer_item) {
8795                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8796                 return -ENOENT;
8797         }
8798
8799         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8800         if (!msg)
8801                 return -ENOMEM;
8802
8803         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
8804                                            DEVLINK_CMD_TRAP_POLICER_NEW,
8805                                            info->snd_portid, info->snd_seq, 0);
8806         if (err)
8807                 goto err_trap_policer_fill;
8808
8809         return genlmsg_reply(msg, info);
8810
8811 err_trap_policer_fill:
8812         nlmsg_free(msg);
8813         return err;
8814 }
8815
8816 static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
8817                                                   struct netlink_callback *cb)
8818 {
8819         enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
8820         struct devlink_trap_policer_item *policer_item;
8821         u32 portid = NETLINK_CB(cb->skb).portid;
8822         struct devlink *devlink;
8823         int start = cb->args[0];
8824         unsigned long index;
8825         int idx = 0;
8826         int err;
8827
8828         mutex_lock(&devlink_mutex);
8829         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8830                 if (!devlink_try_get(devlink))
8831                         continue;
8832
8833                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8834                         goto retry;
8835
8836                 devl_lock(devlink);
8837                 list_for_each_entry(policer_item, &devlink->trap_policer_list,
8838                                     list) {
8839                         if (idx < start) {
8840                                 idx++;
8841                                 continue;
8842                         }
8843                         err = devlink_nl_trap_policer_fill(msg, devlink,
8844                                                            policer_item, cmd,
8845                                                            portid,
8846                                                            cb->nlh->nlmsg_seq,
8847                                                            NLM_F_MULTI);
8848                         if (err) {
8849                                 devl_unlock(devlink);
8850                                 devlink_put(devlink);
8851                                 goto out;
8852                         }
8853                         idx++;
8854                 }
8855                 devl_unlock(devlink);
8856 retry:
8857                 devlink_put(devlink);
8858         }
8859 out:
8860         mutex_unlock(&devlink_mutex);
8861
8862         cb->args[0] = idx;
8863         return msg->len;
8864 }
8865
8866 static int
8867 devlink_trap_policer_set(struct devlink *devlink,
8868                          struct devlink_trap_policer_item *policer_item,
8869                          struct genl_info *info)
8870 {
8871         struct netlink_ext_ack *extack = info->extack;
8872         struct nlattr **attrs = info->attrs;
8873         u64 rate, burst;
8874         int err;
8875
8876         rate = policer_item->rate;
8877         burst = policer_item->burst;
8878
8879         if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
8880                 rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
8881
8882         if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
8883                 burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
8884
8885         if (rate < policer_item->policer->min_rate) {
8886                 NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
8887                 return -EINVAL;
8888         }
8889
8890         if (rate > policer_item->policer->max_rate) {
8891                 NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
8892                 return -EINVAL;
8893         }
8894
8895         if (burst < policer_item->policer->min_burst) {
8896                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
8897                 return -EINVAL;
8898         }
8899
8900         if (burst > policer_item->policer->max_burst) {
8901                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
8902                 return -EINVAL;
8903         }
8904
8905         err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
8906                                              rate, burst, info->extack);
8907         if (err)
8908                 return err;
8909
8910         policer_item->rate = rate;
8911         policer_item->burst = burst;
8912
8913         return 0;
8914 }
8915
8916 static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
8917                                                 struct genl_info *info)
8918 {
8919         struct devlink_trap_policer_item *policer_item;
8920         struct netlink_ext_ack *extack = info->extack;
8921         struct devlink *devlink = info->user_ptr[0];
8922
8923         if (list_empty(&devlink->trap_policer_list))
8924                 return -EOPNOTSUPP;
8925
8926         if (!devlink->ops->trap_policer_set)
8927                 return -EOPNOTSUPP;
8928
8929         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8930         if (!policer_item) {
8931                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8932                 return -ENOENT;
8933         }
8934
8935         return devlink_trap_policer_set(devlink, policer_item, info);
8936 }
8937
8938 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
8939         [DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
8940                 DEVLINK_ATTR_TRAP_POLICER_ID },
8941         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
8942         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
8943         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
8944         [DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
8945                                                     DEVLINK_PORT_TYPE_IB),
8946         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
8947         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
8948         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
8949         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
8950         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
8951         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
8952         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
8953         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
8954         [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
8955                                                        DEVLINK_ESWITCH_MODE_SWITCHDEV),
8956         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
8957         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
8958         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
8959         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
8960         [DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
8961         [DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
8962         [DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
8963         [DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
8964         [DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
8965         [DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
8966         [DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
8967         [DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
8968         [DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
8969         [DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
8970         [DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
8971         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
8972         [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
8973         [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
8974         [DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
8975                 NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
8976         [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
8977         [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
8978         [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
8979         [DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
8980         [DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
8981         [DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
8982         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
8983         [DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
8984         [DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
8985         [DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
8986         [DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
8987         [DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
8988                                                         DEVLINK_RELOAD_ACTION_MAX),
8989         [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
8990         [DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
8991         [DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
8992         [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
8993         [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
8994         [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
8995         [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
8996         [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
8997         [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
8998         [DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
8999         [DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 },
9000         [DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING },
9001 };
9002
9003 static const struct genl_small_ops devlink_nl_ops[] = {
9004         {
9005                 .cmd = DEVLINK_CMD_GET,
9006                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9007                 .doit = devlink_nl_cmd_get_doit,
9008                 .dumpit = devlink_nl_cmd_get_dumpit,
9009                 /* can be retrieved by unprivileged users */
9010         },
9011         {
9012                 .cmd = DEVLINK_CMD_PORT_GET,
9013                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9014                 .doit = devlink_nl_cmd_port_get_doit,
9015                 .dumpit = devlink_nl_cmd_port_get_dumpit,
9016                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9017                 /* can be retrieved by unprivileged users */
9018         },
9019         {
9020                 .cmd = DEVLINK_CMD_PORT_SET,
9021                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9022                 .doit = devlink_nl_cmd_port_set_doit,
9023                 .flags = GENL_ADMIN_PERM,
9024                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9025         },
9026         {
9027                 .cmd = DEVLINK_CMD_RATE_GET,
9028                 .doit = devlink_nl_cmd_rate_get_doit,
9029                 .dumpit = devlink_nl_cmd_rate_get_dumpit,
9030                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
9031                 /* can be retrieved by unprivileged users */
9032         },
9033         {
9034                 .cmd = DEVLINK_CMD_RATE_SET,
9035                 .doit = devlink_nl_cmd_rate_set_doit,
9036                 .flags = GENL_ADMIN_PERM,
9037                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
9038         },
9039         {
9040                 .cmd = DEVLINK_CMD_RATE_NEW,
9041                 .doit = devlink_nl_cmd_rate_new_doit,
9042                 .flags = GENL_ADMIN_PERM,
9043         },
9044         {
9045                 .cmd = DEVLINK_CMD_RATE_DEL,
9046                 .doit = devlink_nl_cmd_rate_del_doit,
9047                 .flags = GENL_ADMIN_PERM,
9048                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
9049         },
9050         {
9051                 .cmd = DEVLINK_CMD_PORT_SPLIT,
9052                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9053                 .doit = devlink_nl_cmd_port_split_doit,
9054                 .flags = GENL_ADMIN_PERM,
9055                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9056         },
9057         {
9058                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
9059                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9060                 .doit = devlink_nl_cmd_port_unsplit_doit,
9061                 .flags = GENL_ADMIN_PERM,
9062                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9063         },
9064         {
9065                 .cmd = DEVLINK_CMD_PORT_NEW,
9066                 .doit = devlink_nl_cmd_port_new_doit,
9067                 .flags = GENL_ADMIN_PERM,
9068         },
9069         {
9070                 .cmd = DEVLINK_CMD_PORT_DEL,
9071                 .doit = devlink_nl_cmd_port_del_doit,
9072                 .flags = GENL_ADMIN_PERM,
9073         },
9074         {
9075                 .cmd = DEVLINK_CMD_LINECARD_GET,
9076                 .doit = devlink_nl_cmd_linecard_get_doit,
9077                 .dumpit = devlink_nl_cmd_linecard_get_dumpit,
9078                 .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
9079                 /* can be retrieved by unprivileged users */
9080         },
9081         {
9082                 .cmd = DEVLINK_CMD_LINECARD_SET,
9083                 .doit = devlink_nl_cmd_linecard_set_doit,
9084                 .flags = GENL_ADMIN_PERM,
9085                 .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
9086         },
9087         {
9088                 .cmd = DEVLINK_CMD_SB_GET,
9089                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9090                 .doit = devlink_nl_cmd_sb_get_doit,
9091                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
9092                 /* can be retrieved by unprivileged users */
9093         },
9094         {
9095                 .cmd = DEVLINK_CMD_SB_POOL_GET,
9096                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9097                 .doit = devlink_nl_cmd_sb_pool_get_doit,
9098                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
9099                 /* can be retrieved by unprivileged users */
9100         },
9101         {
9102                 .cmd = DEVLINK_CMD_SB_POOL_SET,
9103                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9104                 .doit = devlink_nl_cmd_sb_pool_set_doit,
9105                 .flags = GENL_ADMIN_PERM,
9106         },
9107         {
9108                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
9109                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9110                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
9111                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
9112                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9113                 /* can be retrieved by unprivileged users */
9114         },
9115         {
9116                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
9117                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9118                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
9119                 .flags = GENL_ADMIN_PERM,
9120                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9121         },
9122         {
9123                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
9124                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9125                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
9126                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
9127                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9128                 /* can be retrieved by unprivileged users */
9129         },
9130         {
9131                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
9132                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9133                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
9134                 .flags = GENL_ADMIN_PERM,
9135                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9136         },
9137         {
9138                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
9139                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9140                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
9141                 .flags = GENL_ADMIN_PERM,
9142         },
9143         {
9144                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
9145                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9146                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
9147                 .flags = GENL_ADMIN_PERM,
9148         },
9149         {
9150                 .cmd = DEVLINK_CMD_ESWITCH_GET,
9151                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9152                 .doit = devlink_nl_cmd_eswitch_get_doit,
9153                 .flags = GENL_ADMIN_PERM,
9154         },
9155         {
9156                 .cmd = DEVLINK_CMD_ESWITCH_SET,
9157                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9158                 .doit = devlink_nl_cmd_eswitch_set_doit,
9159                 .flags = GENL_ADMIN_PERM,
9160         },
9161         {
9162                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
9163                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9164                 .doit = devlink_nl_cmd_dpipe_table_get,
9165                 /* can be retrieved by unprivileged users */
9166         },
9167         {
9168                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
9169                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9170                 .doit = devlink_nl_cmd_dpipe_entries_get,
9171                 /* can be retrieved by unprivileged users */
9172         },
9173         {
9174                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
9175                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9176                 .doit = devlink_nl_cmd_dpipe_headers_get,
9177                 /* can be retrieved by unprivileged users */
9178         },
9179         {
9180                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
9181                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9182                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
9183                 .flags = GENL_ADMIN_PERM,
9184         },
9185         {
9186                 .cmd = DEVLINK_CMD_RESOURCE_SET,
9187                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9188                 .doit = devlink_nl_cmd_resource_set,
9189                 .flags = GENL_ADMIN_PERM,
9190         },
9191         {
9192                 .cmd = DEVLINK_CMD_RESOURCE_DUMP,
9193                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9194                 .doit = devlink_nl_cmd_resource_dump,
9195                 /* can be retrieved by unprivileged users */
9196         },
9197         {
9198                 .cmd = DEVLINK_CMD_RELOAD,
9199                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9200                 .doit = devlink_nl_cmd_reload,
9201                 .flags = GENL_ADMIN_PERM,
9202                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
9203         },
9204         {
9205                 .cmd = DEVLINK_CMD_PARAM_GET,
9206                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9207                 .doit = devlink_nl_cmd_param_get_doit,
9208                 .dumpit = devlink_nl_cmd_param_get_dumpit,
9209                 /* can be retrieved by unprivileged users */
9210         },
9211         {
9212                 .cmd = DEVLINK_CMD_PARAM_SET,
9213                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9214                 .doit = devlink_nl_cmd_param_set_doit,
9215                 .flags = GENL_ADMIN_PERM,
9216         },
9217         {
9218                 .cmd = DEVLINK_CMD_PORT_PARAM_GET,
9219                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9220                 .doit = devlink_nl_cmd_port_param_get_doit,
9221                 .dumpit = devlink_nl_cmd_port_param_get_dumpit,
9222                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9223                 /* can be retrieved by unprivileged users */
9224         },
9225         {
9226                 .cmd = DEVLINK_CMD_PORT_PARAM_SET,
9227                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9228                 .doit = devlink_nl_cmd_port_param_set_doit,
9229                 .flags = GENL_ADMIN_PERM,
9230                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
9231         },
9232         {
9233                 .cmd = DEVLINK_CMD_REGION_GET,
9234                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9235                 .doit = devlink_nl_cmd_region_get_doit,
9236                 .dumpit = devlink_nl_cmd_region_get_dumpit,
9237                 .flags = GENL_ADMIN_PERM,
9238         },
9239         {
9240                 .cmd = DEVLINK_CMD_REGION_NEW,
9241                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9242                 .doit = devlink_nl_cmd_region_new,
9243                 .flags = GENL_ADMIN_PERM,
9244         },
9245         {
9246                 .cmd = DEVLINK_CMD_REGION_DEL,
9247                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9248                 .doit = devlink_nl_cmd_region_del,
9249                 .flags = GENL_ADMIN_PERM,
9250         },
9251         {
9252                 .cmd = DEVLINK_CMD_REGION_READ,
9253                 .validate = GENL_DONT_VALIDATE_STRICT |
9254                             GENL_DONT_VALIDATE_DUMP_STRICT,
9255                 .dumpit = devlink_nl_cmd_region_read_dumpit,
9256                 .flags = GENL_ADMIN_PERM,
9257         },
9258         {
9259                 .cmd = DEVLINK_CMD_INFO_GET,
9260                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9261                 .doit = devlink_nl_cmd_info_get_doit,
9262                 .dumpit = devlink_nl_cmd_info_get_dumpit,
9263                 /* can be retrieved by unprivileged users */
9264         },
9265         {
9266                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
9267                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9268                 .doit = devlink_nl_cmd_health_reporter_get_doit,
9269                 .dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
9270                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9271                                   DEVLINK_NL_FLAG_NO_LOCK,
9272                 /* can be retrieved by unprivileged users */
9273         },
9274         {
9275                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
9276                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9277                 .doit = devlink_nl_cmd_health_reporter_set_doit,
9278                 .flags = GENL_ADMIN_PERM,
9279                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9280                                   DEVLINK_NL_FLAG_NO_LOCK,
9281         },
9282         {
9283                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
9284                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9285                 .doit = devlink_nl_cmd_health_reporter_recover_doit,
9286                 .flags = GENL_ADMIN_PERM,
9287                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9288                                   DEVLINK_NL_FLAG_NO_LOCK,
9289         },
9290         {
9291                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
9292                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9293                 .doit = devlink_nl_cmd_health_reporter_diagnose_doit,
9294                 .flags = GENL_ADMIN_PERM,
9295                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9296                                   DEVLINK_NL_FLAG_NO_LOCK,
9297         },
9298         {
9299                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
9300                 .validate = GENL_DONT_VALIDATE_STRICT |
9301                             GENL_DONT_VALIDATE_DUMP_STRICT,
9302                 .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
9303                 .flags = GENL_ADMIN_PERM,
9304         },
9305         {
9306                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
9307                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9308                 .doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
9309                 .flags = GENL_ADMIN_PERM,
9310                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9311                                   DEVLINK_NL_FLAG_NO_LOCK,
9312         },
9313         {
9314                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
9315                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9316                 .doit = devlink_nl_cmd_health_reporter_test_doit,
9317                 .flags = GENL_ADMIN_PERM,
9318                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
9319                                   DEVLINK_NL_FLAG_NO_LOCK,
9320         },
9321         {
9322                 .cmd = DEVLINK_CMD_FLASH_UPDATE,
9323                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
9324                 .doit = devlink_nl_cmd_flash_update,
9325                 .flags = GENL_ADMIN_PERM,
9326         },
9327         {
9328                 .cmd = DEVLINK_CMD_TRAP_GET,
9329                 .doit = devlink_nl_cmd_trap_get_doit,
9330                 .dumpit = devlink_nl_cmd_trap_get_dumpit,
9331                 /* can be retrieved by unprivileged users */
9332         },
9333         {
9334                 .cmd = DEVLINK_CMD_TRAP_SET,
9335                 .doit = devlink_nl_cmd_trap_set_doit,
9336                 .flags = GENL_ADMIN_PERM,
9337         },
9338         {
9339                 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
9340                 .doit = devlink_nl_cmd_trap_group_get_doit,
9341                 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
9342                 /* can be retrieved by unprivileged users */
9343         },
9344         {
9345                 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
9346                 .doit = devlink_nl_cmd_trap_group_set_doit,
9347                 .flags = GENL_ADMIN_PERM,
9348         },
9349         {
9350                 .cmd = DEVLINK_CMD_TRAP_POLICER_GET,
9351                 .doit = devlink_nl_cmd_trap_policer_get_doit,
9352                 .dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
9353                 /* can be retrieved by unprivileged users */
9354         },
9355         {
9356                 .cmd = DEVLINK_CMD_TRAP_POLICER_SET,
9357                 .doit = devlink_nl_cmd_trap_policer_set_doit,
9358                 .flags = GENL_ADMIN_PERM,
9359         },
9360 };
9361
9362 static struct genl_family devlink_nl_family __ro_after_init = {
9363         .name           = DEVLINK_GENL_NAME,
9364         .version        = DEVLINK_GENL_VERSION,
9365         .maxattr        = DEVLINK_ATTR_MAX,
9366         .policy = devlink_nl_policy,
9367         .netnsok        = true,
9368         .pre_doit       = devlink_nl_pre_doit,
9369         .post_doit      = devlink_nl_post_doit,
9370         .module         = THIS_MODULE,
9371         .small_ops      = devlink_nl_ops,
9372         .n_small_ops    = ARRAY_SIZE(devlink_nl_ops),
9373         .mcgrps         = devlink_nl_mcgrps,
9374         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
9375 };
9376
9377 static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
9378 {
9379         const struct devlink_reload_combination *comb;
9380         int i;
9381
9382         if (!devlink_reload_supported(ops)) {
9383                 if (WARN_ON(ops->reload_actions))
9384                         return false;
9385                 return true;
9386         }
9387
9388         if (WARN_ON(!ops->reload_actions ||
9389                     ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
9390                     ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
9391                 return false;
9392
9393         if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
9394                     ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
9395                 return false;
9396
9397         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
9398                 comb = &devlink_reload_invalid_combinations[i];
9399                 if (ops->reload_actions == BIT(comb->action) &&
9400                     ops->reload_limits == BIT(comb->limit))
9401                         return false;
9402         }
9403         return true;
9404 }
9405
9406 /**
9407  *      devlink_set_features - Set devlink supported features
9408  *
9409  *      @devlink: devlink
9410  *      @features: devlink support features
9411  *
9412  *      This interface allows us to set reload ops separatelly from
9413  *      the devlink_alloc.
9414  */
9415 void devlink_set_features(struct devlink *devlink, u64 features)
9416 {
9417         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9418
9419         WARN_ON(features & DEVLINK_F_RELOAD &&
9420                 !devlink_reload_supported(devlink->ops));
9421         devlink->features = features;
9422 }
9423 EXPORT_SYMBOL_GPL(devlink_set_features);
9424
9425 /**
9426  *      devlink_alloc_ns - Allocate new devlink instance resources
9427  *      in specific namespace
9428  *
9429  *      @ops: ops
9430  *      @priv_size: size of user private data
9431  *      @net: net namespace
9432  *      @dev: parent device
9433  *
9434  *      Allocate new devlink instance resources, including devlink index
9435  *      and name.
9436  */
9437 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
9438                                  size_t priv_size, struct net *net,
9439                                  struct device *dev)
9440 {
9441         struct devlink *devlink;
9442         static u32 last_id;
9443         int ret;
9444
9445         WARN_ON(!ops || !dev);
9446         if (!devlink_reload_actions_valid(ops))
9447                 return NULL;
9448
9449         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
9450         if (!devlink)
9451                 return NULL;
9452
9453         ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
9454                               &last_id, GFP_KERNEL);
9455         if (ret < 0) {
9456                 kfree(devlink);
9457                 return NULL;
9458         }
9459
9460         devlink->dev = dev;
9461         devlink->ops = ops;
9462         xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
9463         write_pnet(&devlink->_net, net);
9464         INIT_LIST_HEAD(&devlink->port_list);
9465         INIT_LIST_HEAD(&devlink->rate_list);
9466         INIT_LIST_HEAD(&devlink->linecard_list);
9467         INIT_LIST_HEAD(&devlink->sb_list);
9468         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
9469         INIT_LIST_HEAD(&devlink->resource_list);
9470         INIT_LIST_HEAD(&devlink->param_list);
9471         INIT_LIST_HEAD(&devlink->region_list);
9472         INIT_LIST_HEAD(&devlink->reporter_list);
9473         INIT_LIST_HEAD(&devlink->trap_list);
9474         INIT_LIST_HEAD(&devlink->trap_group_list);
9475         INIT_LIST_HEAD(&devlink->trap_policer_list);
9476         mutex_init(&devlink->lock);
9477         mutex_init(&devlink->reporters_lock);
9478         mutex_init(&devlink->linecards_lock);
9479         refcount_set(&devlink->refcount, 1);
9480         init_completion(&devlink->comp);
9481
9482         return devlink;
9483 }
9484 EXPORT_SYMBOL_GPL(devlink_alloc_ns);
9485
9486 static void
9487 devlink_trap_policer_notify(struct devlink *devlink,
9488                             const struct devlink_trap_policer_item *policer_item,
9489                             enum devlink_command cmd);
9490 static void
9491 devlink_trap_group_notify(struct devlink *devlink,
9492                           const struct devlink_trap_group_item *group_item,
9493                           enum devlink_command cmd);
9494 static void devlink_trap_notify(struct devlink *devlink,
9495                                 const struct devlink_trap_item *trap_item,
9496                                 enum devlink_command cmd);
9497
9498 static void devlink_notify_register(struct devlink *devlink)
9499 {
9500         struct devlink_trap_policer_item *policer_item;
9501         struct devlink_trap_group_item *group_item;
9502         struct devlink_param_item *param_item;
9503         struct devlink_trap_item *trap_item;
9504         struct devlink_port *devlink_port;
9505         struct devlink_linecard *linecard;
9506         struct devlink_rate *rate_node;
9507         struct devlink_region *region;
9508
9509         devlink_notify(devlink, DEVLINK_CMD_NEW);
9510         list_for_each_entry(linecard, &devlink->linecard_list, list)
9511                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
9512
9513         list_for_each_entry(devlink_port, &devlink->port_list, list)
9514                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9515
9516         list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
9517                 devlink_trap_policer_notify(devlink, policer_item,
9518                                             DEVLINK_CMD_TRAP_POLICER_NEW);
9519
9520         list_for_each_entry(group_item, &devlink->trap_group_list, list)
9521                 devlink_trap_group_notify(devlink, group_item,
9522                                           DEVLINK_CMD_TRAP_GROUP_NEW);
9523
9524         list_for_each_entry(trap_item, &devlink->trap_list, list)
9525                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
9526
9527         list_for_each_entry(rate_node, &devlink->rate_list, list)
9528                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
9529
9530         list_for_each_entry(region, &devlink->region_list, list)
9531                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9532
9533         list_for_each_entry(param_item, &devlink->param_list, list)
9534                 devlink_param_notify(devlink, 0, param_item,
9535                                      DEVLINK_CMD_PARAM_NEW);
9536 }
9537
9538 static void devlink_notify_unregister(struct devlink *devlink)
9539 {
9540         struct devlink_trap_policer_item *policer_item;
9541         struct devlink_trap_group_item *group_item;
9542         struct devlink_param_item *param_item;
9543         struct devlink_trap_item *trap_item;
9544         struct devlink_port *devlink_port;
9545         struct devlink_rate *rate_node;
9546         struct devlink_region *region;
9547
9548         list_for_each_entry_reverse(param_item, &devlink->param_list, list)
9549                 devlink_param_notify(devlink, 0, param_item,
9550                                      DEVLINK_CMD_PARAM_DEL);
9551
9552         list_for_each_entry_reverse(region, &devlink->region_list, list)
9553                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
9554
9555         list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
9556                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
9557
9558         list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
9559                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
9560
9561         list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
9562                 devlink_trap_group_notify(devlink, group_item,
9563                                           DEVLINK_CMD_TRAP_GROUP_DEL);
9564         list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
9565                                     list)
9566                 devlink_trap_policer_notify(devlink, policer_item,
9567                                             DEVLINK_CMD_TRAP_POLICER_DEL);
9568
9569         list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
9570                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9571         devlink_notify(devlink, DEVLINK_CMD_DEL);
9572 }
9573
9574 /**
9575  *      devlink_register - Register devlink instance
9576  *
9577  *      @devlink: devlink
9578  */
9579 void devlink_register(struct devlink *devlink)
9580 {
9581         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9582         /* Make sure that we are in .probe() routine */
9583
9584         mutex_lock(&devlink_mutex);
9585         xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9586         devlink_notify_register(devlink);
9587         mutex_unlock(&devlink_mutex);
9588 }
9589 EXPORT_SYMBOL_GPL(devlink_register);
9590
9591 /**
9592  *      devlink_unregister - Unregister devlink instance
9593  *
9594  *      @devlink: devlink
9595  */
9596 void devlink_unregister(struct devlink *devlink)
9597 {
9598         ASSERT_DEVLINK_REGISTERED(devlink);
9599         /* Make sure that we are in .remove() routine */
9600
9601         devlink_put(devlink);
9602         wait_for_completion(&devlink->comp);
9603
9604         mutex_lock(&devlink_mutex);
9605         devlink_notify_unregister(devlink);
9606         xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9607         mutex_unlock(&devlink_mutex);
9608 }
9609 EXPORT_SYMBOL_GPL(devlink_unregister);
9610
9611 /**
9612  *      devlink_free - Free devlink instance resources
9613  *
9614  *      @devlink: devlink
9615  */
9616 void devlink_free(struct devlink *devlink)
9617 {
9618         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9619
9620         mutex_destroy(&devlink->linecards_lock);
9621         mutex_destroy(&devlink->reporters_lock);
9622         mutex_destroy(&devlink->lock);
9623         WARN_ON(!list_empty(&devlink->trap_policer_list));
9624         WARN_ON(!list_empty(&devlink->trap_group_list));
9625         WARN_ON(!list_empty(&devlink->trap_list));
9626         WARN_ON(!list_empty(&devlink->reporter_list));
9627         WARN_ON(!list_empty(&devlink->region_list));
9628         WARN_ON(!list_empty(&devlink->param_list));
9629         WARN_ON(!list_empty(&devlink->resource_list));
9630         WARN_ON(!list_empty(&devlink->dpipe_table_list));
9631         WARN_ON(!list_empty(&devlink->sb_list));
9632         WARN_ON(!list_empty(&devlink->rate_list));
9633         WARN_ON(!list_empty(&devlink->linecard_list));
9634         WARN_ON(!list_empty(&devlink->port_list));
9635
9636         xa_destroy(&devlink->snapshot_ids);
9637         xa_erase(&devlinks, devlink->index);
9638
9639         kfree(devlink);
9640 }
9641 EXPORT_SYMBOL_GPL(devlink_free);
9642
9643 static void devlink_port_type_warn(struct work_struct *work)
9644 {
9645         WARN(true, "Type was not set for devlink port.");
9646 }
9647
9648 static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
9649 {
9650         /* Ignore CPU and DSA flavours. */
9651         return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
9652                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
9653                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
9654 }
9655
9656 #define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
9657
9658 static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
9659 {
9660         if (!devlink_port_type_should_warn(devlink_port))
9661                 return;
9662         /* Schedule a work to WARN in case driver does not set port
9663          * type within timeout.
9664          */
9665         schedule_delayed_work(&devlink_port->type_warn_dw,
9666                               DEVLINK_PORT_TYPE_WARN_TIMEOUT);
9667 }
9668
9669 static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
9670 {
9671         if (!devlink_port_type_should_warn(devlink_port))
9672                 return;
9673         cancel_delayed_work_sync(&devlink_port->type_warn_dw);
9674 }
9675
9676 /**
9677  * devl_port_register() - Register devlink port
9678  *
9679  * @devlink: devlink
9680  * @devlink_port: devlink port
9681  * @port_index: driver-specific numerical identifier of the port
9682  *
9683  * Register devlink port with provided port index. User can use
9684  * any indexing, even hw-related one. devlink_port structure
9685  * is convenient to be embedded inside user driver private structure.
9686  * Note that the caller should take care of zeroing the devlink_port
9687  * structure.
9688  */
9689 int devl_port_register(struct devlink *devlink,
9690                        struct devlink_port *devlink_port,
9691                        unsigned int port_index)
9692 {
9693         devl_assert_locked(devlink);
9694
9695         if (devlink_port_index_exists(devlink, port_index))
9696                 return -EEXIST;
9697
9698         WARN_ON(devlink_port->devlink);
9699         devlink_port->devlink = devlink;
9700         devlink_port->index = port_index;
9701         spin_lock_init(&devlink_port->type_lock);
9702         INIT_LIST_HEAD(&devlink_port->reporter_list);
9703         mutex_init(&devlink_port->reporters_lock);
9704         list_add_tail(&devlink_port->list, &devlink->port_list);
9705         INIT_LIST_HEAD(&devlink_port->param_list);
9706         INIT_LIST_HEAD(&devlink_port->region_list);
9707
9708         INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
9709         devlink_port_type_warn_schedule(devlink_port);
9710         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9711         return 0;
9712 }
9713 EXPORT_SYMBOL_GPL(devl_port_register);
9714
9715 /**
9716  *      devlink_port_register - Register devlink port
9717  *
9718  *      @devlink: devlink
9719  *      @devlink_port: devlink port
9720  *      @port_index: driver-specific numerical identifier of the port
9721  *
9722  *      Register devlink port with provided port index. User can use
9723  *      any indexing, even hw-related one. devlink_port structure
9724  *      is convenient to be embedded inside user driver private structure.
9725  *      Note that the caller should take care of zeroing the devlink_port
9726  *      structure.
9727  *
9728  *      Context: Takes and release devlink->lock <mutex>.
9729  */
9730 int devlink_port_register(struct devlink *devlink,
9731                           struct devlink_port *devlink_port,
9732                           unsigned int port_index)
9733 {
9734         int err;
9735
9736         devl_lock(devlink);
9737         err = devl_port_register(devlink, devlink_port, port_index);
9738         devl_unlock(devlink);
9739         return err;
9740 }
9741 EXPORT_SYMBOL_GPL(devlink_port_register);
9742
9743 /**
9744  * devl_port_unregister() - Unregister devlink port
9745  *
9746  * @devlink_port: devlink port
9747  */
9748 void devl_port_unregister(struct devlink_port *devlink_port)
9749 {
9750         lockdep_assert_held(&devlink_port->devlink->lock);
9751
9752         devlink_port_type_warn_cancel(devlink_port);
9753         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9754         list_del(&devlink_port->list);
9755         WARN_ON(!list_empty(&devlink_port->reporter_list));
9756         WARN_ON(!list_empty(&devlink_port->region_list));
9757         mutex_destroy(&devlink_port->reporters_lock);
9758 }
9759 EXPORT_SYMBOL_GPL(devl_port_unregister);
9760
9761 /**
9762  *      devlink_port_unregister - Unregister devlink port
9763  *
9764  *      @devlink_port: devlink port
9765  *
9766  *      Context: Takes and release devlink->lock <mutex>.
9767  */
9768 void devlink_port_unregister(struct devlink_port *devlink_port)
9769 {
9770         struct devlink *devlink = devlink_port->devlink;
9771
9772         devl_lock(devlink);
9773         devl_port_unregister(devlink_port);
9774         devl_unlock(devlink);
9775 }
9776 EXPORT_SYMBOL_GPL(devlink_port_unregister);
9777
9778 static void __devlink_port_type_set(struct devlink_port *devlink_port,
9779                                     enum devlink_port_type type,
9780                                     void *type_dev)
9781 {
9782         if (WARN_ON(!devlink_port->devlink))
9783                 return;
9784         devlink_port_type_warn_cancel(devlink_port);
9785         spin_lock_bh(&devlink_port->type_lock);
9786         devlink_port->type = type;
9787         devlink_port->type_dev = type_dev;
9788         spin_unlock_bh(&devlink_port->type_lock);
9789         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9790 }
9791
9792 static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
9793                                             struct net_device *netdev)
9794 {
9795         const struct net_device_ops *ops = netdev->netdev_ops;
9796
9797         /* If driver registers devlink port, it should set devlink port
9798          * attributes accordingly so the compat functions are called
9799          * and the original ops are not used.
9800          */
9801         if (ops->ndo_get_phys_port_name) {
9802                 /* Some drivers use the same set of ndos for netdevs
9803                  * that have devlink_port registered and also for
9804                  * those who don't. Make sure that ndo_get_phys_port_name
9805                  * returns -EOPNOTSUPP here in case it is defined.
9806                  * Warn if not.
9807                  */
9808                 char name[IFNAMSIZ];
9809                 int err;
9810
9811                 err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
9812                 WARN_ON(err != -EOPNOTSUPP);
9813         }
9814         if (ops->ndo_get_port_parent_id) {
9815                 /* Some drivers use the same set of ndos for netdevs
9816                  * that have devlink_port registered and also for
9817                  * those who don't. Make sure that ndo_get_port_parent_id
9818                  * returns -EOPNOTSUPP here in case it is defined.
9819                  * Warn if not.
9820                  */
9821                 struct netdev_phys_item_id ppid;
9822                 int err;
9823
9824                 err = ops->ndo_get_port_parent_id(netdev, &ppid);
9825                 WARN_ON(err != -EOPNOTSUPP);
9826         }
9827 }
9828
9829 /**
9830  *      devlink_port_type_eth_set - Set port type to Ethernet
9831  *
9832  *      @devlink_port: devlink port
9833  *      @netdev: related netdevice
9834  */
9835 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
9836                                struct net_device *netdev)
9837 {
9838         if (netdev)
9839                 devlink_port_type_netdev_checks(devlink_port, netdev);
9840         else
9841                 dev_warn(devlink_port->devlink->dev,
9842                          "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
9843                          devlink_port->index);
9844
9845         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
9846 }
9847 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
9848
9849 /**
9850  *      devlink_port_type_ib_set - Set port type to InfiniBand
9851  *
9852  *      @devlink_port: devlink port
9853  *      @ibdev: related IB device
9854  */
9855 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
9856                               struct ib_device *ibdev)
9857 {
9858         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
9859 }
9860 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
9861
9862 /**
9863  *      devlink_port_type_clear - Clear port type
9864  *
9865  *      @devlink_port: devlink port
9866  */
9867 void devlink_port_type_clear(struct devlink_port *devlink_port)
9868 {
9869         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
9870         devlink_port_type_warn_schedule(devlink_port);
9871 }
9872 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
9873
9874 static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
9875                                     enum devlink_port_flavour flavour)
9876 {
9877         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9878
9879         devlink_port->attrs_set = true;
9880         attrs->flavour = flavour;
9881         if (attrs->switch_id.id_len) {
9882                 devlink_port->switch_port = true;
9883                 if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
9884                         attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
9885         } else {
9886                 devlink_port->switch_port = false;
9887         }
9888         return 0;
9889 }
9890
9891 /**
9892  *      devlink_port_attrs_set - Set port attributes
9893  *
9894  *      @devlink_port: devlink port
9895  *      @attrs: devlink port attrs
9896  */
9897 void devlink_port_attrs_set(struct devlink_port *devlink_port,
9898                             struct devlink_port_attrs *attrs)
9899 {
9900         int ret;
9901
9902         if (WARN_ON(devlink_port->devlink))
9903                 return;
9904         devlink_port->attrs = *attrs;
9905         ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
9906         if (ret)
9907                 return;
9908         WARN_ON(attrs->splittable && attrs->split);
9909 }
9910 EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
9911
9912 /**
9913  *      devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
9914  *
9915  *      @devlink_port: devlink port
9916  *      @controller: associated controller number for the devlink port instance
9917  *      @pf: associated PF for the devlink port instance
9918  *      @external: indicates if the port is for an external controller
9919  */
9920 void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
9921                                    u16 pf, bool external)
9922 {
9923         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9924         int ret;
9925
9926         if (WARN_ON(devlink_port->devlink))
9927                 return;
9928         ret = __devlink_port_attrs_set(devlink_port,
9929                                        DEVLINK_PORT_FLAVOUR_PCI_PF);
9930         if (ret)
9931                 return;
9932         attrs->pci_pf.controller = controller;
9933         attrs->pci_pf.pf = pf;
9934         attrs->pci_pf.external = external;
9935 }
9936 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
9937
9938 /**
9939  *      devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
9940  *
9941  *      @devlink_port: devlink port
9942  *      @controller: associated controller number for the devlink port instance
9943  *      @pf: associated PF for the devlink port instance
9944  *      @vf: associated VF of a PF for the devlink port instance
9945  *      @external: indicates if the port is for an external controller
9946  */
9947 void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
9948                                    u16 pf, u16 vf, bool external)
9949 {
9950         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9951         int ret;
9952
9953         if (WARN_ON(devlink_port->devlink))
9954                 return;
9955         ret = __devlink_port_attrs_set(devlink_port,
9956                                        DEVLINK_PORT_FLAVOUR_PCI_VF);
9957         if (ret)
9958                 return;
9959         attrs->pci_vf.controller = controller;
9960         attrs->pci_vf.pf = pf;
9961         attrs->pci_vf.vf = vf;
9962         attrs->pci_vf.external = external;
9963 }
9964 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
9965
9966 /**
9967  *      devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
9968  *
9969  *      @devlink_port: devlink port
9970  *      @controller: associated controller number for the devlink port instance
9971  *      @pf: associated PF for the devlink port instance
9972  *      @sf: associated SF of a PF for the devlink port instance
9973  *      @external: indicates if the port is for an external controller
9974  */
9975 void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
9976                                    u16 pf, u32 sf, bool external)
9977 {
9978         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9979         int ret;
9980
9981         if (WARN_ON(devlink_port->devlink))
9982                 return;
9983         ret = __devlink_port_attrs_set(devlink_port,
9984                                        DEVLINK_PORT_FLAVOUR_PCI_SF);
9985         if (ret)
9986                 return;
9987         attrs->pci_sf.controller = controller;
9988         attrs->pci_sf.pf = pf;
9989         attrs->pci_sf.sf = sf;
9990         attrs->pci_sf.external = external;
9991 }
9992 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
9993
9994 /**
9995  * devl_rate_leaf_create - create devlink rate leaf
9996  * @devlink_port: devlink port object to create rate object on
9997  * @priv: driver private data
9998  *
9999  * Create devlink rate object of type leaf on provided @devlink_port.
10000  */
10001 int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
10002 {
10003         struct devlink *devlink = devlink_port->devlink;
10004         struct devlink_rate *devlink_rate;
10005
10006         devl_assert_locked(devlink_port->devlink);
10007
10008         if (WARN_ON(devlink_port->devlink_rate))
10009                 return -EBUSY;
10010
10011         devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
10012         if (!devlink_rate)
10013                 return -ENOMEM;
10014
10015         devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
10016         devlink_rate->devlink = devlink;
10017         devlink_rate->devlink_port = devlink_port;
10018         devlink_rate->priv = priv;
10019         list_add_tail(&devlink_rate->list, &devlink->rate_list);
10020         devlink_port->devlink_rate = devlink_rate;
10021         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
10022
10023         return 0;
10024 }
10025 EXPORT_SYMBOL_GPL(devl_rate_leaf_create);
10026
10027 /**
10028  * devl_rate_leaf_destroy - destroy devlink rate leaf
10029  *
10030  * @devlink_port: devlink port linked to the rate object
10031  *
10032  * Destroy the devlink rate object of type leaf on provided @devlink_port.
10033  */
10034 void devl_rate_leaf_destroy(struct devlink_port *devlink_port)
10035 {
10036         struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
10037
10038         devl_assert_locked(devlink_port->devlink);
10039         if (!devlink_rate)
10040                 return;
10041
10042         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
10043         if (devlink_rate->parent)
10044                 refcount_dec(&devlink_rate->parent->refcnt);
10045         list_del(&devlink_rate->list);
10046         devlink_port->devlink_rate = NULL;
10047         kfree(devlink_rate);
10048 }
10049 EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
10050
10051 /**
10052  * devl_rate_nodes_destroy - destroy all devlink rate nodes on device
10053  * @devlink: devlink instance
10054  *
10055  * Unset parent for all rate objects and destroy all rate nodes
10056  * on specified device.
10057  */
10058 void devl_rate_nodes_destroy(struct devlink *devlink)
10059 {
10060         static struct devlink_rate *devlink_rate, *tmp;
10061         const struct devlink_ops *ops = devlink->ops;
10062
10063         devl_assert_locked(devlink);
10064
10065         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
10066                 if (!devlink_rate->parent)
10067                         continue;
10068
10069                 refcount_dec(&devlink_rate->parent->refcnt);
10070                 if (devlink_rate_is_leaf(devlink_rate))
10071                         ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
10072                                                   NULL, NULL);
10073                 else if (devlink_rate_is_node(devlink_rate))
10074                         ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
10075                                                   NULL, NULL);
10076         }
10077         list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
10078                 if (devlink_rate_is_node(devlink_rate)) {
10079                         ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
10080                         list_del(&devlink_rate->list);
10081                         kfree(devlink_rate->name);
10082                         kfree(devlink_rate);
10083                 }
10084         }
10085 }
10086 EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
10087
10088 /**
10089  *      devlink_port_linecard_set - Link port with a linecard
10090  *
10091  *      @devlink_port: devlink port
10092  *      @linecard: devlink linecard
10093  */
10094 void devlink_port_linecard_set(struct devlink_port *devlink_port,
10095                                struct devlink_linecard *linecard)
10096 {
10097         if (WARN_ON(devlink_port->devlink))
10098                 return;
10099         devlink_port->linecard = linecard;
10100 }
10101 EXPORT_SYMBOL_GPL(devlink_port_linecard_set);
10102
10103 static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
10104                                              char *name, size_t len)
10105 {
10106         struct devlink_port_attrs *attrs = &devlink_port->attrs;
10107         int n = 0;
10108
10109         if (!devlink_port->attrs_set)
10110                 return -EOPNOTSUPP;
10111
10112         switch (attrs->flavour) {
10113         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
10114                 if (devlink_port->linecard)
10115                         n = snprintf(name, len, "l%u",
10116                                      devlink_port->linecard->index);
10117                 if (n < len)
10118                         n += snprintf(name + n, len - n, "p%u",
10119                                       attrs->phys.port_number);
10120                 if (n < len && attrs->split)
10121                         n += snprintf(name + n, len - n, "s%u",
10122                                       attrs->phys.split_subport_number);
10123                 break;
10124         case DEVLINK_PORT_FLAVOUR_CPU:
10125         case DEVLINK_PORT_FLAVOUR_DSA:
10126         case DEVLINK_PORT_FLAVOUR_UNUSED:
10127                 /* As CPU and DSA ports do not have a netdevice associated
10128                  * case should not ever happen.
10129                  */
10130                 WARN_ON(1);
10131                 return -EINVAL;
10132         case DEVLINK_PORT_FLAVOUR_PCI_PF:
10133                 if (attrs->pci_pf.external) {
10134                         n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
10135                         if (n >= len)
10136                                 return -EINVAL;
10137                         len -= n;
10138                         name += n;
10139                 }
10140                 n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
10141                 break;
10142         case DEVLINK_PORT_FLAVOUR_PCI_VF:
10143                 if (attrs->pci_vf.external) {
10144                         n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
10145                         if (n >= len)
10146                                 return -EINVAL;
10147                         len -= n;
10148                         name += n;
10149                 }
10150                 n = snprintf(name, len, "pf%uvf%u",
10151                              attrs->pci_vf.pf, attrs->pci_vf.vf);
10152                 break;
10153         case DEVLINK_PORT_FLAVOUR_PCI_SF:
10154                 if (attrs->pci_sf.external) {
10155                         n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
10156                         if (n >= len)
10157                                 return -EINVAL;
10158                         len -= n;
10159                         name += n;
10160                 }
10161                 n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
10162                              attrs->pci_sf.sf);
10163                 break;
10164         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
10165                 return -EOPNOTSUPP;
10166         }
10167
10168         if (n >= len)
10169                 return -EINVAL;
10170
10171         return 0;
10172 }
10173
10174 static int devlink_linecard_types_init(struct devlink_linecard *linecard)
10175 {
10176         struct devlink_linecard_type *linecard_type;
10177         unsigned int count;
10178         int i;
10179
10180         count = linecard->ops->types_count(linecard, linecard->priv);
10181         linecard->types = kmalloc_array(count, sizeof(*linecard_type),
10182                                         GFP_KERNEL);
10183         if (!linecard->types)
10184                 return -ENOMEM;
10185         linecard->types_count = count;
10186
10187         for (i = 0; i < count; i++) {
10188                 linecard_type = &linecard->types[i];
10189                 linecard->ops->types_get(linecard, linecard->priv, i,
10190                                          &linecard_type->type,
10191                                          &linecard_type->priv);
10192         }
10193         return 0;
10194 }
10195
10196 static void devlink_linecard_types_fini(struct devlink_linecard *linecard)
10197 {
10198         kfree(linecard->types);
10199 }
10200
10201 /**
10202  *      devlink_linecard_create - Create devlink linecard
10203  *
10204  *      @devlink: devlink
10205  *      @linecard_index: driver-specific numerical identifier of the linecard
10206  *      @ops: linecards ops
10207  *      @priv: user priv pointer
10208  *
10209  *      Create devlink linecard instance with provided linecard index.
10210  *      Caller can use any indexing, even hw-related one.
10211  *
10212  *      Return: Line card structure or an ERR_PTR() encoded error code.
10213  */
10214 struct devlink_linecard *
10215 devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index,
10216                         const struct devlink_linecard_ops *ops, void *priv)
10217 {
10218         struct devlink_linecard *linecard;
10219         int err;
10220
10221         if (WARN_ON(!ops || !ops->provision || !ops->unprovision ||
10222                     !ops->types_count || !ops->types_get))
10223                 return ERR_PTR(-EINVAL);
10224
10225         mutex_lock(&devlink->linecards_lock);
10226         if (devlink_linecard_index_exists(devlink, linecard_index)) {
10227                 mutex_unlock(&devlink->linecards_lock);
10228                 return ERR_PTR(-EEXIST);
10229         }
10230
10231         linecard = kzalloc(sizeof(*linecard), GFP_KERNEL);
10232         if (!linecard) {
10233                 mutex_unlock(&devlink->linecards_lock);
10234                 return ERR_PTR(-ENOMEM);
10235         }
10236
10237         linecard->devlink = devlink;
10238         linecard->index = linecard_index;
10239         linecard->ops = ops;
10240         linecard->priv = priv;
10241         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
10242         mutex_init(&linecard->state_lock);
10243
10244         err = devlink_linecard_types_init(linecard);
10245         if (err) {
10246                 mutex_destroy(&linecard->state_lock);
10247                 kfree(linecard);
10248                 mutex_unlock(&devlink->linecards_lock);
10249                 return ERR_PTR(err);
10250         }
10251
10252         list_add_tail(&linecard->list, &devlink->linecard_list);
10253         refcount_set(&linecard->refcount, 1);
10254         mutex_unlock(&devlink->linecards_lock);
10255         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10256         return linecard;
10257 }
10258 EXPORT_SYMBOL_GPL(devlink_linecard_create);
10259
10260 /**
10261  *      devlink_linecard_destroy - Destroy devlink linecard
10262  *
10263  *      @linecard: devlink linecard
10264  */
10265 void devlink_linecard_destroy(struct devlink_linecard *linecard)
10266 {
10267         struct devlink *devlink = linecard->devlink;
10268
10269         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_DEL);
10270         mutex_lock(&devlink->linecards_lock);
10271         list_del(&linecard->list);
10272         devlink_linecard_types_fini(linecard);
10273         mutex_unlock(&devlink->linecards_lock);
10274         devlink_linecard_put(linecard);
10275 }
10276 EXPORT_SYMBOL_GPL(devlink_linecard_destroy);
10277
10278 /**
10279  *      devlink_linecard_provision_set - Set provisioning on linecard
10280  *
10281  *      @linecard: devlink linecard
10282  *      @type: linecard type
10283  *
10284  *      This is either called directly from the provision() op call or
10285  *      as a result of the provision() op call asynchronously.
10286  */
10287 void devlink_linecard_provision_set(struct devlink_linecard *linecard,
10288                                     const char *type)
10289 {
10290         mutex_lock(&linecard->state_lock);
10291         WARN_ON(linecard->type && strcmp(linecard->type, type));
10292         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
10293         linecard->type = type;
10294         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10295         mutex_unlock(&linecard->state_lock);
10296 }
10297 EXPORT_SYMBOL_GPL(devlink_linecard_provision_set);
10298
10299 /**
10300  *      devlink_linecard_provision_clear - Clear provisioning on linecard
10301  *
10302  *      @linecard: devlink linecard
10303  *
10304  *      This is either called directly from the unprovision() op call or
10305  *      as a result of the unprovision() op call asynchronously.
10306  */
10307 void devlink_linecard_provision_clear(struct devlink_linecard *linecard)
10308 {
10309         mutex_lock(&linecard->state_lock);
10310         linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
10311         linecard->type = NULL;
10312         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10313         mutex_unlock(&linecard->state_lock);
10314 }
10315 EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear);
10316
10317 /**
10318  *      devlink_linecard_provision_fail - Fail provisioning on linecard
10319  *
10320  *      @linecard: devlink linecard
10321  *
10322  *      This is either called directly from the provision() op call or
10323  *      as a result of the provision() op call asynchronously.
10324  */
10325 void devlink_linecard_provision_fail(struct devlink_linecard *linecard)
10326 {
10327         mutex_lock(&linecard->state_lock);
10328         linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING_FAILED;
10329         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10330         mutex_unlock(&linecard->state_lock);
10331 }
10332 EXPORT_SYMBOL_GPL(devlink_linecard_provision_fail);
10333
10334 /**
10335  *      devlink_linecard_activate - Set linecard active
10336  *
10337  *      @linecard: devlink linecard
10338  */
10339 void devlink_linecard_activate(struct devlink_linecard *linecard)
10340 {
10341         mutex_lock(&linecard->state_lock);
10342         WARN_ON(linecard->state != DEVLINK_LINECARD_STATE_PROVISIONED);
10343         linecard->state = DEVLINK_LINECARD_STATE_ACTIVE;
10344         devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10345         mutex_unlock(&linecard->state_lock);
10346 }
10347 EXPORT_SYMBOL_GPL(devlink_linecard_activate);
10348
10349 /**
10350  *      devlink_linecard_deactivate - Set linecard inactive
10351  *
10352  *      @linecard: devlink linecard
10353  */
10354 void devlink_linecard_deactivate(struct devlink_linecard *linecard)
10355 {
10356         mutex_lock(&linecard->state_lock);
10357         switch (linecard->state) {
10358         case DEVLINK_LINECARD_STATE_ACTIVE:
10359                 linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
10360                 devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
10361                 break;
10362         case DEVLINK_LINECARD_STATE_UNPROVISIONING:
10363                 /* Line card is being deactivated as part
10364                  * of unprovisioning flow.
10365                  */
10366                 break;
10367         default:
10368                 WARN_ON(1);
10369                 break;
10370         }
10371         mutex_unlock(&linecard->state_lock);
10372 }
10373 EXPORT_SYMBOL_GPL(devlink_linecard_deactivate);
10374
10375 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
10376                         u32 size, u16 ingress_pools_count,
10377                         u16 egress_pools_count, u16 ingress_tc_count,
10378                         u16 egress_tc_count)
10379 {
10380         struct devlink_sb *devlink_sb;
10381         int err = 0;
10382
10383         devl_lock(devlink);
10384         if (devlink_sb_index_exists(devlink, sb_index)) {
10385                 err = -EEXIST;
10386                 goto unlock;
10387         }
10388
10389         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
10390         if (!devlink_sb) {
10391                 err = -ENOMEM;
10392                 goto unlock;
10393         }
10394         devlink_sb->index = sb_index;
10395         devlink_sb->size = size;
10396         devlink_sb->ingress_pools_count = ingress_pools_count;
10397         devlink_sb->egress_pools_count = egress_pools_count;
10398         devlink_sb->ingress_tc_count = ingress_tc_count;
10399         devlink_sb->egress_tc_count = egress_tc_count;
10400         list_add_tail(&devlink_sb->list, &devlink->sb_list);
10401 unlock:
10402         devl_unlock(devlink);
10403         return err;
10404 }
10405 EXPORT_SYMBOL_GPL(devlink_sb_register);
10406
10407 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
10408 {
10409         struct devlink_sb *devlink_sb;
10410
10411         devl_lock(devlink);
10412         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
10413         WARN_ON(!devlink_sb);
10414         list_del(&devlink_sb->list);
10415         devl_unlock(devlink);
10416         kfree(devlink_sb);
10417 }
10418 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
10419
10420 /**
10421  *      devlink_dpipe_headers_register - register dpipe headers
10422  *
10423  *      @devlink: devlink
10424  *      @dpipe_headers: dpipe header array
10425  *
10426  *      Register the headers supported by hardware.
10427  */
10428 int devlink_dpipe_headers_register(struct devlink *devlink,
10429                                    struct devlink_dpipe_headers *dpipe_headers)
10430 {
10431         devl_lock(devlink);
10432         devlink->dpipe_headers = dpipe_headers;
10433         devl_unlock(devlink);
10434         return 0;
10435 }
10436 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
10437
10438 /**
10439  *      devlink_dpipe_headers_unregister - unregister dpipe headers
10440  *
10441  *      @devlink: devlink
10442  *
10443  *      Unregister the headers supported by hardware.
10444  */
10445 void devlink_dpipe_headers_unregister(struct devlink *devlink)
10446 {
10447         devl_lock(devlink);
10448         devlink->dpipe_headers = NULL;
10449         devl_unlock(devlink);
10450 }
10451 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
10452
10453 /**
10454  *      devlink_dpipe_table_counter_enabled - check if counter allocation
10455  *                                            required
10456  *      @devlink: devlink
10457  *      @table_name: tables name
10458  *
10459  *      Used by driver to check if counter allocation is required.
10460  *      After counter allocation is turned on the table entries
10461  *      are updated to include counter statistics.
10462  *
10463  *      After that point on the driver must respect the counter
10464  *      state so that each entry added to the table is added
10465  *      with a counter.
10466  */
10467 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
10468                                          const char *table_name)
10469 {
10470         struct devlink_dpipe_table *table;
10471         bool enabled;
10472
10473         rcu_read_lock();
10474         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10475                                          table_name, devlink);
10476         enabled = false;
10477         if (table)
10478                 enabled = table->counters_enabled;
10479         rcu_read_unlock();
10480         return enabled;
10481 }
10482 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
10483
10484 /**
10485  *      devlink_dpipe_table_register - register dpipe table
10486  *
10487  *      @devlink: devlink
10488  *      @table_name: table name
10489  *      @table_ops: table ops
10490  *      @priv: priv
10491  *      @counter_control_extern: external control for counters
10492  */
10493 int devlink_dpipe_table_register(struct devlink *devlink,
10494                                  const char *table_name,
10495                                  struct devlink_dpipe_table_ops *table_ops,
10496                                  void *priv, bool counter_control_extern)
10497 {
10498         struct devlink_dpipe_table *table;
10499         int err = 0;
10500
10501         if (WARN_ON(!table_ops->size_get))
10502                 return -EINVAL;
10503
10504         devl_lock(devlink);
10505
10506         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
10507                                      devlink)) {
10508                 err = -EEXIST;
10509                 goto unlock;
10510         }
10511
10512         table = kzalloc(sizeof(*table), GFP_KERNEL);
10513         if (!table) {
10514                 err = -ENOMEM;
10515                 goto unlock;
10516         }
10517
10518         table->name = table_name;
10519         table->table_ops = table_ops;
10520         table->priv = priv;
10521         table->counter_control_extern = counter_control_extern;
10522
10523         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
10524 unlock:
10525         devl_unlock(devlink);
10526         return err;
10527 }
10528 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
10529
10530 /**
10531  *      devlink_dpipe_table_unregister - unregister dpipe table
10532  *
10533  *      @devlink: devlink
10534  *      @table_name: table name
10535  */
10536 void devlink_dpipe_table_unregister(struct devlink *devlink,
10537                                     const char *table_name)
10538 {
10539         struct devlink_dpipe_table *table;
10540
10541         devl_lock(devlink);
10542         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10543                                          table_name, devlink);
10544         if (!table)
10545                 goto unlock;
10546         list_del_rcu(&table->list);
10547         devl_unlock(devlink);
10548         kfree_rcu(table, rcu);
10549         return;
10550 unlock:
10551         devl_unlock(devlink);
10552 }
10553 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
10554
10555 /**
10556  *      devlink_resource_register - devlink resource register
10557  *
10558  *      @devlink: devlink
10559  *      @resource_name: resource's name
10560  *      @resource_size: resource's size
10561  *      @resource_id: resource's id
10562  *      @parent_resource_id: resource's parent id
10563  *      @size_params: size parameters
10564  *
10565  *      Generic resources should reuse the same names across drivers.
10566  *      Please see the generic resources list at:
10567  *      Documentation/networking/devlink/devlink-resource.rst
10568  */
10569 int devlink_resource_register(struct devlink *devlink,
10570                               const char *resource_name,
10571                               u64 resource_size,
10572                               u64 resource_id,
10573                               u64 parent_resource_id,
10574                               const struct devlink_resource_size_params *size_params)
10575 {
10576         struct devlink_resource *resource;
10577         struct list_head *resource_list;
10578         bool top_hierarchy;
10579         int err = 0;
10580
10581         top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
10582
10583         devl_lock(devlink);
10584         resource = devlink_resource_find(devlink, NULL, resource_id);
10585         if (resource) {
10586                 err = -EINVAL;
10587                 goto out;
10588         }
10589
10590         resource = kzalloc(sizeof(*resource), GFP_KERNEL);
10591         if (!resource) {
10592                 err = -ENOMEM;
10593                 goto out;
10594         }
10595
10596         if (top_hierarchy) {
10597                 resource_list = &devlink->resource_list;
10598         } else {
10599                 struct devlink_resource *parent_resource;
10600
10601                 parent_resource = devlink_resource_find(devlink, NULL,
10602                                                         parent_resource_id);
10603                 if (parent_resource) {
10604                         resource_list = &parent_resource->resource_list;
10605                         resource->parent = parent_resource;
10606                 } else {
10607                         kfree(resource);
10608                         err = -EINVAL;
10609                         goto out;
10610                 }
10611         }
10612
10613         resource->name = resource_name;
10614         resource->size = resource_size;
10615         resource->size_new = resource_size;
10616         resource->id = resource_id;
10617         resource->size_valid = true;
10618         memcpy(&resource->size_params, size_params,
10619                sizeof(resource->size_params));
10620         INIT_LIST_HEAD(&resource->resource_list);
10621         list_add_tail(&resource->list, resource_list);
10622 out:
10623         devl_unlock(devlink);
10624         return err;
10625 }
10626 EXPORT_SYMBOL_GPL(devlink_resource_register);
10627
10628 static void devlink_resource_unregister(struct devlink *devlink,
10629                                         struct devlink_resource *resource)
10630 {
10631         struct devlink_resource *tmp, *child_resource;
10632
10633         list_for_each_entry_safe(child_resource, tmp, &resource->resource_list,
10634                                  list) {
10635                 devlink_resource_unregister(devlink, child_resource);
10636                 list_del(&child_resource->list);
10637                 kfree(child_resource);
10638         }
10639 }
10640
10641 /**
10642  *      devlink_resources_unregister - free all resources
10643  *
10644  *      @devlink: devlink
10645  */
10646 void devlink_resources_unregister(struct devlink *devlink)
10647 {
10648         struct devlink_resource *tmp, *child_resource;
10649
10650         devl_lock(devlink);
10651
10652         list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list,
10653                                  list) {
10654                 devlink_resource_unregister(devlink, child_resource);
10655                 list_del(&child_resource->list);
10656                 kfree(child_resource);
10657         }
10658
10659         devl_unlock(devlink);
10660 }
10661 EXPORT_SYMBOL_GPL(devlink_resources_unregister);
10662
10663 /**
10664  *      devlink_resource_size_get - get and update size
10665  *
10666  *      @devlink: devlink
10667  *      @resource_id: the requested resource id
10668  *      @p_resource_size: ptr to update
10669  */
10670 int devlink_resource_size_get(struct devlink *devlink,
10671                               u64 resource_id,
10672                               u64 *p_resource_size)
10673 {
10674         struct devlink_resource *resource;
10675         int err = 0;
10676
10677         devl_lock(devlink);
10678         resource = devlink_resource_find(devlink, NULL, resource_id);
10679         if (!resource) {
10680                 err = -EINVAL;
10681                 goto out;
10682         }
10683         *p_resource_size = resource->size_new;
10684         resource->size = resource->size_new;
10685 out:
10686         devl_unlock(devlink);
10687         return err;
10688 }
10689 EXPORT_SYMBOL_GPL(devlink_resource_size_get);
10690
10691 /**
10692  *      devlink_dpipe_table_resource_set - set the resource id
10693  *
10694  *      @devlink: devlink
10695  *      @table_name: table name
10696  *      @resource_id: resource id
10697  *      @resource_units: number of resource's units consumed per table's entry
10698  */
10699 int devlink_dpipe_table_resource_set(struct devlink *devlink,
10700                                      const char *table_name, u64 resource_id,
10701                                      u64 resource_units)
10702 {
10703         struct devlink_dpipe_table *table;
10704         int err = 0;
10705
10706         devl_lock(devlink);
10707         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10708                                          table_name, devlink);
10709         if (!table) {
10710                 err = -EINVAL;
10711                 goto out;
10712         }
10713         table->resource_id = resource_id;
10714         table->resource_units = resource_units;
10715         table->resource_valid = true;
10716 out:
10717         devl_unlock(devlink);
10718         return err;
10719 }
10720 EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set);
10721
10722 /**
10723  *      devlink_resource_occ_get_register - register occupancy getter
10724  *
10725  *      @devlink: devlink
10726  *      @resource_id: resource id
10727  *      @occ_get: occupancy getter callback
10728  *      @occ_get_priv: occupancy getter callback priv
10729  */
10730 void devlink_resource_occ_get_register(struct devlink *devlink,
10731                                        u64 resource_id,
10732                                        devlink_resource_occ_get_t *occ_get,
10733                                        void *occ_get_priv)
10734 {
10735         struct devlink_resource *resource;
10736
10737         devl_lock(devlink);
10738         resource = devlink_resource_find(devlink, NULL, resource_id);
10739         if (WARN_ON(!resource))
10740                 goto out;
10741         WARN_ON(resource->occ_get);
10742
10743         resource->occ_get = occ_get;
10744         resource->occ_get_priv = occ_get_priv;
10745 out:
10746         devl_unlock(devlink);
10747 }
10748 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
10749
10750 /**
10751  *      devlink_resource_occ_get_unregister - unregister occupancy getter
10752  *
10753  *      @devlink: devlink
10754  *      @resource_id: resource id
10755  */
10756 void devlink_resource_occ_get_unregister(struct devlink *devlink,
10757                                          u64 resource_id)
10758 {
10759         struct devlink_resource *resource;
10760
10761         devl_lock(devlink);
10762         resource = devlink_resource_find(devlink, NULL, resource_id);
10763         if (WARN_ON(!resource))
10764                 goto out;
10765         WARN_ON(!resource->occ_get);
10766
10767         resource->occ_get = NULL;
10768         resource->occ_get_priv = NULL;
10769 out:
10770         devl_unlock(devlink);
10771 }
10772 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
10773
10774 static int devlink_param_verify(const struct devlink_param *param)
10775 {
10776         if (!param || !param->name || !param->supported_cmodes)
10777                 return -EINVAL;
10778         if (param->generic)
10779                 return devlink_param_generic_verify(param);
10780         else
10781                 return devlink_param_driver_verify(param);
10782 }
10783
10784 /**
10785  *      devlink_params_register - register configuration parameters
10786  *
10787  *      @devlink: devlink
10788  *      @params: configuration parameters array
10789  *      @params_count: number of parameters provided
10790  *
10791  *      Register the configuration parameters supported by the driver.
10792  */
10793 int devlink_params_register(struct devlink *devlink,
10794                             const struct devlink_param *params,
10795                             size_t params_count)
10796 {
10797         const struct devlink_param *param = params;
10798         int i, err;
10799
10800         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10801
10802         for (i = 0; i < params_count; i++, param++) {
10803                 err = devlink_param_register(devlink, param);
10804                 if (err)
10805                         goto rollback;
10806         }
10807         return 0;
10808
10809 rollback:
10810         if (!i)
10811                 return err;
10812
10813         for (param--; i > 0; i--, param--)
10814                 devlink_param_unregister(devlink, param);
10815         return err;
10816 }
10817 EXPORT_SYMBOL_GPL(devlink_params_register);
10818
10819 /**
10820  *      devlink_params_unregister - unregister configuration parameters
10821  *      @devlink: devlink
10822  *      @params: configuration parameters to unregister
10823  *      @params_count: number of parameters provided
10824  */
10825 void devlink_params_unregister(struct devlink *devlink,
10826                                const struct devlink_param *params,
10827                                size_t params_count)
10828 {
10829         const struct devlink_param *param = params;
10830         int i;
10831
10832         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10833
10834         for (i = 0; i < params_count; i++, param++)
10835                 devlink_param_unregister(devlink, param);
10836 }
10837 EXPORT_SYMBOL_GPL(devlink_params_unregister);
10838
10839 /**
10840  * devlink_param_register - register one configuration parameter
10841  *
10842  * @devlink: devlink
10843  * @param: one configuration parameter
10844  *
10845  * Register the configuration parameter supported by the driver.
10846  * Return: returns 0 on successful registration or error code otherwise.
10847  */
10848 int devlink_param_register(struct devlink *devlink,
10849                            const struct devlink_param *param)
10850 {
10851         struct devlink_param_item *param_item;
10852
10853         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10854
10855         WARN_ON(devlink_param_verify(param));
10856         WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
10857
10858         if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
10859                 WARN_ON(param->get || param->set);
10860         else
10861                 WARN_ON(!param->get || !param->set);
10862
10863         param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
10864         if (!param_item)
10865                 return -ENOMEM;
10866
10867         param_item->param = param;
10868
10869         list_add_tail(&param_item->list, &devlink->param_list);
10870         return 0;
10871 }
10872 EXPORT_SYMBOL_GPL(devlink_param_register);
10873
10874 /**
10875  * devlink_param_unregister - unregister one configuration parameter
10876  * @devlink: devlink
10877  * @param: configuration parameter to unregister
10878  */
10879 void devlink_param_unregister(struct devlink *devlink,
10880                               const struct devlink_param *param)
10881 {
10882         struct devlink_param_item *param_item;
10883
10884         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10885
10886         param_item =
10887                 devlink_param_find_by_name(&devlink->param_list, param->name);
10888         WARN_ON(!param_item);
10889         list_del(&param_item->list);
10890         kfree(param_item);
10891 }
10892 EXPORT_SYMBOL_GPL(devlink_param_unregister);
10893
10894 /**
10895  *      devlink_param_driverinit_value_get - get configuration parameter
10896  *                                           value for driver initializing
10897  *
10898  *      @devlink: devlink
10899  *      @param_id: parameter ID
10900  *      @init_val: value of parameter in driverinit configuration mode
10901  *
10902  *      This function should be used by the driver to get driverinit
10903  *      configuration for initialization after reload command.
10904  */
10905 int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
10906                                        union devlink_param_value *init_val)
10907 {
10908         struct devlink_param_item *param_item;
10909
10910         if (!devlink_reload_supported(devlink->ops))
10911                 return -EOPNOTSUPP;
10912
10913         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10914         if (!param_item)
10915                 return -EINVAL;
10916
10917         if (!param_item->driverinit_value_valid ||
10918             !devlink_param_cmode_is_supported(param_item->param,
10919                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10920                 return -EOPNOTSUPP;
10921
10922         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10923                 strcpy(init_val->vstr, param_item->driverinit_value.vstr);
10924         else
10925                 *init_val = param_item->driverinit_value;
10926
10927         return 0;
10928 }
10929 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
10930
10931 /**
10932  *      devlink_param_driverinit_value_set - set value of configuration
10933  *                                           parameter for driverinit
10934  *                                           configuration mode
10935  *
10936  *      @devlink: devlink
10937  *      @param_id: parameter ID
10938  *      @init_val: value of parameter to set for driverinit configuration mode
10939  *
10940  *      This function should be used by the driver to set driverinit
10941  *      configuration mode default value.
10942  */
10943 int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
10944                                        union devlink_param_value init_val)
10945 {
10946         struct devlink_param_item *param_item;
10947
10948         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10949
10950         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10951         if (!param_item)
10952                 return -EINVAL;
10953
10954         if (!devlink_param_cmode_is_supported(param_item->param,
10955                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10956                 return -EOPNOTSUPP;
10957
10958         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10959                 strcpy(param_item->driverinit_value.vstr, init_val.vstr);
10960         else
10961                 param_item->driverinit_value = init_val;
10962         param_item->driverinit_value_valid = true;
10963         return 0;
10964 }
10965 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
10966
10967 /**
10968  *      devlink_param_value_changed - notify devlink on a parameter's value
10969  *                                    change. Should be called by the driver
10970  *                                    right after the change.
10971  *
10972  *      @devlink: devlink
10973  *      @param_id: parameter ID
10974  *
10975  *      This function should be used by the driver to notify devlink on value
10976  *      change, excluding driverinit configuration mode.
10977  *      For driverinit configuration mode driver should use the function
10978  */
10979 void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
10980 {
10981         struct devlink_param_item *param_item;
10982
10983         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10984         WARN_ON(!param_item);
10985
10986         devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
10987 }
10988 EXPORT_SYMBOL_GPL(devlink_param_value_changed);
10989
10990 /**
10991  *      devlink_region_create - create a new address region
10992  *
10993  *      @devlink: devlink
10994  *      @ops: region operations and name
10995  *      @region_max_snapshots: Maximum supported number of snapshots for region
10996  *      @region_size: size of region
10997  */
10998 struct devlink_region *
10999 devlink_region_create(struct devlink *devlink,
11000                       const struct devlink_region_ops *ops,
11001                       u32 region_max_snapshots, u64 region_size)
11002 {
11003         struct devlink_region *region;
11004         int err = 0;
11005
11006         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
11007                 return ERR_PTR(-EINVAL);
11008
11009         devl_lock(devlink);
11010
11011         if (devlink_region_get_by_name(devlink, ops->name)) {
11012                 err = -EEXIST;
11013                 goto unlock;
11014         }
11015
11016         region = kzalloc(sizeof(*region), GFP_KERNEL);
11017         if (!region) {
11018                 err = -ENOMEM;
11019                 goto unlock;
11020         }
11021
11022         region->devlink = devlink;
11023         region->max_snapshots = region_max_snapshots;
11024         region->ops = ops;
11025         region->size = region_size;
11026         INIT_LIST_HEAD(&region->snapshot_list);
11027         list_add_tail(&region->list, &devlink->region_list);
11028         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
11029
11030         devl_unlock(devlink);
11031         return region;
11032
11033 unlock:
11034         devl_unlock(devlink);
11035         return ERR_PTR(err);
11036 }
11037 EXPORT_SYMBOL_GPL(devlink_region_create);
11038
11039 /**
11040  *      devlink_port_region_create - create a new address region for a port
11041  *
11042  *      @port: devlink port
11043  *      @ops: region operations and name
11044  *      @region_max_snapshots: Maximum supported number of snapshots for region
11045  *      @region_size: size of region
11046  */
11047 struct devlink_region *
11048 devlink_port_region_create(struct devlink_port *port,
11049                            const struct devlink_port_region_ops *ops,
11050                            u32 region_max_snapshots, u64 region_size)
11051 {
11052         struct devlink *devlink = port->devlink;
11053         struct devlink_region *region;
11054         int err = 0;
11055
11056         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
11057                 return ERR_PTR(-EINVAL);
11058
11059         devl_lock(devlink);
11060
11061         if (devlink_port_region_get_by_name(port, ops->name)) {
11062                 err = -EEXIST;
11063                 goto unlock;
11064         }
11065
11066         region = kzalloc(sizeof(*region), GFP_KERNEL);
11067         if (!region) {
11068                 err = -ENOMEM;
11069                 goto unlock;
11070         }
11071
11072         region->devlink = devlink;
11073         region->port = port;
11074         region->max_snapshots = region_max_snapshots;
11075         region->port_ops = ops;
11076         region->size = region_size;
11077         INIT_LIST_HEAD(&region->snapshot_list);
11078         list_add_tail(&region->list, &port->region_list);
11079         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
11080
11081         devl_unlock(devlink);
11082         return region;
11083
11084 unlock:
11085         devl_unlock(devlink);
11086         return ERR_PTR(err);
11087 }
11088 EXPORT_SYMBOL_GPL(devlink_port_region_create);
11089
11090 /**
11091  *      devlink_region_destroy - destroy address region
11092  *
11093  *      @region: devlink region to destroy
11094  */
11095 void devlink_region_destroy(struct devlink_region *region)
11096 {
11097         struct devlink *devlink = region->devlink;
11098         struct devlink_snapshot *snapshot, *ts;
11099
11100         devl_lock(devlink);
11101
11102         /* Free all snapshots of region */
11103         list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
11104                 devlink_region_snapshot_del(region, snapshot);
11105
11106         list_del(&region->list);
11107
11108         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
11109         devl_unlock(devlink);
11110         kfree(region);
11111 }
11112 EXPORT_SYMBOL_GPL(devlink_region_destroy);
11113
11114 /**
11115  *      devlink_region_snapshot_id_get - get snapshot ID
11116  *
11117  *      This callback should be called when adding a new snapshot,
11118  *      Driver should use the same id for multiple snapshots taken
11119  *      on multiple regions at the same time/by the same trigger.
11120  *
11121  *      The caller of this function must use devlink_region_snapshot_id_put
11122  *      when finished creating regions using this id.
11123  *
11124  *      Returns zero on success, or a negative error code on failure.
11125  *
11126  *      @devlink: devlink
11127  *      @id: storage to return id
11128  */
11129 int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
11130 {
11131         int err;
11132
11133         devl_lock(devlink);
11134         err = __devlink_region_snapshot_id_get(devlink, id);
11135         devl_unlock(devlink);
11136
11137         return err;
11138 }
11139 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
11140
11141 /**
11142  *      devlink_region_snapshot_id_put - put snapshot ID reference
11143  *
11144  *      This should be called by a driver after finishing creating snapshots
11145  *      with an id. Doing so ensures that the ID can later be released in the
11146  *      event that all snapshots using it have been destroyed.
11147  *
11148  *      @devlink: devlink
11149  *      @id: id to release reference on
11150  */
11151 void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
11152 {
11153         devl_lock(devlink);
11154         __devlink_snapshot_id_decrement(devlink, id);
11155         devl_unlock(devlink);
11156 }
11157 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
11158
11159 /**
11160  *      devlink_region_snapshot_create - create a new snapshot
11161  *      This will add a new snapshot of a region. The snapshot
11162  *      will be stored on the region struct and can be accessed
11163  *      from devlink. This is useful for future analyses of snapshots.
11164  *      Multiple snapshots can be created on a region.
11165  *      The @snapshot_id should be obtained using the getter function.
11166  *
11167  *      @region: devlink region of the snapshot
11168  *      @data: snapshot data
11169  *      @snapshot_id: snapshot id to be created
11170  */
11171 int devlink_region_snapshot_create(struct devlink_region *region,
11172                                    u8 *data, u32 snapshot_id)
11173 {
11174         struct devlink *devlink = region->devlink;
11175         int err;
11176
11177         devl_lock(devlink);
11178         err = __devlink_region_snapshot_create(region, data, snapshot_id);
11179         devl_unlock(devlink);
11180
11181         return err;
11182 }
11183 EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
11184
11185 #define DEVLINK_TRAP(_id, _type)                                              \
11186         {                                                                     \
11187                 .type = DEVLINK_TRAP_TYPE_##_type,                            \
11188                 .id = DEVLINK_TRAP_GENERIC_ID_##_id,                          \
11189                 .name = DEVLINK_TRAP_GENERIC_NAME_##_id,                      \
11190         }
11191
11192 static const struct devlink_trap devlink_trap_generic[] = {
11193         DEVLINK_TRAP(SMAC_MC, DROP),
11194         DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
11195         DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
11196         DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
11197         DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
11198         DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
11199         DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
11200         DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
11201         DEVLINK_TRAP(TAIL_DROP, DROP),
11202         DEVLINK_TRAP(NON_IP_PACKET, DROP),
11203         DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
11204         DEVLINK_TRAP(DIP_LB, DROP),
11205         DEVLINK_TRAP(SIP_MC, DROP),
11206         DEVLINK_TRAP(SIP_LB, DROP),
11207         DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
11208         DEVLINK_TRAP(IPV4_SIP_BC, DROP),
11209         DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
11210         DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
11211         DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
11212         DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
11213         DEVLINK_TRAP(RPF, EXCEPTION),
11214         DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
11215         DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
11216         DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
11217         DEVLINK_TRAP(NON_ROUTABLE, DROP),
11218         DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
11219         DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
11220         DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
11221         DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
11222         DEVLINK_TRAP(STP, CONTROL),
11223         DEVLINK_TRAP(LACP, CONTROL),
11224         DEVLINK_TRAP(LLDP, CONTROL),
11225         DEVLINK_TRAP(IGMP_QUERY, CONTROL),
11226         DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
11227         DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
11228         DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
11229         DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
11230         DEVLINK_TRAP(MLD_QUERY, CONTROL),
11231         DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
11232         DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
11233         DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
11234         DEVLINK_TRAP(IPV4_DHCP, CONTROL),
11235         DEVLINK_TRAP(IPV6_DHCP, CONTROL),
11236         DEVLINK_TRAP(ARP_REQUEST, CONTROL),
11237         DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
11238         DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
11239         DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
11240         DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
11241         DEVLINK_TRAP(IPV4_BFD, CONTROL),
11242         DEVLINK_TRAP(IPV6_BFD, CONTROL),
11243         DEVLINK_TRAP(IPV4_OSPF, CONTROL),
11244         DEVLINK_TRAP(IPV6_OSPF, CONTROL),
11245         DEVLINK_TRAP(IPV4_BGP, CONTROL),
11246         DEVLINK_TRAP(IPV6_BGP, CONTROL),
11247         DEVLINK_TRAP(IPV4_VRRP, CONTROL),
11248         DEVLINK_TRAP(IPV6_VRRP, CONTROL),
11249         DEVLINK_TRAP(IPV4_PIM, CONTROL),
11250         DEVLINK_TRAP(IPV6_PIM, CONTROL),
11251         DEVLINK_TRAP(UC_LB, CONTROL),
11252         DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
11253         DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
11254         DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
11255         DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
11256         DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
11257         DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
11258         DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
11259         DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
11260         DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
11261         DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
11262         DEVLINK_TRAP(PTP_EVENT, CONTROL),
11263         DEVLINK_TRAP(PTP_GENERAL, CONTROL),
11264         DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
11265         DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
11266         DEVLINK_TRAP(EARLY_DROP, DROP),
11267         DEVLINK_TRAP(VXLAN_PARSING, DROP),
11268         DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
11269         DEVLINK_TRAP(VLAN_PARSING, DROP),
11270         DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
11271         DEVLINK_TRAP(MPLS_PARSING, DROP),
11272         DEVLINK_TRAP(ARP_PARSING, DROP),
11273         DEVLINK_TRAP(IP_1_PARSING, DROP),
11274         DEVLINK_TRAP(IP_N_PARSING, DROP),
11275         DEVLINK_TRAP(GRE_PARSING, DROP),
11276         DEVLINK_TRAP(UDP_PARSING, DROP),
11277         DEVLINK_TRAP(TCP_PARSING, DROP),
11278         DEVLINK_TRAP(IPSEC_PARSING, DROP),
11279         DEVLINK_TRAP(SCTP_PARSING, DROP),
11280         DEVLINK_TRAP(DCCP_PARSING, DROP),
11281         DEVLINK_TRAP(GTP_PARSING, DROP),
11282         DEVLINK_TRAP(ESP_PARSING, DROP),
11283         DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
11284         DEVLINK_TRAP(DMAC_FILTER, DROP),
11285 };
11286
11287 #define DEVLINK_TRAP_GROUP(_id)                                               \
11288         {                                                                     \
11289                 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,                    \
11290                 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,                \
11291         }
11292
11293 static const struct devlink_trap_group devlink_trap_group_generic[] = {
11294         DEVLINK_TRAP_GROUP(L2_DROPS),
11295         DEVLINK_TRAP_GROUP(L3_DROPS),
11296         DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
11297         DEVLINK_TRAP_GROUP(BUFFER_DROPS),
11298         DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
11299         DEVLINK_TRAP_GROUP(ACL_DROPS),
11300         DEVLINK_TRAP_GROUP(STP),
11301         DEVLINK_TRAP_GROUP(LACP),
11302         DEVLINK_TRAP_GROUP(LLDP),
11303         DEVLINK_TRAP_GROUP(MC_SNOOPING),
11304         DEVLINK_TRAP_GROUP(DHCP),
11305         DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
11306         DEVLINK_TRAP_GROUP(BFD),
11307         DEVLINK_TRAP_GROUP(OSPF),
11308         DEVLINK_TRAP_GROUP(BGP),
11309         DEVLINK_TRAP_GROUP(VRRP),
11310         DEVLINK_TRAP_GROUP(PIM),
11311         DEVLINK_TRAP_GROUP(UC_LB),
11312         DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
11313         DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
11314         DEVLINK_TRAP_GROUP(IPV6),
11315         DEVLINK_TRAP_GROUP(PTP_EVENT),
11316         DEVLINK_TRAP_GROUP(PTP_GENERAL),
11317         DEVLINK_TRAP_GROUP(ACL_SAMPLE),
11318         DEVLINK_TRAP_GROUP(ACL_TRAP),
11319         DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
11320 };
11321
11322 static int devlink_trap_generic_verify(const struct devlink_trap *trap)
11323 {
11324         if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
11325                 return -EINVAL;
11326
11327         if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
11328                 return -EINVAL;
11329
11330         if (trap->type != devlink_trap_generic[trap->id].type)
11331                 return -EINVAL;
11332
11333         return 0;
11334 }
11335
11336 static int devlink_trap_driver_verify(const struct devlink_trap *trap)
11337 {
11338         int i;
11339
11340         if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
11341                 return -EINVAL;
11342
11343         for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
11344                 if (!strcmp(trap->name, devlink_trap_generic[i].name))
11345                         return -EEXIST;
11346         }
11347
11348         return 0;
11349 }
11350
11351 static int devlink_trap_verify(const struct devlink_trap *trap)
11352 {
11353         if (!trap || !trap->name)
11354                 return -EINVAL;
11355
11356         if (trap->generic)
11357                 return devlink_trap_generic_verify(trap);
11358         else
11359                 return devlink_trap_driver_verify(trap);
11360 }
11361
11362 static int
11363 devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
11364 {
11365         if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
11366                 return -EINVAL;
11367
11368         if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
11369                 return -EINVAL;
11370
11371         return 0;
11372 }
11373
11374 static int
11375 devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
11376 {
11377         int i;
11378
11379         if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
11380                 return -EINVAL;
11381
11382         for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
11383                 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
11384                         return -EEXIST;
11385         }
11386
11387         return 0;
11388 }
11389
11390 static int devlink_trap_group_verify(const struct devlink_trap_group *group)
11391 {
11392         if (group->generic)
11393                 return devlink_trap_group_generic_verify(group);
11394         else
11395                 return devlink_trap_group_driver_verify(group);
11396 }
11397
11398 static void
11399 devlink_trap_group_notify(struct devlink *devlink,
11400                           const struct devlink_trap_group_item *group_item,
11401                           enum devlink_command cmd)
11402 {
11403         struct sk_buff *msg;
11404         int err;
11405
11406         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
11407                      cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
11408         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11409                 return;
11410
11411         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11412         if (!msg)
11413                 return;
11414
11415         err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
11416                                          0);
11417         if (err) {
11418                 nlmsg_free(msg);
11419                 return;
11420         }
11421
11422         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11423                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11424 }
11425
11426 static int
11427 devlink_trap_item_group_link(struct devlink *devlink,
11428                              struct devlink_trap_item *trap_item)
11429 {
11430         u16 group_id = trap_item->trap->init_group_id;
11431         struct devlink_trap_group_item *group_item;
11432
11433         group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
11434         if (WARN_ON_ONCE(!group_item))
11435                 return -EINVAL;
11436
11437         trap_item->group_item = group_item;
11438
11439         return 0;
11440 }
11441
11442 static void devlink_trap_notify(struct devlink *devlink,
11443                                 const struct devlink_trap_item *trap_item,
11444                                 enum devlink_command cmd)
11445 {
11446         struct sk_buff *msg;
11447         int err;
11448
11449         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
11450                      cmd != DEVLINK_CMD_TRAP_DEL);
11451         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11452                 return;
11453
11454         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11455         if (!msg)
11456                 return;
11457
11458         err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
11459         if (err) {
11460                 nlmsg_free(msg);
11461                 return;
11462         }
11463
11464         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11465                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11466 }
11467
11468 static int
11469 devlink_trap_register(struct devlink *devlink,
11470                       const struct devlink_trap *trap, void *priv)
11471 {
11472         struct devlink_trap_item *trap_item;
11473         int err;
11474
11475         if (devlink_trap_item_lookup(devlink, trap->name))
11476                 return -EEXIST;
11477
11478         trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
11479         if (!trap_item)
11480                 return -ENOMEM;
11481
11482         trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
11483         if (!trap_item->stats) {
11484                 err = -ENOMEM;
11485                 goto err_stats_alloc;
11486         }
11487
11488         trap_item->trap = trap;
11489         trap_item->action = trap->init_action;
11490         trap_item->priv = priv;
11491
11492         err = devlink_trap_item_group_link(devlink, trap_item);
11493         if (err)
11494                 goto err_group_link;
11495
11496         err = devlink->ops->trap_init(devlink, trap, trap_item);
11497         if (err)
11498                 goto err_trap_init;
11499
11500         list_add_tail(&trap_item->list, &devlink->trap_list);
11501         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
11502
11503         return 0;
11504
11505 err_trap_init:
11506 err_group_link:
11507         free_percpu(trap_item->stats);
11508 err_stats_alloc:
11509         kfree(trap_item);
11510         return err;
11511 }
11512
11513 static void devlink_trap_unregister(struct devlink *devlink,
11514                                     const struct devlink_trap *trap)
11515 {
11516         struct devlink_trap_item *trap_item;
11517
11518         trap_item = devlink_trap_item_lookup(devlink, trap->name);
11519         if (WARN_ON_ONCE(!trap_item))
11520                 return;
11521
11522         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
11523         list_del(&trap_item->list);
11524         if (devlink->ops->trap_fini)
11525                 devlink->ops->trap_fini(devlink, trap, trap_item);
11526         free_percpu(trap_item->stats);
11527         kfree(trap_item);
11528 }
11529
11530 static void devlink_trap_disable(struct devlink *devlink,
11531                                  const struct devlink_trap *trap)
11532 {
11533         struct devlink_trap_item *trap_item;
11534
11535         trap_item = devlink_trap_item_lookup(devlink, trap->name);
11536         if (WARN_ON_ONCE(!trap_item))
11537                 return;
11538
11539         devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
11540                                       NULL);
11541         trap_item->action = DEVLINK_TRAP_ACTION_DROP;
11542 }
11543
11544 /**
11545  * devlink_traps_register - Register packet traps with devlink.
11546  * @devlink: devlink.
11547  * @traps: Packet traps.
11548  * @traps_count: Count of provided packet traps.
11549  * @priv: Driver private information.
11550  *
11551  * Return: Non-zero value on failure.
11552  */
11553 int devlink_traps_register(struct devlink *devlink,
11554                            const struct devlink_trap *traps,
11555                            size_t traps_count, void *priv)
11556 {
11557         int i, err;
11558
11559         if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
11560                 return -EINVAL;
11561
11562         devl_lock(devlink);
11563         for (i = 0; i < traps_count; i++) {
11564                 const struct devlink_trap *trap = &traps[i];
11565
11566                 err = devlink_trap_verify(trap);
11567                 if (err)
11568                         goto err_trap_verify;
11569
11570                 err = devlink_trap_register(devlink, trap, priv);
11571                 if (err)
11572                         goto err_trap_register;
11573         }
11574         devl_unlock(devlink);
11575
11576         return 0;
11577
11578 err_trap_register:
11579 err_trap_verify:
11580         for (i--; i >= 0; i--)
11581                 devlink_trap_unregister(devlink, &traps[i]);
11582         devl_unlock(devlink);
11583         return err;
11584 }
11585 EXPORT_SYMBOL_GPL(devlink_traps_register);
11586
11587 /**
11588  * devlink_traps_unregister - Unregister packet traps from devlink.
11589  * @devlink: devlink.
11590  * @traps: Packet traps.
11591  * @traps_count: Count of provided packet traps.
11592  */
11593 void devlink_traps_unregister(struct devlink *devlink,
11594                               const struct devlink_trap *traps,
11595                               size_t traps_count)
11596 {
11597         int i;
11598
11599         devl_lock(devlink);
11600         /* Make sure we do not have any packets in-flight while unregistering
11601          * traps by disabling all of them and waiting for a grace period.
11602          */
11603         for (i = traps_count - 1; i >= 0; i--)
11604                 devlink_trap_disable(devlink, &traps[i]);
11605         synchronize_rcu();
11606         for (i = traps_count - 1; i >= 0; i--)
11607                 devlink_trap_unregister(devlink, &traps[i]);
11608         devl_unlock(devlink);
11609 }
11610 EXPORT_SYMBOL_GPL(devlink_traps_unregister);
11611
11612 static void
11613 devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
11614                           size_t skb_len)
11615 {
11616         struct devlink_stats *stats;
11617
11618         stats = this_cpu_ptr(trap_stats);
11619         u64_stats_update_begin(&stats->syncp);
11620         u64_stats_add(&stats->rx_bytes, skb_len);
11621         u64_stats_inc(&stats->rx_packets);
11622         u64_stats_update_end(&stats->syncp);
11623 }
11624
11625 static void
11626 devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
11627                                  const struct devlink_trap_item *trap_item,
11628                                  struct devlink_port *in_devlink_port,
11629                                  const struct flow_action_cookie *fa_cookie)
11630 {
11631         metadata->trap_name = trap_item->trap->name;
11632         metadata->trap_group_name = trap_item->group_item->group->name;
11633         metadata->fa_cookie = fa_cookie;
11634         metadata->trap_type = trap_item->trap->type;
11635
11636         spin_lock(&in_devlink_port->type_lock);
11637         if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
11638                 metadata->input_dev = in_devlink_port->type_dev;
11639         spin_unlock(&in_devlink_port->type_lock);
11640 }
11641
11642 /**
11643  * devlink_trap_report - Report trapped packet to drop monitor.
11644  * @devlink: devlink.
11645  * @skb: Trapped packet.
11646  * @trap_ctx: Trap context.
11647  * @in_devlink_port: Input devlink port.
11648  * @fa_cookie: Flow action cookie. Could be NULL.
11649  */
11650 void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
11651                          void *trap_ctx, struct devlink_port *in_devlink_port,
11652                          const struct flow_action_cookie *fa_cookie)
11653
11654 {
11655         struct devlink_trap_item *trap_item = trap_ctx;
11656
11657         devlink_trap_stats_update(trap_item->stats, skb->len);
11658         devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
11659
11660         if (trace_devlink_trap_report_enabled()) {
11661                 struct devlink_trap_metadata metadata = {};
11662
11663                 devlink_trap_report_metadata_set(&metadata, trap_item,
11664                                                  in_devlink_port, fa_cookie);
11665                 trace_devlink_trap_report(devlink, skb, &metadata);
11666         }
11667 }
11668 EXPORT_SYMBOL_GPL(devlink_trap_report);
11669
11670 /**
11671  * devlink_trap_ctx_priv - Trap context to driver private information.
11672  * @trap_ctx: Trap context.
11673  *
11674  * Return: Driver private information passed during registration.
11675  */
11676 void *devlink_trap_ctx_priv(void *trap_ctx)
11677 {
11678         struct devlink_trap_item *trap_item = trap_ctx;
11679
11680         return trap_item->priv;
11681 }
11682 EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
11683
11684 static int
11685 devlink_trap_group_item_policer_link(struct devlink *devlink,
11686                                      struct devlink_trap_group_item *group_item)
11687 {
11688         u32 policer_id = group_item->group->init_policer_id;
11689         struct devlink_trap_policer_item *policer_item;
11690
11691         if (policer_id == 0)
11692                 return 0;
11693
11694         policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
11695         if (WARN_ON_ONCE(!policer_item))
11696                 return -EINVAL;
11697
11698         group_item->policer_item = policer_item;
11699
11700         return 0;
11701 }
11702
11703 static int
11704 devlink_trap_group_register(struct devlink *devlink,
11705                             const struct devlink_trap_group *group)
11706 {
11707         struct devlink_trap_group_item *group_item;
11708         int err;
11709
11710         if (devlink_trap_group_item_lookup(devlink, group->name))
11711                 return -EEXIST;
11712
11713         group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
11714         if (!group_item)
11715                 return -ENOMEM;
11716
11717         group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
11718         if (!group_item->stats) {
11719                 err = -ENOMEM;
11720                 goto err_stats_alloc;
11721         }
11722
11723         group_item->group = group;
11724
11725         err = devlink_trap_group_item_policer_link(devlink, group_item);
11726         if (err)
11727                 goto err_policer_link;
11728
11729         if (devlink->ops->trap_group_init) {
11730                 err = devlink->ops->trap_group_init(devlink, group);
11731                 if (err)
11732                         goto err_group_init;
11733         }
11734
11735         list_add_tail(&group_item->list, &devlink->trap_group_list);
11736         devlink_trap_group_notify(devlink, group_item,
11737                                   DEVLINK_CMD_TRAP_GROUP_NEW);
11738
11739         return 0;
11740
11741 err_group_init:
11742 err_policer_link:
11743         free_percpu(group_item->stats);
11744 err_stats_alloc:
11745         kfree(group_item);
11746         return err;
11747 }
11748
11749 static void
11750 devlink_trap_group_unregister(struct devlink *devlink,
11751                               const struct devlink_trap_group *group)
11752 {
11753         struct devlink_trap_group_item *group_item;
11754
11755         group_item = devlink_trap_group_item_lookup(devlink, group->name);
11756         if (WARN_ON_ONCE(!group_item))
11757                 return;
11758
11759         devlink_trap_group_notify(devlink, group_item,
11760                                   DEVLINK_CMD_TRAP_GROUP_DEL);
11761         list_del(&group_item->list);
11762         free_percpu(group_item->stats);
11763         kfree(group_item);
11764 }
11765
11766 /**
11767  * devlink_trap_groups_register - Register packet trap groups with devlink.
11768  * @devlink: devlink.
11769  * @groups: Packet trap groups.
11770  * @groups_count: Count of provided packet trap groups.
11771  *
11772  * Return: Non-zero value on failure.
11773  */
11774 int devlink_trap_groups_register(struct devlink *devlink,
11775                                  const struct devlink_trap_group *groups,
11776                                  size_t groups_count)
11777 {
11778         int i, err;
11779
11780         devl_lock(devlink);
11781         for (i = 0; i < groups_count; i++) {
11782                 const struct devlink_trap_group *group = &groups[i];
11783
11784                 err = devlink_trap_group_verify(group);
11785                 if (err)
11786                         goto err_trap_group_verify;
11787
11788                 err = devlink_trap_group_register(devlink, group);
11789                 if (err)
11790                         goto err_trap_group_register;
11791         }
11792         devl_unlock(devlink);
11793
11794         return 0;
11795
11796 err_trap_group_register:
11797 err_trap_group_verify:
11798         for (i--; i >= 0; i--)
11799                 devlink_trap_group_unregister(devlink, &groups[i]);
11800         devl_unlock(devlink);
11801         return err;
11802 }
11803 EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
11804
11805 /**
11806  * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
11807  * @devlink: devlink.
11808  * @groups: Packet trap groups.
11809  * @groups_count: Count of provided packet trap groups.
11810  */
11811 void devlink_trap_groups_unregister(struct devlink *devlink,
11812                                     const struct devlink_trap_group *groups,
11813                                     size_t groups_count)
11814 {
11815         int i;
11816
11817         devl_lock(devlink);
11818         for (i = groups_count - 1; i >= 0; i--)
11819                 devlink_trap_group_unregister(devlink, &groups[i]);
11820         devl_unlock(devlink);
11821 }
11822 EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
11823
11824 static void
11825 devlink_trap_policer_notify(struct devlink *devlink,
11826                             const struct devlink_trap_policer_item *policer_item,
11827                             enum devlink_command cmd)
11828 {
11829         struct sk_buff *msg;
11830         int err;
11831
11832         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
11833                      cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
11834         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11835                 return;
11836
11837         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11838         if (!msg)
11839                 return;
11840
11841         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
11842                                            0, 0);
11843         if (err) {
11844                 nlmsg_free(msg);
11845                 return;
11846         }
11847
11848         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11849                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11850 }
11851
11852 static int
11853 devlink_trap_policer_register(struct devlink *devlink,
11854                               const struct devlink_trap_policer *policer)
11855 {
11856         struct devlink_trap_policer_item *policer_item;
11857         int err;
11858
11859         if (devlink_trap_policer_item_lookup(devlink, policer->id))
11860                 return -EEXIST;
11861
11862         policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
11863         if (!policer_item)
11864                 return -ENOMEM;
11865
11866         policer_item->policer = policer;
11867         policer_item->rate = policer->init_rate;
11868         policer_item->burst = policer->init_burst;
11869
11870         if (devlink->ops->trap_policer_init) {
11871                 err = devlink->ops->trap_policer_init(devlink, policer);
11872                 if (err)
11873                         goto err_policer_init;
11874         }
11875
11876         list_add_tail(&policer_item->list, &devlink->trap_policer_list);
11877         devlink_trap_policer_notify(devlink, policer_item,
11878                                     DEVLINK_CMD_TRAP_POLICER_NEW);
11879
11880         return 0;
11881
11882 err_policer_init:
11883         kfree(policer_item);
11884         return err;
11885 }
11886
11887 static void
11888 devlink_trap_policer_unregister(struct devlink *devlink,
11889                                 const struct devlink_trap_policer *policer)
11890 {
11891         struct devlink_trap_policer_item *policer_item;
11892
11893         policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
11894         if (WARN_ON_ONCE(!policer_item))
11895                 return;
11896
11897         devlink_trap_policer_notify(devlink, policer_item,
11898                                     DEVLINK_CMD_TRAP_POLICER_DEL);
11899         list_del(&policer_item->list);
11900         if (devlink->ops->trap_policer_fini)
11901                 devlink->ops->trap_policer_fini(devlink, policer);
11902         kfree(policer_item);
11903 }
11904
11905 /**
11906  * devlink_trap_policers_register - Register packet trap policers with devlink.
11907  * @devlink: devlink.
11908  * @policers: Packet trap policers.
11909  * @policers_count: Count of provided packet trap policers.
11910  *
11911  * Return: Non-zero value on failure.
11912  */
11913 int
11914 devlink_trap_policers_register(struct devlink *devlink,
11915                                const struct devlink_trap_policer *policers,
11916                                size_t policers_count)
11917 {
11918         int i, err;
11919
11920         devl_lock(devlink);
11921         for (i = 0; i < policers_count; i++) {
11922                 const struct devlink_trap_policer *policer = &policers[i];
11923
11924                 if (WARN_ON(policer->id == 0 ||
11925                             policer->max_rate < policer->min_rate ||
11926                             policer->max_burst < policer->min_burst)) {
11927                         err = -EINVAL;
11928                         goto err_trap_policer_verify;
11929                 }
11930
11931                 err = devlink_trap_policer_register(devlink, policer);
11932                 if (err)
11933                         goto err_trap_policer_register;
11934         }
11935         devl_unlock(devlink);
11936
11937         return 0;
11938
11939 err_trap_policer_register:
11940 err_trap_policer_verify:
11941         for (i--; i >= 0; i--)
11942                 devlink_trap_policer_unregister(devlink, &policers[i]);
11943         devl_unlock(devlink);
11944         return err;
11945 }
11946 EXPORT_SYMBOL_GPL(devlink_trap_policers_register);
11947
11948 /**
11949  * devlink_trap_policers_unregister - Unregister packet trap policers from devlink.
11950  * @devlink: devlink.
11951  * @policers: Packet trap policers.
11952  * @policers_count: Count of provided packet trap policers.
11953  */
11954 void
11955 devlink_trap_policers_unregister(struct devlink *devlink,
11956                                  const struct devlink_trap_policer *policers,
11957                                  size_t policers_count)
11958 {
11959         int i;
11960
11961         devl_lock(devlink);
11962         for (i = policers_count - 1; i >= 0; i--)
11963                 devlink_trap_policer_unregister(devlink, &policers[i]);
11964         devl_unlock(devlink);
11965 }
11966 EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister);
11967
11968 static void __devlink_compat_running_version(struct devlink *devlink,
11969                                              char *buf, size_t len)
11970 {
11971         const struct nlattr *nlattr;
11972         struct devlink_info_req req;
11973         struct sk_buff *msg;
11974         int rem, err;
11975
11976         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11977         if (!msg)
11978                 return;
11979
11980         req.msg = msg;
11981         err = devlink->ops->info_get(devlink, &req, NULL);
11982         if (err)
11983                 goto free_msg;
11984
11985         nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
11986                 const struct nlattr *kv;
11987                 int rem_kv;
11988
11989                 if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
11990                         continue;
11991
11992                 nla_for_each_nested(kv, nlattr, rem_kv) {
11993                         if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
11994                                 continue;
11995
11996                         strlcat(buf, nla_data(kv), len);
11997                         strlcat(buf, " ", len);
11998                 }
11999         }
12000 free_msg:
12001         nlmsg_free(msg);
12002 }
12003
12004 static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
12005 {
12006         if (!dev->netdev_ops->ndo_get_devlink_port)
12007                 return NULL;
12008
12009         return dev->netdev_ops->ndo_get_devlink_port(dev);
12010 }
12011
12012 void devlink_compat_running_version(struct devlink *devlink,
12013                                     char *buf, size_t len)
12014 {
12015         if (!devlink->ops->info_get)
12016                 return;
12017
12018         devl_lock(devlink);
12019         __devlink_compat_running_version(devlink, buf, len);
12020         devl_unlock(devlink);
12021 }
12022
12023 int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
12024 {
12025         struct devlink_flash_update_params params = {};
12026         int ret;
12027
12028         if (!devlink->ops->flash_update)
12029                 return -EOPNOTSUPP;
12030
12031         ret = request_firmware(&params.fw, file_name, devlink->dev);
12032         if (ret)
12033                 return ret;
12034
12035         devl_lock(devlink);
12036         devlink_flash_update_begin_notify(devlink);
12037         ret = devlink->ops->flash_update(devlink, &params, NULL);
12038         devlink_flash_update_end_notify(devlink);
12039         devl_unlock(devlink);
12040
12041         release_firmware(params.fw);
12042
12043         return ret;
12044 }
12045
12046 int devlink_compat_phys_port_name_get(struct net_device *dev,
12047                                       char *name, size_t len)
12048 {
12049         struct devlink_port *devlink_port;
12050
12051         /* RTNL mutex is held here which ensures that devlink_port
12052          * instance cannot disappear in the middle. No need to take
12053          * any devlink lock as only permanent values are accessed.
12054          */
12055         ASSERT_RTNL();
12056
12057         devlink_port = netdev_to_devlink_port(dev);
12058         if (!devlink_port)
12059                 return -EOPNOTSUPP;
12060
12061         return __devlink_port_phys_port_name_get(devlink_port, name, len);
12062 }
12063
12064 int devlink_compat_switch_id_get(struct net_device *dev,
12065                                  struct netdev_phys_item_id *ppid)
12066 {
12067         struct devlink_port *devlink_port;
12068
12069         /* Caller must hold RTNL mutex or reference to dev, which ensures that
12070          * devlink_port instance cannot disappear in the middle. No need to take
12071          * any devlink lock as only permanent values are accessed.
12072          */
12073         devlink_port = netdev_to_devlink_port(dev);
12074         if (!devlink_port || !devlink_port->switch_port)
12075                 return -EOPNOTSUPP;
12076
12077         memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
12078
12079         return 0;
12080 }
12081
12082 static void __net_exit devlink_pernet_pre_exit(struct net *net)
12083 {
12084         struct devlink *devlink;
12085         u32 actions_performed;
12086         unsigned long index;
12087         int err;
12088
12089         /* In case network namespace is getting destroyed, reload
12090          * all devlink instances from this namespace into init_net.
12091          */
12092         mutex_lock(&devlink_mutex);
12093         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
12094                 if (!devlink_try_get(devlink))
12095                         continue;
12096
12097                 if (!net_eq(devlink_net(devlink), net))
12098                         goto retry;
12099
12100                 WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
12101                 err = devlink_reload(devlink, &init_net,
12102                                      DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
12103                                      DEVLINK_RELOAD_LIMIT_UNSPEC,
12104                                      &actions_performed, NULL);
12105                 if (err && err != -EOPNOTSUPP)
12106                         pr_warn("Failed to reload devlink instance into init_net\n");
12107 retry:
12108                 devlink_put(devlink);
12109         }
12110         mutex_unlock(&devlink_mutex);
12111 }
12112
12113 static struct pernet_operations devlink_pernet_ops __net_initdata = {
12114         .pre_exit = devlink_pernet_pre_exit,
12115 };
12116
12117 static int __init devlink_init(void)
12118 {
12119         int err;
12120
12121         err = genl_register_family(&devlink_nl_family);
12122         if (err)
12123                 goto out;
12124         err = register_pernet_subsys(&devlink_pernet_ops);
12125
12126 out:
12127         WARN_ON(err);
12128         return err;
12129 }
12130
12131 subsys_initcall(devlink_init);