drivers/infiniband/core/nldev.c

   1 /*
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  * 3. Neither the names of the copyright holders nor the names of its
  13  *    contributors may be used to endorse or promote products derived from
  14  *    this software without specific prior written permission.
  15  *
  16  * Alternatively, this software may be distributed under the terms of the
  17  * GNU General Public License ("GPL") version 2 as published by the Free
  18  * Software Foundation.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <linux/module.h>
  34 #include <linux/pid.h>
  35 #include <linux/pid_namespace.h>
  36 #include <linux/mutex.h>
  37 #include <net/netlink.h>
  38 #include <rdma/rdma_cm.h>
  39 #include <rdma/rdma_netlink.h>
  40
  41 #include "core_priv.h"
  42 #include "cma_priv.h"
  43 #include "restrack.h"
  44 #include "uverbs.h"
  45
  46 typedef int (*res_fill_func_t)(struct sk_buff*, bool,
  47                                struct rdma_restrack_entry*, uint32_t);
  48
  49 /*
  50  * Sort array elements by the netlink attribute name
  51  */
  52 static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
  53         [RDMA_NLDEV_ATTR_CHARDEV]               = { .type = NLA_U64 },
  54         [RDMA_NLDEV_ATTR_CHARDEV_ABI]           = { .type = NLA_U64 },
  55         [RDMA_NLDEV_ATTR_CHARDEV_NAME]          = { .type = NLA_NUL_STRING,
  56                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  57         [RDMA_NLDEV_ATTR_CHARDEV_TYPE]          = { .type = NLA_NUL_STRING,
  58                                         .len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE },
  59         [RDMA_NLDEV_ATTR_DEV_DIM]               = { .type = NLA_U8 },
  60         [RDMA_NLDEV_ATTR_DEV_INDEX]             = { .type = NLA_U32 },
  61         [RDMA_NLDEV_ATTR_DEV_NAME]              = { .type = NLA_NUL_STRING,
  62                                         .len = IB_DEVICE_NAME_MAX },
  63         [RDMA_NLDEV_ATTR_DEV_NODE_TYPE]         = { .type = NLA_U8 },
  64         [RDMA_NLDEV_ATTR_DEV_PROTOCOL]          = { .type = NLA_NUL_STRING,
  65                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  66         [RDMA_NLDEV_ATTR_DRIVER]                = { .type = NLA_NESTED },
  67         [RDMA_NLDEV_ATTR_DRIVER_ENTRY]          = { .type = NLA_NESTED },
  68         [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]     = { .type = NLA_U8 },
  69         [RDMA_NLDEV_ATTR_DRIVER_STRING]         = { .type = NLA_NUL_STRING,
  70                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  71         [RDMA_NLDEV_ATTR_DRIVER_S32]            = { .type = NLA_S32 },
  72         [RDMA_NLDEV_ATTR_DRIVER_S64]            = { .type = NLA_S64 },
  73         [RDMA_NLDEV_ATTR_DRIVER_U32]            = { .type = NLA_U32 },
  74         [RDMA_NLDEV_ATTR_DRIVER_U64]            = { .type = NLA_U64 },
  75         [RDMA_NLDEV_ATTR_FW_VERSION]            = { .type = NLA_NUL_STRING,
  76                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  77         [RDMA_NLDEV_ATTR_LID]                   = { .type = NLA_U32 },
  78         [RDMA_NLDEV_ATTR_LINK_TYPE]             = { .type = NLA_NUL_STRING,
  79                                         .len = IFNAMSIZ },
  80         [RDMA_NLDEV_ATTR_LMC]                   = { .type = NLA_U8 },
  81         [RDMA_NLDEV_ATTR_NDEV_INDEX]            = { .type = NLA_U32 },
  82         [RDMA_NLDEV_ATTR_NDEV_NAME]             = { .type = NLA_NUL_STRING,
  83                                         .len = IFNAMSIZ },
  84         [RDMA_NLDEV_ATTR_NODE_GUID]             = { .type = NLA_U64 },
  85         [RDMA_NLDEV_ATTR_PORT_INDEX]            = { .type = NLA_U32 },
  86         [RDMA_NLDEV_ATTR_PORT_PHYS_STATE]       = { .type = NLA_U8 },
  87         [RDMA_NLDEV_ATTR_PORT_STATE]            = { .type = NLA_U8 },
  88         [RDMA_NLDEV_ATTR_RES_CM_ID]             = { .type = NLA_NESTED },
  89         [RDMA_NLDEV_ATTR_RES_CM_IDN]            = { .type = NLA_U32 },
  90         [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]       = { .type = NLA_NESTED },
  91         [RDMA_NLDEV_ATTR_RES_CQ]                = { .type = NLA_NESTED },
  92         [RDMA_NLDEV_ATTR_RES_CQE]               = { .type = NLA_U32 },
  93         [RDMA_NLDEV_ATTR_RES_CQN]               = { .type = NLA_U32 },
  94         [RDMA_NLDEV_ATTR_RES_CQ_ENTRY]          = { .type = NLA_NESTED },
  95         [RDMA_NLDEV_ATTR_RES_CTX]               = { .type = NLA_NESTED },
  96         [RDMA_NLDEV_ATTR_RES_CTXN]              = { .type = NLA_U32 },
  97         [RDMA_NLDEV_ATTR_RES_CTX_ENTRY]         = { .type = NLA_NESTED },
  98         [RDMA_NLDEV_ATTR_RES_DST_ADDR]          = {
  99                         .len = sizeof(struct __kernel_sockaddr_storage) },
 100         [RDMA_NLDEV_ATTR_RES_IOVA]              = { .type = NLA_U64 },
 101         [RDMA_NLDEV_ATTR_RES_KERN_NAME]         = { .type = NLA_NUL_STRING,
 102                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 103         [RDMA_NLDEV_ATTR_RES_LKEY]              = { .type = NLA_U32 },
 104         [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]    = { .type = NLA_U32 },
 105         [RDMA_NLDEV_ATTR_RES_LQPN]              = { .type = NLA_U32 },
 106         [RDMA_NLDEV_ATTR_RES_MR]                = { .type = NLA_NESTED },
 107         [RDMA_NLDEV_ATTR_RES_MRLEN]             = { .type = NLA_U64 },
 108         [RDMA_NLDEV_ATTR_RES_MRN]               = { .type = NLA_U32 },
 109         [RDMA_NLDEV_ATTR_RES_MR_ENTRY]          = { .type = NLA_NESTED },
 110         [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE]    = { .type = NLA_U8 },
 111         [RDMA_NLDEV_ATTR_RES_PD]                = { .type = NLA_NESTED },
 112         [RDMA_NLDEV_ATTR_RES_PDN]               = { .type = NLA_U32 },
 113         [RDMA_NLDEV_ATTR_RES_PD_ENTRY]          = { .type = NLA_NESTED },
 114         [RDMA_NLDEV_ATTR_RES_PID]               = { .type = NLA_U32 },
 115         [RDMA_NLDEV_ATTR_RES_POLL_CTX]          = { .type = NLA_U8 },
 116         [RDMA_NLDEV_ATTR_RES_PS]                = { .type = NLA_U32 },
 117         [RDMA_NLDEV_ATTR_RES_QP]                = { .type = NLA_NESTED },
 118         [RDMA_NLDEV_ATTR_RES_QP_ENTRY]          = { .type = NLA_NESTED },
 119         [RDMA_NLDEV_ATTR_RES_RAW]               = { .type = NLA_BINARY },
 120         [RDMA_NLDEV_ATTR_RES_RKEY]              = { .type = NLA_U32 },
 121         [RDMA_NLDEV_ATTR_RES_RQPN]              = { .type = NLA_U32 },
 122         [RDMA_NLDEV_ATTR_RES_RQ_PSN]            = { .type = NLA_U32 },
 123         [RDMA_NLDEV_ATTR_RES_SQ_PSN]            = { .type = NLA_U32 },
 124         [RDMA_NLDEV_ATTR_RES_SRC_ADDR]          = {
 125                         .len = sizeof(struct __kernel_sockaddr_storage) },
 126         [RDMA_NLDEV_ATTR_RES_STATE]             = { .type = NLA_U8 },
 127         [RDMA_NLDEV_ATTR_RES_SUMMARY]           = { .type = NLA_NESTED },
 128         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]     = { .type = NLA_NESTED },
 129         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 },
 130         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,
 131                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 132         [RDMA_NLDEV_ATTR_RES_TYPE]              = { .type = NLA_U8 },
 133         [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },
 134         [RDMA_NLDEV_ATTR_RES_USECNT]            = { .type = NLA_U64 },
 135         [RDMA_NLDEV_ATTR_RES_SRQ]               = { .type = NLA_NESTED },
 136         [RDMA_NLDEV_ATTR_RES_SRQN]              = { .type = NLA_U32 },
 137         [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY]         = { .type = NLA_NESTED },
 138         [RDMA_NLDEV_ATTR_MIN_RANGE]             = { .type = NLA_U32 },
 139         [RDMA_NLDEV_ATTR_MAX_RANGE]             = { .type = NLA_U32 },
 140         [RDMA_NLDEV_ATTR_SM_LID]                = { .type = NLA_U32 },
 141         [RDMA_NLDEV_ATTR_SUBNET_PREFIX]         = { .type = NLA_U64 },
 142         [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]   = { .type = NLA_U32 },
 143         [RDMA_NLDEV_ATTR_STAT_MODE]             = { .type = NLA_U32 },
 144         [RDMA_NLDEV_ATTR_STAT_RES]              = { .type = NLA_U32 },
 145         [RDMA_NLDEV_ATTR_STAT_COUNTER]          = { .type = NLA_NESTED },
 146         [RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY]    = { .type = NLA_NESTED },
 147         [RDMA_NLDEV_ATTR_STAT_COUNTER_ID]       = { .type = NLA_U32 },
 148         [RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]       = { .type = NLA_NESTED },
 149         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY]  = { .type = NLA_NESTED },
 150         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME] = { .type = NLA_NUL_STRING },
 151         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE] = { .type = NLA_U64 },
 152         [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID]        = { .type = NLA_U64 },
 153         [RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID]      = { .type = NLA_U32 },
 154         [RDMA_NLDEV_NET_NS_FD]                  = { .type = NLA_U32 },
 155         [RDMA_NLDEV_SYS_ATTR_NETNS_MODE]        = { .type = NLA_U8 },
 156         [RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]      = { .type = NLA_U8 },
 157         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX]  = { .type = NLA_U32 },
 158         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 },
 159 };
 160
 161 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
 162                                       enum rdma_nldev_print_type print_type)
 163 {
 164         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, name))
 165                 return -EMSGSIZE;
 166         if (print_type != RDMA_NLDEV_PRINT_TYPE_UNSPEC &&
 167             nla_put_u8(msg, RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE, print_type))
 168                 return -EMSGSIZE;
 169
 170         return 0;
 171 }
 172
 173 static int _rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name,
 174                                    enum rdma_nldev_print_type print_type,
 175                                    u32 value)
 176 {
 177         if (put_driver_name_print_type(msg, name, print_type))
 178                 return -EMSGSIZE;
 179         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DRIVER_U32, value))
 180                 return -EMSGSIZE;
 181
 182         return 0;
 183 }
 184
 185 static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name,
 186                                    enum rdma_nldev_print_type print_type,
 187                                    u64 value)
 188 {
 189         if (put_driver_name_print_type(msg, name, print_type))
 190                 return -EMSGSIZE;
 191         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_DRIVER_U64, value,
 192                               RDMA_NLDEV_ATTR_PAD))
 193                 return -EMSGSIZE;
 194
 195         return 0;
 196 }
 197
 198 int rdma_nl_put_driver_string(struct sk_buff *msg, const char *name,
 199                               const char *str)
 200 {
 201         if (put_driver_name_print_type(msg, name,
 202                                        RDMA_NLDEV_PRINT_TYPE_UNSPEC))
 203                 return -EMSGSIZE;
 204         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, str))
 205                 return -EMSGSIZE;
 206
 207         return 0;
 208 }
 209 EXPORT_SYMBOL(rdma_nl_put_driver_string);
 210
 211 int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value)
 212 {
 213         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 214                                        value);
 215 }
 216 EXPORT_SYMBOL(rdma_nl_put_driver_u32);
 217
 218 int rdma_nl_put_driver_u32_hex(struct sk_buff *msg, const char *name,
 219                                u32 value)
 220 {
 221         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 222                                        value);
 223 }
 224 EXPORT_SYMBOL(rdma_nl_put_driver_u32_hex);
 225
 226 int rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, u64 value)
 227 {
 228         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 229                                        value);
 230 }
 231 EXPORT_SYMBOL(rdma_nl_put_driver_u64);
 232
 233 int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value)
 234 {
 235         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 236                                        value);
 237 }
 238 EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);
 239
 240 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 241 {
 242         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
 243                 return -EMSGSIZE;
 244         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME,
 245                            dev_name(&device->dev)))
 246                 return -EMSGSIZE;
 247
 248         return 0;
 249 }
 250
 251 static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 252 {
 253         char fw[IB_FW_VERSION_NAME_MAX];
 254         int ret = 0;
 255         u32 port;
 256
 257         if (fill_nldev_handle(msg, device))
 258                 return -EMSGSIZE;
 259
 260         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device)))
 261                 return -EMSGSIZE;
 262
 263         BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64));
 264         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 265                               device->attrs.device_cap_flags,
 266                               RDMA_NLDEV_ATTR_PAD))
 267                 return -EMSGSIZE;
 268
 269         ib_get_device_fw_str(device, fw);
 270         /* Device without FW has strlen(fw) = 0 */
 271         if (strlen(fw) && nla_put_string(msg, RDMA_NLDEV_ATTR_FW_VERSION, fw))
 272                 return -EMSGSIZE;
 273
 274         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID,
 275                               be64_to_cpu(device->node_guid),
 276                               RDMA_NLDEV_ATTR_PAD))
 277                 return -EMSGSIZE;
 278         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID,
 279                               be64_to_cpu(device->attrs.sys_image_guid),
 280                               RDMA_NLDEV_ATTR_PAD))
 281                 return -EMSGSIZE;
 282         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
 283                 return -EMSGSIZE;
 284         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, device->use_cq_dim))
 285                 return -EMSGSIZE;
 286
 287         /*
 288          * Link type is determined on first port and mlx4 device
 289          * which can potentially have two different link type for the same
 290          * IB device is considered as better to be avoided in the future,
 291          */
 292         port = rdma_start_port(device);
 293         if (rdma_cap_opa_mad(device, port))
 294                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "opa");
 295         else if (rdma_protocol_ib(device, port))
 296                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "ib");
 297         else if (rdma_protocol_iwarp(device, port))
 298                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "iw");
 299         else if (rdma_protocol_roce(device, port))
 300                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "roce");
 301         else if (rdma_protocol_usnic(device, port))
 302                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL,
 303                                      "usnic");
 304         return ret;
 305 }
 306
 307 static int fill_port_info(struct sk_buff *msg,
 308                           struct ib_device *device, u32 port,
 309                           const struct net *net)
 310 {
 311         struct net_device *netdev = NULL;
 312         struct ib_port_attr attr;
 313         int ret;
 314         u64 cap_flags = 0;
 315
 316         if (fill_nldev_handle(msg, device))
 317                 return -EMSGSIZE;
 318
 319         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
 320                 return -EMSGSIZE;
 321
 322         ret = ib_query_port(device, port, &attr);
 323         if (ret)
 324                 return ret;
 325
 326         if (rdma_protocol_ib(device, port)) {
 327                 BUILD_BUG_ON((sizeof(attr.port_cap_flags) +
 328                                 sizeof(attr.port_cap_flags2)) > sizeof(u64));
 329                 cap_flags = attr.port_cap_flags |
 330                         ((u64)attr.port_cap_flags2 << 32);
 331                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 332                                       cap_flags, RDMA_NLDEV_ATTR_PAD))
 333                         return -EMSGSIZE;
 334                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX,
 335                                       attr.subnet_prefix, RDMA_NLDEV_ATTR_PAD))
 336                         return -EMSGSIZE;
 337                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid))
 338                         return -EMSGSIZE;
 339                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_SM_LID, attr.sm_lid))
 340                         return -EMSGSIZE;
 341                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_LMC, attr.lmc))
 342                         return -EMSGSIZE;
 343         }
 344         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_STATE, attr.state))
 345                 return -EMSGSIZE;
 346         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_PHYS_STATE, attr.phys_state))
 347                 return -EMSGSIZE;
 348
 349         netdev = ib_device_get_netdev(device, port);
 350         if (netdev && net_eq(dev_net(netdev), net)) {
 351                 ret = nla_put_u32(msg,
 352                                   RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex);
 353                 if (ret)
 354                         goto out;
 355                 ret = nla_put_string(msg,
 356                                      RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name);
 357         }
 358
 359 out:
 360         if (netdev)
 361                 dev_put(netdev);
 362         return ret;
 363 }
 364
 365 static int fill_res_info_entry(struct sk_buff *msg,
 366                                const char *name, u64 curr)
 367 {
 368         struct nlattr *entry_attr;
 369
 370         entry_attr = nla_nest_start_noflag(msg,
 371                                            RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
 372         if (!entry_attr)
 373                 return -EMSGSIZE;
 374
 375         if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
 376                 goto err;
 377         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr,
 378                               RDMA_NLDEV_ATTR_PAD))
 379                 goto err;
 380
 381         nla_nest_end(msg, entry_attr);
 382         return 0;
 383
 384 err:
 385         nla_nest_cancel(msg, entry_attr);
 386         return -EMSGSIZE;
 387 }
 388
 389 static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
 390 {
 391         static const char * const names[RDMA_RESTRACK_MAX] = {
 392                 [RDMA_RESTRACK_PD] = "pd",
 393                 [RDMA_RESTRACK_CQ] = "cq",
 394                 [RDMA_RESTRACK_QP] = "qp",
 395                 [RDMA_RESTRACK_CM_ID] = "cm_id",
 396                 [RDMA_RESTRACK_MR] = "mr",
 397                 [RDMA_RESTRACK_CTX] = "ctx",
 398                 [RDMA_RESTRACK_SRQ] = "srq",
 399         };
 400
 401         struct nlattr *table_attr;
 402         int ret, i, curr;
 403
 404         if (fill_nldev_handle(msg, device))
 405                 return -EMSGSIZE;
 406
 407         table_attr = nla_nest_start_noflag(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
 408         if (!table_attr)
 409                 return -EMSGSIZE;
 410
 411         for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
 412                 if (!names[i])
 413                         continue;
 414                 curr = rdma_restrack_count(device, i);
 415                 ret = fill_res_info_entry(msg, names[i], curr);
 416                 if (ret)
 417                         goto err;
 418         }
 419
 420         nla_nest_end(msg, table_attr);
 421         return 0;
 422
 423 err:
 424         nla_nest_cancel(msg, table_attr);
 425         return ret;
 426 }
 427
 428 static int fill_res_name_pid(struct sk_buff *msg,
 429                              struct rdma_restrack_entry *res)
 430 {
 431         int err = 0;
 432
 433         /*
 434          * For user resources, user is should read /proc/PID/comm to get the
 435          * name of the task file.
 436          */
 437         if (rdma_is_kernel_res(res)) {
 438                 err = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,
 439                                      res->kern_name);
 440         } else {
 441                 pid_t pid;
 442
 443                 pid = task_pid_vnr(res->task);
 444                 /*
 445                  * Task is dead and in zombie state.
 446                  * There is no need to print PID anymore.
 447                  */
 448                 if (pid)
 449                         /*
 450                          * This part is racy, task can be killed and PID will
 451                          * be zero right here but it is ok, next query won't
 452                          * return PID. We don't promise real-time reflection
 453                          * of SW objects.
 454                          */
 455                         err = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, pid);
 456         }
 457
 458         return err ? -EMSGSIZE : 0;
 459 }
 460
 461 static int fill_res_qp_entry_query(struct sk_buff *msg,
 462                                    struct rdma_restrack_entry *res,
 463                                    struct ib_device *dev,
 464                                    struct ib_qp *qp)
 465 {
 466         struct ib_qp_init_attr qp_init_attr;
 467         struct ib_qp_attr qp_attr;
 468         int ret;
 469
 470         ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr);
 471         if (ret)
 472                 return ret;
 473
 474         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
 475                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
 476                                 qp_attr.dest_qp_num))
 477                         goto err;
 478                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQ_PSN,
 479                                 qp_attr.rq_psn))
 480                         goto err;
 481         }
 482
 483         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SQ_PSN, qp_attr.sq_psn))
 484                 goto err;
 485
 486         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC ||
 487             qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) {
 488                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE,
 489                                qp_attr.path_mig_state))
 490                         goto err;
 491         }
 492         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type))
 493                 goto err;
 494         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
 495                 goto err;
 496
 497         if (dev->ops.fill_res_qp_entry)
 498                 return dev->ops.fill_res_qp_entry(msg, qp);
 499         return 0;
 500
 501 err:    return -EMSGSIZE;
 502 }
 503
 504 static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
 505                              struct rdma_restrack_entry *res, uint32_t port)
 506 {
 507         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 508         struct ib_device *dev = qp->device;
 509         int ret;
 510
 511         if (port && port != qp->port)
 512                 return -EAGAIN;
 513
 514         /* In create_qp() port is not set yet */
 515         if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
 516                 return -EMSGSIZE;
 517
 518         ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
 519         if (ret)
 520                 return -EMSGSIZE;
 521
 522         if (!rdma_is_kernel_res(res) &&
 523             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
 524                 return -EMSGSIZE;
 525
 526         ret = fill_res_name_pid(msg, res);
 527         if (ret)
 528                 return -EMSGSIZE;
 529
 530         return fill_res_qp_entry_query(msg, res, dev, qp);
 531 }
 532
 533 static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 534                                  struct rdma_restrack_entry *res, uint32_t port)
 535 {
 536         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 537         struct ib_device *dev = qp->device;
 538
 539         if (port && port != qp->port)
 540                 return -EAGAIN;
 541         if (!dev->ops.fill_res_qp_entry_raw)
 542                 return -EINVAL;
 543         return dev->ops.fill_res_qp_entry_raw(msg, qp);
 544 }
 545
 546 static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,
 547                                 struct rdma_restrack_entry *res, uint32_t port)
 548 {
 549         struct rdma_id_private *id_priv =
 550                                 container_of(res, struct rdma_id_private, res);
 551         struct ib_device *dev = id_priv->id.device;
 552         struct rdma_cm_id *cm_id = &id_priv->id;
 553
 554         if (port && port != cm_id->port_num)
 555                 return -EAGAIN;
 556
 557         if (cm_id->port_num &&
 558             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
 559                 goto err;
 560
 561         if (id_priv->qp_num) {
 562                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv->qp_num))
 563                         goto err;
 564                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type))
 565                         goto err;
 566         }
 567
 568         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps))
 569                 goto err;
 570
 571         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state))
 572                 goto err;
 573
 574         if (cm_id->route.addr.src_addr.ss_family &&
 575             nla_put(msg, RDMA_NLDEV_ATTR_RES_SRC_ADDR,
 576                     sizeof(cm_id->route.addr.src_addr),
 577                     &cm_id->route.addr.src_addr))
 578                 goto err;
 579         if (cm_id->route.addr.dst_addr.ss_family &&
 580             nla_put(msg, RDMA_NLDEV_ATTR_RES_DST_ADDR,
 581                     sizeof(cm_id->route.addr.dst_addr),
 582                     &cm_id->route.addr.dst_addr))
 583                 goto err;
 584
 585         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CM_IDN, res->id))
 586                 goto err;
 587
 588         if (fill_res_name_pid(msg, res))
 589                 goto err;
 590
 591         if (dev->ops.fill_res_cm_id_entry)
 592                 return dev->ops.fill_res_cm_id_entry(msg, cm_id);
 593         return 0;
 594
 595 err: return -EMSGSIZE;
 596 }
 597
 598 static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 599                              struct rdma_restrack_entry *res, uint32_t port)
 600 {
 601         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 602         struct ib_device *dev = cq->device;
 603
 604         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
 605                 return -EMSGSIZE;
 606         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 607                               atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
 608                 return -EMSGSIZE;
 609
 610         /* Poll context is only valid for kernel CQs */
 611         if (rdma_is_kernel_res(res) &&
 612             nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx))
 613                 return -EMSGSIZE;
 614
 615         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL)))
 616                 return -EMSGSIZE;
 617
 618         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
 619                 return -EMSGSIZE;
 620         if (!rdma_is_kernel_res(res) &&
 621             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 622                         cq->uobject->uevent.uobject.context->res.id))
 623                 return -EMSGSIZE;
 624
 625         if (fill_res_name_pid(msg, res))
 626                 return -EMSGSIZE;
 627
 628         return (dev->ops.fill_res_cq_entry) ?
 629                 dev->ops.fill_res_cq_entry(msg, cq) : 0;
 630 }
 631
 632 static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 633                                  struct rdma_restrack_entry *res, uint32_t port)
 634 {
 635         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 636         struct ib_device *dev = cq->device;
 637
 638         if (!dev->ops.fill_res_cq_entry_raw)
 639                 return -EINVAL;
 640         return dev->ops.fill_res_cq_entry_raw(msg, cq);
 641 }
 642
 643 static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 644                              struct rdma_restrack_entry *res, uint32_t port)
 645 {
 646         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 647         struct ib_device *dev = mr->pd->device;
 648
 649         if (has_cap_net_admin) {
 650                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
 651                         return -EMSGSIZE;
 652                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
 653                         return -EMSGSIZE;
 654         }
 655
 656         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
 657                               RDMA_NLDEV_ATTR_PAD))
 658                 return -EMSGSIZE;
 659
 660         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 661                 return -EMSGSIZE;
 662
 663         if (!rdma_is_kernel_res(res) &&
 664             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
 665                 return -EMSGSIZE;
 666
 667         if (fill_res_name_pid(msg, res))
 668                 return -EMSGSIZE;
 669
 670         return (dev->ops.fill_res_mr_entry) ?
 671                        dev->ops.fill_res_mr_entry(msg, mr) :
 672                        0;
 673 }
 674
 675 static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 676                                  struct rdma_restrack_entry *res, uint32_t port)
 677 {
 678         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 679         struct ib_device *dev = mr->pd->device;
 680
 681         if (!dev->ops.fill_res_mr_entry_raw)
 682                 return -EINVAL;
 683         return dev->ops.fill_res_mr_entry_raw(msg, mr);
 684 }
 685
 686 static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 687                              struct rdma_restrack_entry *res, uint32_t port)
 688 {
 689         struct ib_pd *pd = container_of(res, struct ib_pd, res);
 690
 691         if (has_cap_net_admin) {
 692                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,
 693                                 pd->local_dma_lkey))
 694                         goto err;
 695                 if ((pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) &&
 696                     nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,
 697                                 pd->unsafe_global_rkey))
 698                         goto err;
 699         }
 700         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 701                               atomic_read(&pd->usecnt), RDMA_NLDEV_ATTR_PAD))
 702                 goto err;
 703
 704         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
 705                 goto err;
 706
 707         if (!rdma_is_kernel_res(res) &&
 708             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 709                         pd->uobject->context->res.id))
 710                 goto err;
 711
 712         return fill_res_name_pid(msg, res);
 713
 714 err:    return -EMSGSIZE;
 715 }
 716
 717 static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,
 718                               struct rdma_restrack_entry *res, uint32_t port)
 719 {
 720         struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res);
 721
 722         if (rdma_is_kernel_res(res))
 723                 return 0;
 724
 725         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, ctx->res.id))
 726                 return -EMSGSIZE;
 727
 728         return fill_res_name_pid(msg, res);
 729 }
 730
 731 static int fill_res_range_qp_entry(struct sk_buff *msg, uint32_t min_range,
 732                                    uint32_t max_range)
 733 {
 734         struct nlattr *entry_attr;
 735
 736         if (!min_range)
 737                 return 0;
 738
 739         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 740         if (!entry_attr)
 741                 return -EMSGSIZE;
 742
 743         if (min_range == max_range) {
 744                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, min_range))
 745                         goto err;
 746         } else {
 747                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MIN_RANGE, min_range))
 748                         goto err;
 749                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MAX_RANGE, max_range))
 750                         goto err;
 751         }
 752         nla_nest_end(msg, entry_attr);
 753         return 0;
 754
 755 err:
 756         nla_nest_cancel(msg, entry_attr);
 757         return -EMSGSIZE;
 758 }
 759
 760 static int fill_res_srq_qps(struct sk_buff *msg, struct ib_srq *srq)
 761 {
 762         uint32_t min_range = 0, prev = 0;
 763         struct rdma_restrack_entry *res;
 764         struct rdma_restrack_root *rt;
 765         struct nlattr *table_attr;
 766         struct ib_qp *qp = NULL;
 767         unsigned long id = 0;
 768
 769         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 770         if (!table_attr)
 771                 return -EMSGSIZE;
 772
 773         rt = &srq->device->res[RDMA_RESTRACK_QP];
 774         xa_lock(&rt->xa);
 775         xa_for_each(&rt->xa, id, res) {
 776                 if (!rdma_restrack_get(res))
 777                         continue;
 778
 779                 qp = container_of(res, struct ib_qp, res);
 780                 if (!qp->srq || (qp->srq->res.id != srq->res.id)) {
 781                         rdma_restrack_put(res);
 782                         continue;
 783                 }
 784
 785                 if (qp->qp_num < prev)
 786                         /* qp_num should be ascending */
 787                         goto err_loop;
 788
 789                 if (min_range == 0) {
 790                         min_range = qp->qp_num;
 791                 } else if (qp->qp_num > (prev + 1)) {
 792                         if (fill_res_range_qp_entry(msg, min_range, prev))
 793                                 goto err_loop;
 794
 795                         min_range = qp->qp_num;
 796                 }
 797                 prev = qp->qp_num;
 798                 rdma_restrack_put(res);
 799         }
 800
 801         xa_unlock(&rt->xa);
 802
 803         if (fill_res_range_qp_entry(msg, min_range, prev))
 804                 goto err;
 805
 806         nla_nest_end(msg, table_attr);
 807         return 0;
 808
 809 err_loop:
 810         rdma_restrack_put(res);
 811         xa_unlock(&rt->xa);
 812 err:
 813         nla_nest_cancel(msg, table_attr);
 814         return -EMSGSIZE;
 815 }
 816
 817 static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 818                               struct rdma_restrack_entry *res, uint32_t port)
 819 {
 820         struct ib_srq *srq = container_of(res, struct ib_srq, res);
 821
 822         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SRQN, srq->res.id))
 823                 goto err;
 824
 825         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, srq->srq_type))
 826                 goto err;
 827
 828         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, srq->pd->res.id))
 829                 goto err;
 830
 831         if (ib_srq_has_cq(srq->srq_type)) {
 832                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN,
 833                                 srq->ext.cq->res.id))
 834                         goto err;
 835         }
 836
 837         if (fill_res_srq_qps(msg, srq))
 838                 goto err;
 839
 840         return fill_res_name_pid(msg, res);
 841
 842 err:
 843         return -EMSGSIZE;
 844 }
 845
 846 static int fill_stat_counter_mode(struct sk_buff *msg,
 847                                   struct rdma_counter *counter)
 848 {
 849         struct rdma_counter_mode *m = &counter->mode;
 850
 851         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode))
 852                 return -EMSGSIZE;
 853
 854         if (m->mode == RDMA_COUNTER_MODE_AUTO) {
 855                 if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) &&
 856                     nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type))
 857                         return -EMSGSIZE;
 858
 859                 if ((m->mask & RDMA_COUNTER_MASK_PID) &&
 860                     fill_res_name_pid(msg, &counter->res))
 861                         return -EMSGSIZE;
 862         }
 863
 864         return 0;
 865 }
 866
 867 static int fill_stat_counter_qp_entry(struct sk_buff *msg, u32 qpn)
 868 {
 869         struct nlattr *entry_attr;
 870
 871         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 872         if (!entry_attr)
 873                 return -EMSGSIZE;
 874
 875         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn))
 876                 goto err;
 877
 878         nla_nest_end(msg, entry_attr);
 879         return 0;
 880
 881 err:
 882         nla_nest_cancel(msg, entry_attr);
 883         return -EMSGSIZE;
 884 }
 885
 886 static int fill_stat_counter_qps(struct sk_buff *msg,
 887                                  struct rdma_counter *counter)
 888 {
 889         struct rdma_restrack_entry *res;
 890         struct rdma_restrack_root *rt;
 891         struct nlattr *table_attr;
 892         struct ib_qp *qp = NULL;
 893         unsigned long id = 0;
 894         int ret = 0;
 895
 896         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 897         if (!table_attr)
 898                 return -EMSGSIZE;
 899
 900         rt = &counter->device->res[RDMA_RESTRACK_QP];
 901         xa_lock(&rt->xa);
 902         xa_for_each(&rt->xa, id, res) {
 903                 qp = container_of(res, struct ib_qp, res);
 904                 if (!qp->counter || (qp->counter->id != counter->id))
 905                         continue;
 906
 907                 ret = fill_stat_counter_qp_entry(msg, qp->qp_num);
 908                 if (ret)
 909                         goto err;
 910         }
 911
 912         xa_unlock(&rt->xa);
 913         nla_nest_end(msg, table_attr);
 914         return 0;
 915
 916 err:
 917         xa_unlock(&rt->xa);
 918         nla_nest_cancel(msg, table_attr);
 919         return ret;
 920 }
 921
 922 int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name,
 923                                  u64 value)
 924 {
 925         struct nlattr *entry_attr;
 926
 927         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);
 928         if (!entry_attr)
 929                 return -EMSGSIZE;
 930
 931         if (nla_put_string(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,
 932                            name))
 933                 goto err;
 934         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE,
 935                               value, RDMA_NLDEV_ATTR_PAD))
 936                 goto err;
 937
 938         nla_nest_end(msg, entry_attr);
 939         return 0;
 940
 941 err:
 942         nla_nest_cancel(msg, entry_attr);
 943         return -EMSGSIZE;
 944 }
 945 EXPORT_SYMBOL(rdma_nl_stat_hwcounter_entry);
 946
 947 static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 948                               struct rdma_restrack_entry *res, uint32_t port)
 949 {
 950         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 951         struct ib_device *dev = mr->pd->device;
 952
 953         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 954                 goto err;
 955
 956         if (dev->ops.fill_stat_mr_entry)
 957                 return dev->ops.fill_stat_mr_entry(msg, mr);
 958         return 0;
 959
 960 err:
 961         return -EMSGSIZE;
 962 }
 963
 964 static int fill_stat_counter_hwcounters(struct sk_buff *msg,
 965                                         struct rdma_counter *counter)
 966 {
 967         struct rdma_hw_stats *st = counter->stats;
 968         struct nlattr *table_attr;
 969         int i;
 970
 971         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
 972         if (!table_attr)
 973                 return -EMSGSIZE;
 974
 975         mutex_lock(&st->lock);
 976         for (i = 0; i < st->num_counters; i++) {
 977                 if (test_bit(i, st->is_disabled))
 978                         continue;
 979                 if (rdma_nl_stat_hwcounter_entry(msg, st->descs[i].name,
 980                                                  st->value[i]))
 981                         goto err;
 982         }
 983         mutex_unlock(&st->lock);
 984
 985         nla_nest_end(msg, table_attr);
 986         return 0;
 987
 988 err:
 989         mutex_unlock(&st->lock);
 990         nla_nest_cancel(msg, table_attr);
 991         return -EMSGSIZE;
 992 }
 993
 994 static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin,
 995                                   struct rdma_restrack_entry *res,
 996                                   uint32_t port)
 997 {
 998         struct rdma_counter *counter =
 999                 container_of(res, struct rdma_counter, res);
1000
1001         if (port && port != counter->port)
1002                 return -EAGAIN;
1003
1004         /* Dump it even query failed */
1005         rdma_counter_query_stats(counter);
1006
1007         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) ||
1008             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) ||
1009             fill_stat_counter_mode(msg, counter) ||
1010             fill_stat_counter_qps(msg, counter) ||
1011             fill_stat_counter_hwcounters(msg, counter))
1012                 return -EMSGSIZE;
1013
1014         return 0;
1015 }
1016
1017 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1018                           struct netlink_ext_ack *extack)
1019 {
1020         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1021         struct ib_device *device;
1022         struct sk_buff *msg;
1023         u32 index;
1024         int err;
1025
1026         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1027                                      nldev_policy, extack);
1028         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1029                 return -EINVAL;
1030
1031         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1032
1033         device = ib_device_get_by_index(sock_net(skb->sk), index);
1034         if (!device)
1035                 return -EINVAL;
1036
1037         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1038         if (!msg) {
1039                 err = -ENOMEM;
1040                 goto err;
1041         }
1042
1043         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1044                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1045                         0, 0);
1046         if (!nlh) {
1047                 err = -EMSGSIZE;
1048                 goto err_free;
1049         }
1050
1051         err = fill_dev_info(msg, device);
1052         if (err)
1053                 goto err_free;
1054
1055         nlmsg_end(msg, nlh);
1056
1057         ib_device_put(device);
1058         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1059
1060 err_free:
1061         nlmsg_free(msg);
1062 err:
1063         ib_device_put(device);
1064         return err;
1065 }
1066
1067 static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1068                           struct netlink_ext_ack *extack)
1069 {
1070         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1071         struct ib_device *device;
1072         u32 index;
1073         int err;
1074
1075         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1076                                      nldev_policy, extack);
1077         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1078                 return -EINVAL;
1079
1080         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1081         device = ib_device_get_by_index(sock_net(skb->sk), index);
1082         if (!device)
1083                 return -EINVAL;
1084
1085         if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) {
1086                 char name[IB_DEVICE_NAME_MAX] = {};
1087
1088                 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1089                             IB_DEVICE_NAME_MAX);
1090                 if (strlen(name) == 0) {
1091                         err = -EINVAL;
1092                         goto done;
1093                 }
1094                 err = ib_device_rename(device, name);
1095                 goto done;
1096         }
1097
1098         if (tb[RDMA_NLDEV_NET_NS_FD]) {
1099                 u32 ns_fd;
1100
1101                 ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]);
1102                 err = ib_device_set_netns_put(skb, device, ns_fd);
1103                 goto put_done;
1104         }
1105
1106         if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) {
1107                 u8 use_dim;
1108
1109                 use_dim = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]);
1110                 err = ib_device_set_dim(device,  use_dim);
1111                 goto done;
1112         }
1113
1114 done:
1115         ib_device_put(device);
1116 put_done:
1117         return err;
1118 }
1119
1120 static int _nldev_get_dumpit(struct ib_device *device,
1121                              struct sk_buff *skb,
1122                              struct netlink_callback *cb,
1123                              unsigned int idx)
1124 {
1125         int start = cb->args[0];
1126         struct nlmsghdr *nlh;
1127
1128         if (idx < start)
1129                 return 0;
1130
1131         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1132                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1133                         0, NLM_F_MULTI);
1134
1135         if (!nlh || fill_dev_info(skb, device)) {
1136                 nlmsg_cancel(skb, nlh);
1137                 goto out;
1138         }
1139
1140         nlmsg_end(skb, nlh);
1141
1142         idx++;
1143
1144 out:    cb->args[0] = idx;
1145         return skb->len;
1146 }
1147
1148 static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
1149 {
1150         /*
1151          * There is no need to take lock, because
1152          * we are relying on ib_core's locking.
1153          */
1154         return ib_enum_all_devs(_nldev_get_dumpit, skb, cb);
1155 }
1156
1157 static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1158                                struct netlink_ext_ack *extack)
1159 {
1160         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1161         struct ib_device *device;
1162         struct sk_buff *msg;
1163         u32 index;
1164         u32 port;
1165         int err;
1166
1167         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1168                                      nldev_policy, extack);
1169         if (err ||
1170             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
1171             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
1172                 return -EINVAL;
1173
1174         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1175         device = ib_device_get_by_index(sock_net(skb->sk), index);
1176         if (!device)
1177                 return -EINVAL;
1178
1179         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1180         if (!rdma_is_port_valid(device, port)) {
1181                 err = -EINVAL;
1182                 goto err;
1183         }
1184
1185         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1186         if (!msg) {
1187                 err = -ENOMEM;
1188                 goto err;
1189         }
1190
1191         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1192                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1193                         0, 0);
1194         if (!nlh) {
1195                 err = -EMSGSIZE;
1196                 goto err_free;
1197         }
1198
1199         err = fill_port_info(msg, device, port, sock_net(skb->sk));
1200         if (err)
1201                 goto err_free;
1202
1203         nlmsg_end(msg, nlh);
1204         ib_device_put(device);
1205
1206         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1207
1208 err_free:
1209         nlmsg_free(msg);
1210 err:
1211         ib_device_put(device);
1212         return err;
1213 }
1214
1215 static int nldev_port_get_dumpit(struct sk_buff *skb,
1216                                  struct netlink_callback *cb)
1217 {
1218         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1219         struct ib_device *device;
1220         int start = cb->args[0];
1221         struct nlmsghdr *nlh;
1222         u32 idx = 0;
1223         u32 ifindex;
1224         int err;
1225         unsigned int p;
1226
1227         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1228                                      nldev_policy, NULL);
1229         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1230                 return -EINVAL;
1231
1232         ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1233         device = ib_device_get_by_index(sock_net(skb->sk), ifindex);
1234         if (!device)
1235                 return -EINVAL;
1236
1237         rdma_for_each_port (device, p) {
1238                 /*
1239                  * The dumpit function returns all information from specific
1240                  * index. This specific index is taken from the netlink
1241                  * messages request sent by user and it is available
1242                  * in cb->args[0].
1243                  *
1244                  * Usually, the user doesn't fill this field and it causes
1245                  * to return everything.
1246                  *
1247                  */
1248                 if (idx < start) {
1249                         idx++;
1250                         continue;
1251                 }
1252
1253                 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
1254                                 cb->nlh->nlmsg_seq,
1255                                 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1256                                                  RDMA_NLDEV_CMD_PORT_GET),
1257                                 0, NLM_F_MULTI);
1258
1259                 if (!nlh || fill_port_info(skb, device, p, sock_net(skb->sk))) {
1260                         nlmsg_cancel(skb, nlh);
1261                         goto out;
1262                 }
1263                 idx++;
1264                 nlmsg_end(skb, nlh);
1265         }
1266
1267 out:
1268         ib_device_put(device);
1269         cb->args[0] = idx;
1270         return skb->len;
1271 }
1272
1273 static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1274                               struct netlink_ext_ack *extack)
1275 {
1276         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1277         struct ib_device *device;
1278         struct sk_buff *msg;
1279         u32 index;
1280         int ret;
1281
1282         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1283                                      nldev_policy, extack);
1284         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1285                 return -EINVAL;
1286
1287         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1288         device = ib_device_get_by_index(sock_net(skb->sk), index);
1289         if (!device)
1290                 return -EINVAL;
1291
1292         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1293         if (!msg) {
1294                 ret = -ENOMEM;
1295                 goto err;
1296         }
1297
1298         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1299                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1300                         0, 0);
1301         if (!nlh) {
1302                 ret = -EMSGSIZE;
1303                 goto err_free;
1304         }
1305
1306         ret = fill_res_info(msg, device);
1307         if (ret)
1308                 goto err_free;
1309
1310         nlmsg_end(msg, nlh);
1311         ib_device_put(device);
1312         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1313
1314 err_free:
1315         nlmsg_free(msg);
1316 err:
1317         ib_device_put(device);
1318         return ret;
1319 }
1320
1321 static int _nldev_res_get_dumpit(struct ib_device *device,
1322                                  struct sk_buff *skb,
1323                                  struct netlink_callback *cb,
1324                                  unsigned int idx)
1325 {
1326         int start = cb->args[0];
1327         struct nlmsghdr *nlh;
1328
1329         if (idx < start)
1330                 return 0;
1331
1332         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1333                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1334                         0, NLM_F_MULTI);
1335
1336         if (!nlh || fill_res_info(skb, device)) {
1337                 nlmsg_cancel(skb, nlh);
1338                 goto out;
1339         }
1340         nlmsg_end(skb, nlh);
1341
1342         idx++;
1343
1344 out:
1345         cb->args[0] = idx;
1346         return skb->len;
1347 }
1348
1349 static int nldev_res_get_dumpit(struct sk_buff *skb,
1350                                 struct netlink_callback *cb)
1351 {
1352         return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
1353 }
1354
1355 struct nldev_fill_res_entry {
1356         enum rdma_nldev_attr nldev_attr;
1357         u8 flags;
1358         u32 entry;
1359         u32 id;
1360 };
1361
1362 enum nldev_res_flags {
1363         NLDEV_PER_DEV = 1 << 0,
1364 };
1365
1366 static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
1367         [RDMA_RESTRACK_QP] = {
1368                 .nldev_attr = RDMA_NLDEV_ATTR_RES_QP,
1369                 .entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,
1370                 .id = RDMA_NLDEV_ATTR_RES_LQPN,
1371         },
1372         [RDMA_RESTRACK_CM_ID] = {
1373                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,
1374                 .entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,
1375                 .id = RDMA_NLDEV_ATTR_RES_CM_IDN,
1376         },
1377         [RDMA_RESTRACK_CQ] = {
1378                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
1379                 .flags = NLDEV_PER_DEV,
1380                 .entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,
1381                 .id = RDMA_NLDEV_ATTR_RES_CQN,
1382         },
1383         [RDMA_RESTRACK_MR] = {
1384                 .nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
1385                 .flags = NLDEV_PER_DEV,
1386                 .entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,
1387                 .id = RDMA_NLDEV_ATTR_RES_MRN,
1388         },
1389         [RDMA_RESTRACK_PD] = {
1390                 .nldev_attr = RDMA_NLDEV_ATTR_RES_PD,
1391                 .flags = NLDEV_PER_DEV,
1392                 .entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,
1393                 .id = RDMA_NLDEV_ATTR_RES_PDN,
1394         },
1395         [RDMA_RESTRACK_COUNTER] = {
1396                 .nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,
1397                 .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,
1398                 .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,
1399         },
1400         [RDMA_RESTRACK_CTX] = {
1401                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CTX,
1402                 .flags = NLDEV_PER_DEV,
1403                 .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY,
1404                 .id = RDMA_NLDEV_ATTR_RES_CTXN,
1405         },
1406         [RDMA_RESTRACK_SRQ] = {
1407                 .nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ,
1408                 .flags = NLDEV_PER_DEV,
1409                 .entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY,
1410                 .id = RDMA_NLDEV_ATTR_RES_SRQN,
1411         },
1412
1413 };
1414
1415 static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1416                                struct netlink_ext_ack *extack,
1417                                enum rdma_restrack_type res_type,
1418                                res_fill_func_t fill_func)
1419 {
1420         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1421         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1422         struct rdma_restrack_entry *res;
1423         struct ib_device *device;
1424         u32 index, id, port = 0;
1425         bool has_cap_net_admin;
1426         struct sk_buff *msg;
1427         int ret;
1428
1429         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1430                                      nldev_policy, extack);
1431         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !fe->id || !tb[fe->id])
1432                 return -EINVAL;
1433
1434         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1435         device = ib_device_get_by_index(sock_net(skb->sk), index);
1436         if (!device)
1437                 return -EINVAL;
1438
1439         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1440                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1441                 if (!rdma_is_port_valid(device, port)) {
1442                         ret = -EINVAL;
1443                         goto err;
1444                 }
1445         }
1446
1447         if ((port && fe->flags & NLDEV_PER_DEV) ||
1448             (!port && ~fe->flags & NLDEV_PER_DEV)) {
1449                 ret = -EINVAL;
1450                 goto err;
1451         }
1452
1453         id = nla_get_u32(tb[fe->id]);
1454         res = rdma_restrack_get_byid(device, res_type, id);
1455         if (IS_ERR(res)) {
1456                 ret = PTR_ERR(res);
1457                 goto err;
1458         }
1459
1460         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1461         if (!msg) {
1462                 ret = -ENOMEM;
1463                 goto err_get;
1464         }
1465
1466         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1467                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1468                                          RDMA_NL_GET_OP(nlh->nlmsg_type)),
1469                         0, 0);
1470
1471         if (!nlh || fill_nldev_handle(msg, device)) {
1472                 ret = -EMSGSIZE;
1473                 goto err_free;
1474         }
1475
1476         has_cap_net_admin = netlink_capable(skb, CAP_NET_ADMIN);
1477
1478         ret = fill_func(msg, has_cap_net_admin, res, port);
1479         if (ret)
1480                 goto err_free;
1481
1482         rdma_restrack_put(res);
1483         nlmsg_end(msg, nlh);
1484         ib_device_put(device);
1485         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1486
1487 err_free:
1488         nlmsg_free(msg);
1489 err_get:
1490         rdma_restrack_put(res);
1491 err:
1492         ib_device_put(device);
1493         return ret;
1494 }
1495
1496 static int res_get_common_dumpit(struct sk_buff *skb,
1497                                  struct netlink_callback *cb,
1498                                  enum rdma_restrack_type res_type,
1499                                  res_fill_func_t fill_func)
1500 {
1501         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1502         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1503         struct rdma_restrack_entry *res;
1504         struct rdma_restrack_root *rt;
1505         int err, ret = 0, idx = 0;
1506         struct nlattr *table_attr;
1507         struct nlattr *entry_attr;
1508         struct ib_device *device;
1509         int start = cb->args[0];
1510         bool has_cap_net_admin;
1511         struct nlmsghdr *nlh;
1512         unsigned long id;
1513         u32 index, port = 0;
1514         bool filled = false;
1515
1516         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1517                                      nldev_policy, NULL);
1518         /*
1519          * Right now, we are expecting the device index to get res information,
1520          * but it is possible to extend this code to return all devices in
1521          * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX.
1522          * if it doesn't exist, we will iterate over all devices.
1523          *
1524          * But it is not needed for now.
1525          */
1526         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1527                 return -EINVAL;
1528
1529         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1530         device = ib_device_get_by_index(sock_net(skb->sk), index);
1531         if (!device)
1532                 return -EINVAL;
1533
1534         /*
1535          * If no PORT_INDEX is supplied, we will return all QPs from that device
1536          */
1537         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1538                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1539                 if (!rdma_is_port_valid(device, port)) {
1540                         ret = -EINVAL;
1541                         goto err_index;
1542                 }
1543         }
1544
1545         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1546                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1547                                          RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
1548                         0, NLM_F_MULTI);
1549
1550         if (!nlh || fill_nldev_handle(skb, device)) {
1551                 ret = -EMSGSIZE;
1552                 goto err;
1553         }
1554
1555         table_attr = nla_nest_start_noflag(skb, fe->nldev_attr);
1556         if (!table_attr) {
1557                 ret = -EMSGSIZE;
1558                 goto err;
1559         }
1560
1561         has_cap_net_admin = netlink_capable(cb->skb, CAP_NET_ADMIN);
1562
1563         rt = &device->res[res_type];
1564         xa_lock(&rt->xa);
1565         /*
1566          * FIXME: if the skip ahead is something common this loop should
1567          * use xas_for_each & xas_pause to optimize, we can have a lot of
1568          * objects.
1569          */
1570         xa_for_each(&rt->xa, id, res) {
1571                 if (idx < start || !rdma_restrack_get(res))
1572                         goto next;
1573
1574                 xa_unlock(&rt->xa);
1575
1576                 filled = true;
1577
1578                 entry_attr = nla_nest_start_noflag(skb, fe->entry);
1579                 if (!entry_attr) {
1580                         ret = -EMSGSIZE;
1581                         rdma_restrack_put(res);
1582                         goto msg_full;
1583                 }
1584
1585                 ret = fill_func(skb, has_cap_net_admin, res, port);
1586
1587                 rdma_restrack_put(res);
1588
1589                 if (ret) {
1590                         nla_nest_cancel(skb, entry_attr);
1591                         if (ret == -EMSGSIZE)
1592                                 goto msg_full;
1593                         if (ret == -EAGAIN)
1594                                 goto again;
1595                         goto res_err;
1596                 }
1597                 nla_nest_end(skb, entry_attr);
1598 again:          xa_lock(&rt->xa);
1599 next:           idx++;
1600         }
1601         xa_unlock(&rt->xa);
1602
1603 msg_full:
1604         nla_nest_end(skb, table_attr);
1605         nlmsg_end(skb, nlh);
1606         cb->args[0] = idx;
1607
1608         /*
1609          * No more entries to fill, cancel the message and
1610          * return 0 to mark end of dumpit.
1611          */
1612         if (!filled)
1613                 goto err;
1614
1615         ib_device_put(device);
1616         return skb->len;
1617
1618 res_err:
1619         nla_nest_cancel(skb, table_attr);
1620
1621 err:
1622         nlmsg_cancel(skb, nlh);
1623
1624 err_index:
1625         ib_device_put(device);
1626         return ret;
1627 }
1628
1629 #define RES_GET_FUNCS(name, type)                                              \
1630         static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,          \
1631                                                  struct netlink_callback *cb)  \
1632         {                                                                      \
1633                 return res_get_common_dumpit(skb, cb, type,                    \
1634                                              fill_res_##name##_entry);         \
1635         }                                                                      \
1636         static int nldev_res_get_##name##_doit(struct sk_buff *skb,            \
1637                                                struct nlmsghdr *nlh,           \
1638                                                struct netlink_ext_ack *extack) \
1639         {                                                                      \
1640                 return res_get_common_doit(skb, nlh, extack, type,             \
1641                                            fill_res_##name##_entry);           \
1642         }
1643
1644 RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);
1645 RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);
1646 RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);
1647 RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);
1648 RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);
1649 RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);
1650 RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);
1651 RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);
1652 RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);
1653 RES_GET_FUNCS(ctx, RDMA_RESTRACK_CTX);
1654 RES_GET_FUNCS(srq, RDMA_RESTRACK_SRQ);
1655
1656 static LIST_HEAD(link_ops);
1657 static DECLARE_RWSEM(link_ops_rwsem);
1658
1659 static const struct rdma_link_ops *link_ops_get(const char *type)
1660 {
1661         const struct rdma_link_ops *ops;
1662
1663         list_for_each_entry(ops, &link_ops, list) {
1664                 if (!strcmp(ops->type, type))
1665                         goto out;
1666         }
1667         ops = NULL;
1668 out:
1669         return ops;
1670 }
1671
1672 void rdma_link_register(struct rdma_link_ops *ops)
1673 {
1674         down_write(&link_ops_rwsem);
1675         if (WARN_ON_ONCE(link_ops_get(ops->type)))
1676                 goto out;
1677         list_add(&ops->list, &link_ops);
1678 out:
1679         up_write(&link_ops_rwsem);
1680 }
1681 EXPORT_SYMBOL(rdma_link_register);
1682
1683 void rdma_link_unregister(struct rdma_link_ops *ops)
1684 {
1685         down_write(&link_ops_rwsem);
1686         list_del(&ops->list);
1687         up_write(&link_ops_rwsem);
1688 }
1689 EXPORT_SYMBOL(rdma_link_unregister);
1690
1691 static int nldev_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
1692                           struct netlink_ext_ack *extack)
1693 {
1694         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1695         char ibdev_name[IB_DEVICE_NAME_MAX];
1696         const struct rdma_link_ops *ops;
1697         char ndev_name[IFNAMSIZ];
1698         struct net_device *ndev;
1699         char type[IFNAMSIZ];
1700         int err;
1701
1702         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1703                                      nldev_policy, extack);
1704         if (err || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
1705             !tb[RDMA_NLDEV_ATTR_LINK_TYPE] || !tb[RDMA_NLDEV_ATTR_NDEV_NAME])
1706                 return -EINVAL;
1707
1708         nla_strscpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1709                     sizeof(ibdev_name));
1710         if (strchr(ibdev_name, '%') || strlen(ibdev_name) == 0)
1711                 return -EINVAL;
1712
1713         nla_strscpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type));
1714         nla_strscpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME],
1715                     sizeof(ndev_name));
1716
1717         ndev = dev_get_by_name(sock_net(skb->sk), ndev_name);
1718         if (!ndev)
1719                 return -ENODEV;
1720
1721         down_read(&link_ops_rwsem);
1722         ops = link_ops_get(type);
1723 #ifdef CONFIG_MODULES
1724         if (!ops) {
1725                 up_read(&link_ops_rwsem);
1726                 request_module("rdma-link-%s", type);
1727                 down_read(&link_ops_rwsem);
1728                 ops = link_ops_get(type);
1729         }
1730 #endif
1731         err = ops ? ops->newlink(ibdev_name, ndev) : -EINVAL;
1732         up_read(&link_ops_rwsem);
1733         dev_put(ndev);
1734
1735         return err;
1736 }
1737
1738 static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
1739                           struct netlink_ext_ack *extack)
1740 {
1741         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1742         struct ib_device *device;
1743         u32 index;
1744         int err;
1745
1746         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1747                                      nldev_policy, extack);
1748         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1749                 return -EINVAL;
1750
1751         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1752         device = ib_device_get_by_index(sock_net(skb->sk), index);
1753         if (!device)
1754                 return -EINVAL;
1755
1756         if (!(device->attrs.kernel_cap_flags & IBK_ALLOW_USER_UNREG)) {
1757                 ib_device_put(device);
1758                 return -EINVAL;
1759         }
1760
1761         ib_unregister_device_and_put(device);
1762         return 0;
1763 }
1764
1765 static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh,
1766                              struct netlink_ext_ack *extack)
1767 {
1768         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1769         char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE];
1770         struct ib_client_nl_info data = {};
1771         struct ib_device *ibdev = NULL;
1772         struct sk_buff *msg;
1773         u32 index;
1774         int err;
1775
1776         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
1777                           extack);
1778         if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE])
1779                 return -EINVAL;
1780
1781         nla_strscpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE],
1782                     sizeof(client_name));
1783
1784         if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {
1785                 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1786                 ibdev = ib_device_get_by_index(sock_net(skb->sk), index);
1787                 if (!ibdev)
1788                         return -EINVAL;
1789
1790                 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1791                         data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1792                         if (!rdma_is_port_valid(ibdev, data.port)) {
1793                                 err = -EINVAL;
1794                                 goto out_put;
1795                         }
1796                 } else {
1797                         data.port = -1;
1798                 }
1799         } else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1800                 return -EINVAL;
1801         }
1802
1803         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1804         if (!msg) {
1805                 err = -ENOMEM;
1806                 goto out_put;
1807         }
1808         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1809                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1810                                          RDMA_NLDEV_CMD_GET_CHARDEV),
1811                         0, 0);
1812         if (!nlh) {
1813                 err = -EMSGSIZE;
1814                 goto out_nlmsg;
1815         }
1816
1817         data.nl_msg = msg;
1818         err = ib_get_client_nl_info(ibdev, client_name, &data);
1819         if (err)
1820                 goto out_nlmsg;
1821
1822         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV,
1823                                 huge_encode_dev(data.cdev->devt),
1824                                 RDMA_NLDEV_ATTR_PAD);
1825         if (err)
1826                 goto out_data;
1827         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi,
1828                                 RDMA_NLDEV_ATTR_PAD);
1829         if (err)
1830                 goto out_data;
1831         if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME,
1832                            dev_name(data.cdev))) {
1833                 err = -EMSGSIZE;
1834                 goto out_data;
1835         }
1836
1837         nlmsg_end(msg, nlh);
1838         put_device(data.cdev);
1839         if (ibdev)
1840                 ib_device_put(ibdev);
1841         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1842
1843 out_data:
1844         put_device(data.cdev);
1845 out_nlmsg:
1846         nlmsg_free(msg);
1847 out_put:
1848         if (ibdev)
1849                 ib_device_put(ibdev);
1850         return err;
1851 }
1852
1853 static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1854                               struct netlink_ext_ack *extack)
1855 {
1856         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1857         struct sk_buff *msg;
1858         int err;
1859
1860         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1861                           nldev_policy, extack);
1862         if (err)
1863                 return err;
1864
1865         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1866         if (!msg)
1867                 return -ENOMEM;
1868
1869         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1870                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1871                                          RDMA_NLDEV_CMD_SYS_GET),
1872                         0, 0);
1873         if (!nlh) {
1874                 nlmsg_free(msg);
1875                 return -EMSGSIZE;
1876         }
1877
1878         err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
1879                          (u8)ib_devices_shared_netns);
1880         if (err) {
1881                 nlmsg_free(msg);
1882                 return err;
1883         }
1884
1885         /*
1886          * Copy-on-fork is supported.
1887          * See commits:
1888          * 70e806e4e645 ("mm: Do early cow for pinned pages during fork() for ptes")
1889          * 4eae4efa2c29 ("hugetlb: do early cow when page pinned on src mm")
1890          * for more details. Don't backport this without them.
1891          *
1892          * Return value ignored on purpose, assume copy-on-fork is not
1893          * supported in case of failure.
1894          */
1895         nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK, 1);
1896
1897         nlmsg_end(msg, nlh);
1898         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1899 }
1900
1901 static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1902                                   struct netlink_ext_ack *extack)
1903 {
1904         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1905         u8 enable;
1906         int err;
1907
1908         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1909                           nldev_policy, extack);
1910         if (err || !tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])
1911                 return -EINVAL;
1912
1913         enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);
1914         /* Only 0 and 1 are supported */
1915         if (enable > 1)
1916                 return -EINVAL;
1917
1918         err = rdma_compatdev_set(enable);
1919         return err;
1920 }
1921
1922 static int nldev_stat_set_mode_doit(struct sk_buff *msg,
1923                                     struct netlink_ext_ack *extack,
1924                                     struct nlattr *tb[],
1925                                     struct ib_device *device, u32 port)
1926 {
1927         u32 mode, mask = 0, qpn, cntn = 0;
1928         int ret;
1929
1930         /* Currently only counter for QP is supported */
1931         if (!tb[RDMA_NLDEV_ATTR_STAT_RES] ||
1932             nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
1933                 return -EINVAL;
1934
1935         mode = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_MODE]);
1936         if (mode == RDMA_COUNTER_MODE_AUTO) {
1937                 if (tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK])
1938                         mask = nla_get_u32(
1939                                 tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]);
1940                 return rdma_counter_set_auto_mode(device, port, mask, extack);
1941         }
1942
1943         if (!tb[RDMA_NLDEV_ATTR_RES_LQPN])
1944                 return -EINVAL;
1945
1946         qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
1947         if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) {
1948                 cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
1949                 ret = rdma_counter_bind_qpn(device, port, qpn, cntn);
1950                 if (ret)
1951                         return ret;
1952         } else {
1953                 ret = rdma_counter_bind_qpn_alloc(device, port, qpn, &cntn);
1954                 if (ret)
1955                         return ret;
1956         }
1957
1958         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
1959             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
1960                 ret = -EMSGSIZE;
1961                 goto err_fill;
1962         }
1963
1964         return 0;
1965
1966 err_fill:
1967         rdma_counter_unbind_qpn(device, port, qpn, cntn);
1968         return ret;
1969 }
1970
1971 static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[],
1972                                                struct ib_device *device,
1973                                                u32 port)
1974 {
1975         struct rdma_hw_stats *stats;
1976         struct nlattr *entry_attr;
1977         unsigned long *target;
1978         int rem, i, ret = 0;
1979         u32 index;
1980
1981         stats = ib_get_hw_stats_port(device, port);
1982         if (!stats)
1983                 return -EINVAL;
1984
1985         target = kcalloc(BITS_TO_LONGS(stats->num_counters),
1986                          sizeof(*stats->is_disabled), GFP_KERNEL);
1987         if (!target)
1988                 return -ENOMEM;
1989
1990         nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS],
1991                             rem) {
1992                 index = nla_get_u32(entry_attr);
1993                 if ((index >= stats->num_counters) ||
1994                     !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) {
1995                         ret = -EINVAL;
1996                         goto out;
1997                 }
1998
1999                 set_bit(index, target);
2000         }
2001
2002         for (i = 0; i < stats->num_counters; i++) {
2003                 if (!(stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL))
2004                         continue;
2005
2006                 ret = rdma_counter_modify(device, port, i, test_bit(i, target));
2007                 if (ret)
2008                         goto out;
2009         }
2010
2011 out:
2012         kfree(target);
2013         return ret;
2014 }
2015
2016 static int nldev_stat_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2017                                struct netlink_ext_ack *extack)
2018 {
2019         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2020         struct ib_device *device;
2021         struct sk_buff *msg;
2022         u32 index, port;
2023         int ret;
2024
2025         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
2026                           extack);
2027         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
2028             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2029                 return -EINVAL;
2030
2031         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2032         device = ib_device_get_by_index(sock_net(skb->sk), index);
2033         if (!device)
2034                 return -EINVAL;
2035
2036         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2037         if (!rdma_is_port_valid(device, port)) {
2038                 ret = -EINVAL;
2039                 goto err_put_device;
2040         }
2041
2042         if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] &&
2043             !tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
2044                 ret = -EINVAL;
2045                 goto err_put_device;
2046         }
2047
2048         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2049         if (!msg) {
2050                 ret = -ENOMEM;
2051                 goto err_put_device;
2052         }
2053         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2054                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2055                                          RDMA_NLDEV_CMD_STAT_SET),
2056                         0, 0);
2057         if (!nlh || fill_nldev_handle(msg, device) ||
2058             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2059                 ret = -EMSGSIZE;
2060                 goto err_free_msg;
2061         }
2062
2063         if (tb[RDMA_NLDEV_ATTR_STAT_MODE]) {
2064                 ret = nldev_stat_set_mode_doit(msg, extack, tb, device, port);
2065                 if (ret)
2066                         goto err_free_msg;
2067         }
2068
2069         if (tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
2070                 ret = nldev_stat_set_counter_dynamic_doit(tb, device, port);
2071                 if (ret)
2072                         goto err_free_msg;
2073         }
2074
2075         nlmsg_end(msg, nlh);
2076         ib_device_put(device);
2077         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2078
2079 err_free_msg:
2080         nlmsg_free(msg);
2081 err_put_device:
2082         ib_device_put(device);
2083         return ret;
2084 }
2085
2086 static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2087                                struct netlink_ext_ack *extack)
2088 {
2089         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2090         struct ib_device *device;
2091         struct sk_buff *msg;
2092         u32 index, port, qpn, cntn;
2093         int ret;
2094
2095         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2096                           nldev_policy, extack);
2097         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] ||
2098             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX] ||
2099             !tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID] ||
2100             !tb[RDMA_NLDEV_ATTR_RES_LQPN])
2101                 return -EINVAL;
2102
2103         if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
2104                 return -EINVAL;
2105
2106         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2107         device = ib_device_get_by_index(sock_net(skb->sk), index);
2108         if (!device)
2109                 return -EINVAL;
2110
2111         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2112         if (!rdma_is_port_valid(device, port)) {
2113                 ret = -EINVAL;
2114                 goto err;
2115         }
2116
2117         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2118         if (!msg) {
2119                 ret = -ENOMEM;
2120                 goto err;
2121         }
2122         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2123                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2124                                          RDMA_NLDEV_CMD_STAT_SET),
2125                         0, 0);
2126         if (!nlh) {
2127                 ret = -EMSGSIZE;
2128                 goto err_fill;
2129         }
2130
2131         cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
2132         qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
2133         if (fill_nldev_handle(msg, device) ||
2134             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2135             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
2136             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
2137                 ret = -EMSGSIZE;
2138                 goto err_fill;
2139         }
2140
2141         ret = rdma_counter_unbind_qpn(device, port, qpn, cntn);
2142         if (ret)
2143                 goto err_fill;
2144
2145         nlmsg_end(msg, nlh);
2146         ib_device_put(device);
2147         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2148
2149 err_fill:
2150         nlmsg_free(msg);
2151 err:
2152         ib_device_put(device);
2153         return ret;
2154 }
2155
2156 static int stat_get_doit_default_counter(struct sk_buff *skb,
2157                                          struct nlmsghdr *nlh,
2158                                          struct netlink_ext_ack *extack,
2159                                          struct nlattr *tb[])
2160 {
2161         struct rdma_hw_stats *stats;
2162         struct nlattr *table_attr;
2163         struct ib_device *device;
2164         int ret, num_cnts, i;
2165         struct sk_buff *msg;
2166         u32 index, port;
2167         u64 v;
2168
2169         if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2170                 return -EINVAL;
2171
2172         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2173         device = ib_device_get_by_index(sock_net(skb->sk), index);
2174         if (!device)
2175                 return -EINVAL;
2176
2177         if (!device->ops.alloc_hw_port_stats || !device->ops.get_hw_stats) {
2178                 ret = -EINVAL;
2179                 goto err;
2180         }
2181
2182         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2183         stats = ib_get_hw_stats_port(device, port);
2184         if (!stats) {
2185                 ret = -EINVAL;
2186                 goto err;
2187         }
2188
2189         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2190         if (!msg) {
2191                 ret = -ENOMEM;
2192                 goto err;
2193         }
2194
2195         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2196                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2197                                          RDMA_NLDEV_CMD_STAT_GET),
2198                         0, 0);
2199
2200         if (!nlh || fill_nldev_handle(msg, device) ||
2201             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2202                 ret = -EMSGSIZE;
2203                 goto err_msg;
2204         }
2205
2206         mutex_lock(&stats->lock);
2207
2208         num_cnts = device->ops.get_hw_stats(device, stats, port, 0);
2209         if (num_cnts < 0) {
2210                 ret = -EINVAL;
2211                 goto err_stats;
2212         }
2213
2214         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
2215         if (!table_attr) {
2216                 ret = -EMSGSIZE;
2217                 goto err_stats;
2218         }
2219         for (i = 0; i < num_cnts; i++) {
2220                 if (test_bit(i, stats->is_disabled))
2221                         continue;
2222
2223                 v = stats->value[i] +
2224                         rdma_counter_get_hwstat_value(device, port, i);
2225                 if (rdma_nl_stat_hwcounter_entry(msg,
2226                                                  stats->descs[i].name, v)) {
2227                         ret = -EMSGSIZE;
2228                         goto err_table;
2229                 }
2230         }
2231         nla_nest_end(msg, table_attr);
2232
2233         mutex_unlock(&stats->lock);
2234         nlmsg_end(msg, nlh);
2235         ib_device_put(device);
2236         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2237
2238 err_table:
2239         nla_nest_cancel(msg, table_attr);
2240 err_stats:
2241         mutex_unlock(&stats->lock);
2242 err_msg:
2243         nlmsg_free(msg);
2244 err:
2245         ib_device_put(device);
2246         return ret;
2247 }
2248
2249 static int stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,
2250                             struct netlink_ext_ack *extack, struct nlattr *tb[])
2251
2252 {
2253         static enum rdma_nl_counter_mode mode;
2254         static enum rdma_nl_counter_mask mask;
2255         struct ib_device *device;
2256         struct sk_buff *msg;
2257         u32 index, port;
2258         int ret;
2259
2260         if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID])
2261                 return nldev_res_get_counter_doit(skb, nlh, extack);
2262
2263         if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] ||
2264             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2265                 return -EINVAL;
2266
2267         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2268         device = ib_device_get_by_index(sock_net(skb->sk), index);
2269         if (!device)
2270                 return -EINVAL;
2271
2272         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2273         if (!rdma_is_port_valid(device, port)) {
2274                 ret = -EINVAL;
2275                 goto err;
2276         }
2277
2278         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2279         if (!msg) {
2280                 ret = -ENOMEM;
2281                 goto err;
2282         }
2283
2284         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2285                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2286                                          RDMA_NLDEV_CMD_STAT_GET),
2287                         0, 0);
2288         if (!nlh) {
2289                 ret = -EMSGSIZE;
2290                 goto err_msg;
2291         }
2292
2293         ret = rdma_counter_get_mode(device, port, &mode, &mask);
2294         if (ret)
2295                 goto err_msg;
2296
2297         if (fill_nldev_handle(msg, device) ||
2298             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2299             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, mode)) {
2300                 ret = -EMSGSIZE;
2301                 goto err_msg;
2302         }
2303
2304         if ((mode == RDMA_COUNTER_MODE_AUTO) &&
2305             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, mask)) {
2306                 ret = -EMSGSIZE;
2307                 goto err_msg;
2308         }
2309
2310         nlmsg_end(msg, nlh);
2311         ib_device_put(device);
2312         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2313
2314 err_msg:
2315         nlmsg_free(msg);
2316 err:
2317         ib_device_put(device);
2318         return ret;
2319 }
2320
2321 static int nldev_stat_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2322                                struct netlink_ext_ack *extack)
2323 {
2324         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2325         int ret;
2326
2327         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2328                           nldev_policy, extack);
2329         if (ret)
2330                 return -EINVAL;
2331
2332         if (!tb[RDMA_NLDEV_ATTR_STAT_RES])
2333                 return stat_get_doit_default_counter(skb, nlh, extack, tb);
2334
2335         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2336         case RDMA_NLDEV_ATTR_RES_QP:
2337                 ret = stat_get_doit_qp(skb, nlh, extack, tb);
2338                 break;
2339         case RDMA_NLDEV_ATTR_RES_MR:
2340                 ret = res_get_common_doit(skb, nlh, extack, RDMA_RESTRACK_MR,
2341                                           fill_stat_mr_entry);
2342                 break;
2343         default:
2344                 ret = -EINVAL;
2345                 break;
2346         }
2347
2348         return ret;
2349 }
2350
2351 static int nldev_stat_get_dumpit(struct sk_buff *skb,
2352                                  struct netlink_callback *cb)
2353 {
2354         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2355         int ret;
2356
2357         ret = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2358                           nldev_policy, NULL);
2359         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES])
2360                 return -EINVAL;
2361
2362         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2363         case RDMA_NLDEV_ATTR_RES_QP:
2364                 ret = nldev_res_get_counter_dumpit(skb, cb);
2365                 break;
2366         case RDMA_NLDEV_ATTR_RES_MR:
2367                 ret = res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR,
2368                                             fill_stat_mr_entry);
2369                 break;
2370         default:
2371                 ret = -EINVAL;
2372                 break;
2373         }
2374
2375         return ret;
2376 }
2377
2378 static int nldev_stat_get_counter_status_doit(struct sk_buff *skb,
2379                                               struct nlmsghdr *nlh,
2380                                               struct netlink_ext_ack *extack)
2381 {
2382         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX], *table, *entry;
2383         struct rdma_hw_stats *stats;
2384         struct ib_device *device;
2385         struct sk_buff *msg;
2386         u32 devid, port;
2387         int ret, i;
2388
2389         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2390                           nldev_policy, extack);
2391         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
2392             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2393                 return -EINVAL;
2394
2395         devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2396         device = ib_device_get_by_index(sock_net(skb->sk), devid);
2397         if (!device)
2398                 return -EINVAL;
2399
2400         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2401         if (!rdma_is_port_valid(device, port)) {
2402                 ret = -EINVAL;
2403                 goto err;
2404         }
2405
2406         stats = ib_get_hw_stats_port(device, port);
2407         if (!stats) {
2408                 ret = -EINVAL;
2409                 goto err;
2410         }
2411
2412         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2413         if (!msg) {
2414                 ret = -ENOMEM;
2415                 goto err;
2416         }
2417
2418         nlh = nlmsg_put(
2419                 msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2420                 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET_STATUS),
2421                 0, 0);
2422
2423         ret = -EMSGSIZE;
2424         if (!nlh || fill_nldev_handle(msg, device) ||
2425             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
2426                 goto err_msg;
2427
2428         table = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
2429         if (!table)
2430                 goto err_msg;
2431
2432         mutex_lock(&stats->lock);
2433         for (i = 0; i < stats->num_counters; i++) {
2434                 entry = nla_nest_start(msg,
2435                                        RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);
2436                 if (!entry)
2437                         goto err_msg_table;
2438
2439                 if (nla_put_string(msg,
2440                                    RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,
2441                                    stats->descs[i].name) ||
2442                     nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX, i))
2443                         goto err_msg_entry;
2444
2445                 if ((stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL) &&
2446                     (nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC,
2447                                 !test_bit(i, stats->is_disabled))))
2448                         goto err_msg_entry;
2449
2450                 nla_nest_end(msg, entry);
2451         }
2452         mutex_unlock(&stats->lock);
2453
2454         nla_nest_end(msg, table);
2455         nlmsg_end(msg, nlh);
2456         ib_device_put(device);
2457         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2458
2459 err_msg_entry:
2460         nla_nest_cancel(msg, entry);
2461 err_msg_table:
2462         mutex_unlock(&stats->lock);
2463         nla_nest_cancel(msg, table);
2464 err_msg:
2465         nlmsg_free(msg);
2466 err:
2467         ib_device_put(device);
2468         return ret;
2469 }
2470
2471 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
2472         [RDMA_NLDEV_CMD_GET] = {
2473                 .doit = nldev_get_doit,
2474                 .dump = nldev_get_dumpit,
2475         },
2476         [RDMA_NLDEV_CMD_GET_CHARDEV] = {
2477                 .doit = nldev_get_chardev,
2478         },
2479         [RDMA_NLDEV_CMD_SET] = {
2480                 .doit = nldev_set_doit,
2481                 .flags = RDMA_NL_ADMIN_PERM,
2482         },
2483         [RDMA_NLDEV_CMD_NEWLINK] = {
2484                 .doit = nldev_newlink,
2485                 .flags = RDMA_NL_ADMIN_PERM,
2486         },
2487         [RDMA_NLDEV_CMD_DELLINK] = {
2488                 .doit = nldev_dellink,
2489                 .flags = RDMA_NL_ADMIN_PERM,
2490         },
2491         [RDMA_NLDEV_CMD_PORT_GET] = {
2492                 .doit = nldev_port_get_doit,
2493                 .dump = nldev_port_get_dumpit,
2494         },
2495         [RDMA_NLDEV_CMD_RES_GET] = {
2496                 .doit = nldev_res_get_doit,
2497                 .dump = nldev_res_get_dumpit,
2498         },
2499         [RDMA_NLDEV_CMD_RES_QP_GET] = {
2500                 .doit = nldev_res_get_qp_doit,
2501                 .dump = nldev_res_get_qp_dumpit,
2502         },
2503         [RDMA_NLDEV_CMD_RES_CM_ID_GET] = {
2504                 .doit = nldev_res_get_cm_id_doit,
2505                 .dump = nldev_res_get_cm_id_dumpit,
2506         },
2507         [RDMA_NLDEV_CMD_RES_CQ_GET] = {
2508                 .doit = nldev_res_get_cq_doit,
2509                 .dump = nldev_res_get_cq_dumpit,
2510         },
2511         [RDMA_NLDEV_CMD_RES_MR_GET] = {
2512                 .doit = nldev_res_get_mr_doit,
2513                 .dump = nldev_res_get_mr_dumpit,
2514         },
2515         [RDMA_NLDEV_CMD_RES_PD_GET] = {
2516                 .doit = nldev_res_get_pd_doit,
2517                 .dump = nldev_res_get_pd_dumpit,
2518         },
2519         [RDMA_NLDEV_CMD_RES_CTX_GET] = {
2520                 .doit = nldev_res_get_ctx_doit,
2521                 .dump = nldev_res_get_ctx_dumpit,
2522         },
2523         [RDMA_NLDEV_CMD_RES_SRQ_GET] = {
2524                 .doit = nldev_res_get_srq_doit,
2525                 .dump = nldev_res_get_srq_dumpit,
2526         },
2527         [RDMA_NLDEV_CMD_SYS_GET] = {
2528                 .doit = nldev_sys_get_doit,
2529         },
2530         [RDMA_NLDEV_CMD_SYS_SET] = {
2531                 .doit = nldev_set_sys_set_doit,
2532         },
2533         [RDMA_NLDEV_CMD_STAT_SET] = {
2534                 .doit = nldev_stat_set_doit,
2535                 .flags = RDMA_NL_ADMIN_PERM,
2536         },
2537         [RDMA_NLDEV_CMD_STAT_GET] = {
2538                 .doit = nldev_stat_get_doit,
2539                 .dump = nldev_stat_get_dumpit,
2540         },
2541         [RDMA_NLDEV_CMD_STAT_DEL] = {
2542                 .doit = nldev_stat_del_doit,
2543                 .flags = RDMA_NL_ADMIN_PERM,
2544         },
2545         [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {
2546                 .doit = nldev_res_get_qp_raw_doit,
2547                 .dump = nldev_res_get_qp_raw_dumpit,
2548                 .flags = RDMA_NL_ADMIN_PERM,
2549         },
2550         [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {
2551                 .doit = nldev_res_get_cq_raw_doit,
2552                 .dump = nldev_res_get_cq_raw_dumpit,
2553                 .flags = RDMA_NL_ADMIN_PERM,
2554         },
2555         [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {
2556                 .doit = nldev_res_get_mr_raw_doit,
2557                 .dump = nldev_res_get_mr_raw_dumpit,
2558                 .flags = RDMA_NL_ADMIN_PERM,
2559         },
2560         [RDMA_NLDEV_CMD_STAT_GET_STATUS] = {
2561                 .doit = nldev_stat_get_counter_status_doit,
2562         },
2563 };
2564
2565 void __init nldev_init(void)
2566 {
2567         rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table);
2568 }
2569
2570 void nldev_exit(void)
2571 {
2572         rdma_nl_unregister(RDMA_NL_NLDEV);
2573 }
2574
2575 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_NLDEV, 5);