RDMA/nldev: Provide global resource utilization

Expose through the netlink interface the global per-device utilization of
the supported object types.

Provide both dumpit and doit callbacks.

As an example of possible output from rdmatool for system with 5
mlx5 cards:

$ rdma res
1: mlx5_0: qp 4 cq 5 pd 3
2: mlx5_1: qp 4 cq 5 pd 3
3: mlx5_2: qp 4 cq 5 pd 3
4: mlx5_3: qp 2 cq 3 pd 2
5: mlx5_4: qp 4 cq 5 pd 3

Reviewed-by: Mark Bloch <markb@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Reviewed-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 5d790c5..c37bb04 100644 (file)
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -31,6 +31,8 @@
  */
 
 #include <linux/module.h>
+#include <linux/pid.h>
+#include <linux/pid_namespace.h>
 #include <net/netlink.h>
 #include <rdma/rdma_netlink.h>
 
@@ -52,6 +54,11 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
        [RDMA_NLDEV_ATTR_PORT_STATE]    = { .type = NLA_U8 },
        [RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
        [RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
+       [RDMA_NLDEV_ATTR_RES_SUMMARY]   = { .type = NLA_NESTED },
+       [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]     = { .type = NLA_NESTED },
+       [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,
+                                            .len = 16 },
+       [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },
 };
 
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
@@ -134,6 +141,65 @@ static int fill_port_info(struct sk_buff *msg,
        return 0;
 }
 
+static int fill_res_info_entry(struct sk_buff *msg,
+                              const char *name, u64 curr)
+{
+       struct nlattr *entry_attr;
+
+       entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
+       if (!entry_attr)
+               return -EMSGSIZE;
+
+       if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
+               goto err;
+       if (nla_put_u64_64bit(msg,
+                             RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 0))
+               goto err;
+
+       nla_nest_end(msg, entry_attr);
+       return 0;
+
+err:
+       nla_nest_cancel(msg, entry_attr);
+       return -EMSGSIZE;
+}
+
+static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
+{
+       static const char * const names[RDMA_RESTRACK_MAX] = {
+               [RDMA_RESTRACK_PD] = "pd",
+               [RDMA_RESTRACK_CQ] = "cq",
+               [RDMA_RESTRACK_QP] = "qp",
+       };
+
+       struct rdma_restrack_root *res = &device->res;
+       struct nlattr *table_attr;
+       int ret, i, curr;
+
+       if (fill_nldev_handle(msg, device))
+               return -EMSGSIZE;
+
+       table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
+       if (!table_attr)
+               return -EMSGSIZE;
+
+       for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
+               if (!names[i])
+                       continue;
+               curr = rdma_restrack_count(res, i, task_active_pid_ns(current));
+               ret = fill_res_info_entry(msg, names[i], curr);
+               if (ret)
+                       goto err;
+       }
+
+       nla_nest_end(msg, table_attr);
+       return 0;
+
+err:
+       nla_nest_cancel(msg, table_attr);
+       return ret;
+}
+
 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
 {
@@ -329,6 +395,83 @@ out:
        return skb->len;
 }
 
+static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+                             struct netlink_ext_ack *extack)
+{
+       struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+       struct ib_device *device;
+       struct sk_buff *msg;
+       u32 index;
+       int ret;
+
+       ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+                         nldev_policy, extack);
+       if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+               return -EINVAL;
+
+       index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+       device = ib_device_get_by_index(index);
+       if (!device)
+               return -EINVAL;
+
+       msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+       if (!msg)
+               goto err;
+
+       nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+                       RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+                       0, 0);
+
+       ret = fill_res_info(msg, device);
+       if (ret)
+               goto err_free;
+
+       nlmsg_end(msg, nlh);
+       put_device(&device->dev);
+       return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+
+err_free:
+       nlmsg_free(msg);
+err:
+       put_device(&device->dev);
+       return ret;
+}
+
+static int _nldev_res_get_dumpit(struct ib_device *device,
+                                struct sk_buff *skb,
+                                struct netlink_callback *cb,
+                                unsigned int idx)
+{
+       int start = cb->args[0];
+       struct nlmsghdr *nlh;
+
+       if (idx < start)
+               return 0;
+
+       nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+                       RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+                       0, NLM_F_MULTI);
+
+       if (fill_res_info(skb, device)) {
+               nlmsg_cancel(skb, nlh);
+               goto out;
+       }
+
+       nlmsg_end(skb, nlh);
+
+       idx++;
+
+out:
+       cb->args[0] = idx;
+       return skb->len;
+}
+
+static int nldev_res_get_dumpit(struct sk_buff *skb,
+                               struct netlink_callback *cb)
+{
+       return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
        [RDMA_NLDEV_CMD_GET] = {
                .doit = nldev_get_doit,
@@ -338,6 +481,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
                .doit = nldev_port_get_doit,
                .dump = nldev_port_get_dumpit,
        },
+       [RDMA_NLDEV_CMD_RES_GET] = {
+               .doit = nldev_res_get_doit,
+               .dump = nldev_res_get_dumpit,
+       },
 };
 
 void __init nldev_init(void)

diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index cc002e3..22c3953 100644 (file)
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -236,6 +236,8 @@ enum rdma_nldev_command {
        RDMA_NLDEV_CMD_PORT_NEW,
        RDMA_NLDEV_CMD_PORT_DEL,
 
+       RDMA_NLDEV_CMD_RES_GET, /* can dump */
+
        RDMA_NLDEV_NUM_OPS
 };
 
@@ -303,6 +305,11 @@ enum rdma_nldev_attr {
 
        RDMA_NLDEV_ATTR_DEV_NODE_TYPE,          /* u8 */
 
+       RDMA_NLDEV_ATTR_RES_SUMMARY,            /* nested table */
+       RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY,      /* nested table */
+       RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, /* string */
+       RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, /* u64 */
+
        RDMA_NLDEV_ATTR_MAX
 };
 #endif /* _UAPI_RDMA_NETLINK_H */