RDMA/nldev: provide detailed MR information

Implement the RDMA nldev netlink interface for dumping detailed
MR information.

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 83e4392..4c6626e 100644 (file)
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -85,6 +85,12 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
        [RDMA_NLDEV_ATTR_RES_CQE]               = { .type = NLA_U32 },
        [RDMA_NLDEV_ATTR_RES_USECNT]            = { .type = NLA_U64 },
        [RDMA_NLDEV_ATTR_RES_POLL_CTX]          = { .type = NLA_U8 },
+       [RDMA_NLDEV_ATTR_RES_MR]                = { .type = NLA_NESTED },
+       [RDMA_NLDEV_ATTR_RES_MR_ENTRY]          = { .type = NLA_NESTED },
+       [RDMA_NLDEV_ATTR_RES_RKEY]              = { .type = NLA_U32 },
+       [RDMA_NLDEV_ATTR_RES_LKEY]              = { .type = NLA_U32 },
+       [RDMA_NLDEV_ATTR_RES_IOVA]              = { .type = NLA_U64 },
+       [RDMA_NLDEV_ATTR_RES_MRLEN]             = { .type = NLA_U64 },
 };
 
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
@@ -197,6 +203,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
                [RDMA_RESTRACK_CQ] = "cq",
                [RDMA_RESTRACK_QP] = "qp",
                [RDMA_RESTRACK_CM_ID] = "cm_id",
+               [RDMA_RESTRACK_MR] = "mr",
        };
 
        struct rdma_restrack_root *res = &device->res;
@@ -397,6 +404,41 @@ out:
        return -EMSGSIZE;
 }
 
+static int fill_res_mr_entry(struct sk_buff *msg, struct netlink_callback *cb,
+                            struct rdma_restrack_entry *res, uint32_t port)
+{
+       struct ib_mr *mr = container_of(res, struct ib_mr, res);
+       struct nlattr *entry_attr;
+
+       entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_MR_ENTRY);
+       if (!entry_attr)
+               goto out;
+
+       if (netlink_capable(cb->skb, CAP_NET_ADMIN)) {
+               if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
+                       goto err;
+               if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
+                       goto err;
+               if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_IOVA,
+                                     mr->iova, 0))
+                       goto err;
+       }
+
+       if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length, 0))
+               goto err;
+
+       if (fill_res_name_pid(msg, res))
+               goto err;
+
+       nla_nest_end(msg, entry_attr);
+       return 0;
+
+err:
+       nla_nest_cancel(msg, entry_attr);
+out:
+       return -EMSGSIZE;
+}
+
 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
 {
@@ -694,6 +736,11 @@ static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
                .nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET,
                .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
        },
+       [RDMA_RESTRACK_MR] = {
+               .fill_res_func = fill_res_mr_entry,
+               .nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET,
+               .nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
+       },
 };
 
 static int res_get_common_dumpit(struct sk_buff *skb,
@@ -848,6 +895,12 @@ static int nldev_res_get_cq_dumpit(struct sk_buff *skb,
        return res_get_common_dumpit(skb, cb, RDMA_RESTRACK_CQ);
 }
 
+static int nldev_res_get_mr_dumpit(struct sk_buff *skb,
+                                  struct netlink_callback *cb)
+{
+       return res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR);
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
        [RDMA_NLDEV_CMD_GET] = {
                .doit = nldev_get_doit,
@@ -880,6 +933,9 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
        [RDMA_NLDEV_CMD_RES_CQ_GET] = {
                .dump = nldev_res_get_cq_dumpit,
        },
+       [RDMA_NLDEV_CMD_RES_MR_GET] = {
+               .dump = nldev_res_get_mr_dumpit,
+       },
 };
 
 void __init nldev_init(void)

diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index 6da949e..e1d9934 100644 (file)
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -43,22 +43,28 @@ EXPORT_SYMBOL(rdma_restrack_count);
 
 static void set_kern_name(struct rdma_restrack_entry *res)
 {
-       enum rdma_restrack_type type = res->type;
-       struct ib_qp *qp;
+       struct ib_pd *pd;
 
-       if (type != RDMA_RESTRACK_QP)
-               /* Other types already have this name embedded in */
-               return;
-
-       qp = container_of(res, struct ib_qp, res);
-       if (!qp->pd) {
-               WARN_ONCE(true, "XRC QPs are not supported\n");
-               /* Survive, despite the programmer's error */
-               res->kern_name = " ";
-               return;
+       switch (res->type) {
+       case RDMA_RESTRACK_QP:
+               pd = container_of(res, struct ib_qp, res)->pd;
+               if (!pd) {
+                       WARN_ONCE(true, "XRC QPs are not supported\n");
+                       /* Survive, despite the programmer's error */
+                       res->kern_name = " ";
+               }
+               break;
+       case RDMA_RESTRACK_MR:
+               pd = container_of(res, struct ib_mr, res)->pd;
+               break;
+       default:
+               /* Other types set kern_name directly */
+               pd = NULL;
+               break;
        }
 
-       res->kern_name = qp->pd->res.kern_name;
+       if (pd)
+               res->kern_name = pd->res.kern_name;
 }
 
 static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
@@ -73,6 +79,8 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
        case RDMA_RESTRACK_CM_ID:
                return container_of(res, struct rdma_id_private,
                                    res)->id.device;
+       case RDMA_RESTRACK_MR:
+               return container_of(res, struct ib_mr, res)->device;
        default:
                WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
                return NULL;
@@ -90,6 +98,8 @@ static bool res_is_user(struct rdma_restrack_entry *res)
                return container_of(res, struct ib_qp, res)->uobject;
        case RDMA_RESTRACK_CM_ID:
                return !res->kern_name;
+       case RDMA_RESTRACK_MR:
+               return container_of(res, struct ib_mr, res)->pd->uobject;
        default:
                WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
                return false;

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index a148de3..9f9fc14 100644 (file)
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -693,6 +693,8 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
        mr->pd      = pd;
        mr->uobject = uobj;
        atomic_inc(&pd->usecnt);
+       mr->res.type = RDMA_RESTRACK_MR;
+       rdma_restrack_add(&mr->res);
 
        uobj->object = mr;
 

diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 4e2b231..873b7aa 100644 (file)
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -1622,6 +1622,7 @@ int ib_dereg_mr(struct ib_mr *mr)
        struct ib_pd *pd = mr->pd;
        int ret;
 
+       rdma_restrack_del(&mr->res);
        ret = mr->device->dereg_mr(mr);
        if (!ret)
                atomic_dec(&pd->usecnt);
@@ -1658,6 +1659,8 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
                mr->uobject = NULL;
                atomic_inc(&pd->usecnt);
                mr->need_inval = false;
+               mr->res.type = RDMA_RESTRACK_MR;
+               rdma_restrack_add(&mr->res);
        }
 
        return mr;

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 73b2387..7df3274 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1772,6 +1772,11 @@ struct ib_mr {
                struct ib_uobject       *uobject;       /* user */
                struct list_head        qp_entry;       /* FR */
        };
+
+       /*
+        * Implementation details of the RDMA core, don't use in drivers:
+        */
+       struct rdma_restrack_entry res;
 };
 
 struct ib_mw {

diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h
index af88667..a56f4f2 100644 (file)
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -34,6 +34,10 @@ enum rdma_restrack_type {
         */
        RDMA_RESTRACK_CM_ID,
        /**
+        * @RDMA_RESTRACK_MR: Memory Region (MR)
+        */
+       RDMA_RESTRACK_MR,
+       /**
         * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations
         */
        RDMA_RESTRACK_MAX

diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 36cf1f0..6d9ec38 100644 (file)
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -242,6 +242,8 @@ enum rdma_nldev_command {
 
        RDMA_NLDEV_CMD_RES_CQ_GET, /* can dump */
 
+       RDMA_NLDEV_CMD_RES_MR_GET, /* can dump */
+
        RDMA_NLDEV_NUM_OPS
 };
 
@@ -372,6 +374,13 @@ enum rdma_nldev_attr {
        RDMA_NLDEV_ATTR_RES_USECNT,             /* u64 */
        RDMA_NLDEV_ATTR_RES_POLL_CTX,           /* u8 */
 
+       RDMA_NLDEV_ATTR_RES_MR,                 /* nested table */
+       RDMA_NLDEV_ATTR_RES_MR_ENTRY,           /* nested table */
+       RDMA_NLDEV_ATTR_RES_RKEY,               /* u32 */
+       RDMA_NLDEV_ATTR_RES_LKEY,               /* u32 */
+       RDMA_NLDEV_ATTR_RES_IOVA,               /* u64 */
+       RDMA_NLDEV_ATTR_RES_MRLEN,              /* u64 */
+
        RDMA_NLDEV_ATTR_MAX
 };
 #endif /* _UAPI_RDMA_NETLINK_H */