RDMA/restrack: Store all special QPs in restrack DB

Special QPs (SMI and GSI) have different rules in regards of their QP
numbers. While all other QP numbers are unique per-device, the QP0 and QP1
are created per-port as requested by IBTA.

In multiple port devices, the number of SMI and GSI QPs with be equal to
the number ports.

$ rdma dev
0: ibp0s9: node_type ca fw 4.4.9999 node_guid 5254:00c0:fe12:3455 sys_image_guid 5254:00c0:fe12:3455
$ rdma link
0/1: ibp0s9/1: subnet_prefix fe80:0000:0000:0000 lid 13397 sm_lid 49151 lmc 0 state ACTIVE physical_state LINK_UP
0/2: ibp0s9/2: subnet_prefix fe80:0000:0000:0000 lid 13397 sm_lid 49151 lmc 0 state UNKNOWN physical_state UNKNOWN

Before:
$ rdma res show qp type SMI,GSI
link ibp0s9/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core]
link ibp0s9/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core]

After:
$ rdma res show qp type SMI,GSI
link ibp0s9/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core]
link ibp0s9/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core]
link ibp0s9/2 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core]
link ibp0s9/2 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core]

Link: https://lore.kernel.org/r/20201104144008.3808124-4-leon@kernel.org
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index e84b0fe..7c4752c 100644 (file)
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -347,6 +347,8 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
        qp->srq = attr->srq;
        qp->rwq_ind_tbl = attr->rwq_ind_tbl;
        qp->event_handler = attr->event_handler;
+       qp->qp_type = attr->qp_type;
+       qp->port = attr->port_num;
 
        atomic_set(&qp->usecnt, 0);
        spin_lock_init(&qp->mr_lock);

diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index 4aeeaae..a123b22 100644 (file)
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -232,8 +232,15 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
                /* Special case to ensure that LQPN points to right QP */
                struct ib_qp *qp = container_of(res, struct ib_qp, res);
 
-               ret = xa_insert(&rt->xa, qp->qp_num, res, GFP_KERNEL);
-               res->id = ret ? 0 : qp->qp_num;
+               WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8,
+                         "QP number 0x%0X and port 0x%0X", qp->qp_num,
+                         qp->port);
+               res->id = qp->qp_num;
+               if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI)
+                       res->id |= qp->port << 24;
+               ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
+               if (ret)
+                       res->id = 0;
        } else if (res->type == RDMA_RESTRACK_COUNTER) {
                /* Special case to ensure that cntn points to right counter */
                struct rdma_counter *counter;