IB/rdmavt, hfi1, qib: Enhance rdmavt and hfi1 to use 32 bit lids

Increase lid used in hfi1 driver to 32 bits. qib continues
to use 16 bit lids.

Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Don Hiatt <don.hiatt@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index ee1324c..cbda373 100644 (file)
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -10067,10 +10067,16 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
        struct hfi1_devdata *dd = ppd->dd;
        u32 mask = ~((1U << ppd->lmc) - 1);
        u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
+       u32 lid;
 
+       /*
+        * Program 0 in CSR if port lid is extended. This prevents
+        * 9B packets being sent out for large lids.
+        */
+       lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
        c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
                | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
-       c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
+       c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
                        << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
              ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
                        << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
@@ -10081,7 +10087,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
         */
        sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
                        SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
-              (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
+              (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
                        SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
 
        for (i = 0; i < dd->chip_send_contexts; i++) {
@@ -10091,7 +10097,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
        }
 
        /* Now we have to do the same thing for the sdma engines */
-       sdma_update_lmc(dd, mask, ppd->lid);
+       sdma_update_lmc(dd, mask, lid);
 }
 
 static const char *state_completed_string(u32 completed)

diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index b07f42c..52cae11 100644 (file)
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -718,7 +718,7 @@ struct hfi1_pportdata {
        u32 ibmaxlen;
        u32 current_egress_rate; /* units [10^6 bits/sec] */
        /* LID programmed for this instance */
-       u16 lid;
+       u32 lid;
        /* list of pkeys programmed; 0 if not set */
        u16 pkeys[MAX_PKEY_VALUES];
        u16 link_width_supported;

diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 1509bc6..cdcb4d0 100644 (file)
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -234,6 +234,61 @@ static void subn_handle_opa_trap_repress(struct hfi1_ibport *ibp,
        spin_unlock_irqrestore(&ibp->rvp.lock, flags);
 }
 
+static void hfi1_update_sm_ah_attr(struct hfi1_ibport *ibp,
+                                  struct rdma_ah_attr *attr, u32 dlid)
+{
+       rdma_ah_set_dlid(attr, dlid);
+       rdma_ah_set_port_num(attr, ppd_from_ibp(ibp)->port);
+       if (dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+               struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
+
+               rdma_ah_set_ah_flags(attr, IB_AH_GRH);
+               grh->sgid_index = 0;
+               grh->hop_limit = 1;
+               grh->dgid.global.subnet_prefix =
+                       ibp->rvp.gid_prefix;
+               grh->dgid.global.interface_id = OPA_MAKE_ID(dlid);
+       }
+}
+
+static int hfi1_modify_qp0_ah(struct hfi1_ibport *ibp,
+                             struct rvt_ah *ah, u32 dlid)
+{
+       struct rdma_ah_attr attr;
+       struct rvt_qp *qp0;
+       int ret = -EINVAL;
+
+       memset(&attr, 0, sizeof(attr));
+       attr.type = ah->ibah.type;
+       hfi1_update_sm_ah_attr(ibp, &attr, dlid);
+       rcu_read_lock();
+       qp0 = rcu_dereference(ibp->rvp.qp[0]);
+       if (qp0)
+               ret = rdma_modify_ah(&ah->ibah, &attr);
+       rcu_read_unlock();
+       return ret;
+}
+
+static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid)
+{
+       struct rdma_ah_attr attr;
+       struct ib_ah *ah = ERR_PTR(-EINVAL);
+       struct rvt_qp *qp0;
+       struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+       struct hfi1_devdata *dd = dd_from_ppd(ppd);
+       u8 port_num = ppd->port;
+
+       memset(&attr, 0, sizeof(attr));
+       attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
+       hfi1_update_sm_ah_attr(ibp, &attr, dlid);
+       rcu_read_lock();
+       qp0 = rcu_dereference(ibp->rvp.qp[0]);
+       if (qp0)
+               ah = rdma_create_ah(qp0->ibqp.pd, &attr);
+       rcu_read_unlock();
+       return ah;
+}
+
 static void send_trap(struct hfi1_ibport *ibp, struct trap_node *trap)
 {
        struct ib_mad_send_buf *send_buf;
@@ -1283,8 +1338,8 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
        struct hfi1_ibport *ibp;
        u8 clientrereg;
        unsigned long flags;
-       u32 smlid, opa_lid; /* tmp vars to hold LID values */
-       u16 lid;
+       u32 smlid;
+       u32 lid;
        u8 ls_old, ls_new, ps_new;
        u8 vls;
        u8 msl;
@@ -1301,22 +1356,20 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
                return reply((struct ib_mad_hdr *)smp);
        }
 
-       opa_lid = be32_to_cpu(pi->lid);
-       if (opa_lid & 0xFFFF0000) {
-               pr_warn("OPA_PortInfo lid out of range: %X\n", opa_lid);
+       lid = be32_to_cpu(pi->lid);
+       if (lid & 0xFF000000) {
+               pr_warn("OPA_PortInfo lid out of range: %X\n", lid);
                smp->status |= IB_SMP_INVALID_FIELD;
                goto get_only;
        }
 
-       lid = (u16)(opa_lid & 0x0000FFFF);
 
        smlid = be32_to_cpu(pi->sm_lid);
-       if (smlid & 0xFFFF0000) {
+       if (smlid & 0xFF000000) {
                pr_warn("OPA_PortInfo SM lid out of range: %X\n", smlid);
                smp->status |= IB_SMP_INVALID_FIELD;
                goto get_only;
        }
-       smlid &= 0x0000FFFF;
 
        clientrereg = (pi->clientrereg_subnettimeout &
                        OPA_PI_MASK_CLIENT_REREGISTER);
@@ -1331,12 +1384,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
        ls_old = driver_lstate(ppd);
 
        ibp->rvp.mkey = pi->mkey;
-       ibp->rvp.gid_prefix = pi->subnet_prefix;
+       if (ibp->rvp.gid_prefix != pi->subnet_prefix) {
+               ibp->rvp.gid_prefix = pi->subnet_prefix;
+               event.event = IB_EVENT_GID_CHANGE;
+               ib_dispatch_event(&event);
+       }
        ibp->rvp.mkey_lease_period = be16_to_cpu(pi->mkey_lease_period);
 
        /* Must be a valid unicast LID address. */
        if ((lid == 0 && ls_old > IB_PORT_INIT) ||
-           lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+            (hfi1_is_16B_mcast(lid))) {
                smp->status |= IB_SMP_INVALID_FIELD;
                pr_warn("SubnSet(OPA_PortInfo) lid invalid 0x%x\n",
                        lid);
@@ -1349,6 +1406,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
                hfi1_set_lid(ppd, lid, pi->mkeyprotect_lmc & OPA_PI_MASK_LMC);
                event.event = IB_EVENT_LID_CHANGE;
                ib_dispatch_event(&event);
+
+               if (HFI1_PORT_GUID_INDEX + 1 < HFI1_GUIDS_PER_PORT) {
+                       /* Manufacture GID from LID to support extended
+                        * addresses
+                        */
+                       ppd->guids[HFI1_PORT_GUID_INDEX + 1] =
+                               be64_to_cpu(OPA_MAKE_ID(lid));
+                       event.event = IB_EVENT_GID_CHANGE;
+                       ib_dispatch_event(&event);
+               }
        }
 
        msl = pi->smsl & OPA_PI_MASK_SMSL;
@@ -1359,7 +1426,7 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 
        /* Must be a valid unicast LID address. */
        if ((smlid == 0 && ls_old > IB_PORT_INIT) ||
-           smlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+            (hfi1_is_16B_mcast(smlid))) {
                smp->status |= IB_SMP_INVALID_FIELD;
                pr_warn("SubnSet(OPA_PortInfo) smlid invalid 0x%x\n", smlid);
        } else if (smlid != ibp->rvp.sm_lid || msl != ibp->rvp.sm_sl) {
@@ -1367,7 +1434,7 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
                spin_lock_irqsave(&ibp->rvp.lock, flags);
                if (ibp->rvp.sm_ah) {
                        if (smlid != ibp->rvp.sm_lid)
-                               rdma_ah_set_dlid(&ibp->rvp.sm_ah->attr, smlid);
+                               hfi1_modify_qp0_ah(ibp, ibp->rvp.sm_ah, smlid);
                        if (msl != ibp->rvp.sm_sl)
                                rdma_ah_set_sl(&ibp->rvp.sm_ah->attr, msl);
                }

diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 18b2727..83565e5 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1394,7 +1394,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
        struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
        struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
        struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];
-       u16 lid = ppd->lid;
+       u32 lid = ppd->lid;
 
        /* props being zeroed by the caller, avoid zeroing it here */
        props->lid = lid ? lid : 0;
@@ -1555,27 +1555,6 @@ static void hfi1_notify_new_ah(struct ib_device *ibdev,
                ah->log_pmtu = ilog2(dd->vld[ah->vl].mtu);
 }
 
-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid)
-{
-       struct rdma_ah_attr attr;
-       struct ib_ah *ah = ERR_PTR(-EINVAL);
-       struct rvt_qp *qp0;
-       struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-       struct hfi1_devdata *dd = dd_from_ppd(ppd);
-       u8 port_num = ppd->port;
-
-       memset(&attr, 0, sizeof(attr));
-       attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
-       rdma_ah_set_dlid(&attr, dlid);
-       rdma_ah_set_port_num(&attr, ppd_from_ibp(ibp)->port);
-       rcu_read_lock();
-       qp0 = rcu_dereference(ibp->rvp.qp[0]);
-       if (qp0)
-               ah = rdma_create_ah(qp0->ibqp.pd, &attr);
-       rcu_read_unlock();
-       return ah;
-}
-
 /**
  * hfi1_get_npkeys - return the size of the PKEY table for context 0
  * @dd: the hfi1_ib device

diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index 4928ee4..ab1618e 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -334,8 +334,6 @@ void hfi1_rc_hdrerr(
 
 u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
 
-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid);
-
 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);
 
 void hfi1_ud_rcv(struct hfi1_packet *packet);

diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c
index 6b9b43b..549c719 100644 (file)
--- a/drivers/infiniband/hw/qib/qib_mad.c
+++ b/drivers/infiniband/hw/qib/qib_mad.c
@@ -105,7 +105,7 @@ static void qib_send_trap(struct qib_ibport *ibp, void *data, unsigned len)
                if (ibp->rvp.sm_lid != be16_to_cpu(IB_LID_PERMISSIVE)) {
                        struct ib_ah *ah;
 
-                       ah = qib_create_qp0_ah(ibp, ibp->rvp.sm_lid);
+                       ah = qib_create_qp0_ah(ibp, (u16)ibp->rvp.sm_lid);
                        if (IS_ERR(ah))
                                ret = PTR_ERR(ah);
                        else {
@@ -496,7 +496,7 @@ static int subn_get_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
                pip->mkey = ibp->rvp.mkey;
        pip->gid_prefix = ibp->rvp.gid_prefix;
        pip->lid = cpu_to_be16(ppd->lid);
-       pip->sm_lid = cpu_to_be16(ibp->rvp.sm_lid);
+       pip->sm_lid = cpu_to_be16((u16)ibp->rvp.sm_lid);
        pip->cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);
        /* pip->diag_code; */
        pip->mkey_lease_period = cpu_to_be16(ibp->rvp.mkey_lease_period);

diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index fdfac0f..1d94f3c 100644 (file)
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -91,7 +91,7 @@ struct rvt_ibport {
        __be16 pma_counter_select[5];
        u16 pma_tag;
        u16 mkey_lease_period;
-       u16 sm_lid;
+       u32 sm_lid;
        u8 sm_sl;
        u8 mkeyprot;
        u8 subnet_timeout;