IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev

The current rdma_netdev handling in ipoib hooks the tx_timeout handler,
but prints out a totally useless message that prevents effective debugging
especially when multiple transmit queues are being used.

Add a tx_timeout rdma_netdev hook and implement the callback in the hfi1
to print additional information.

The existing non-helpful message is avoided when the driver has presented
a callback.

Link: https://lore.kernel.org/r/1617026056-50483-3-git-send-email-dennis.dalessandro@cornelisnetworks.com
Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h
index 02c5b0d1ffc4cd65dee1f45de005c27d3421f094..97b9fce6173449768428f899fb27425a2ba4f2d3 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ipoib.h
+++ b/drivers/infiniband/hw/hfi1/ipoib.h
@@ -147,4 +147,6 @@ int hfi1_ipoib_rn_get_params(struct ib_device *device,
                             enum rdma_netdev_t type,
                             struct rdma_netdev_alloc_params *params);
 
+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q);
+
 #endif /* _IPOIB_H */

diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c
index 673dc754ea183316b5eab63403aa93381ef01038..771d8d7886b54beac815aeea934f51a397adfd4d 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ipoib_main.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_main.c
@@ -204,6 +204,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device,
        int rc;
 
        rn->send = hfi1_ipoib_send;
+       rn->tx_timeout = hfi1_ipoib_tx_timeout;
        rn->attach_mcast = hfi1_ipoib_mcast_attach;
        rn->detach_mcast = hfi1_ipoib_mcast_detach;
        rn->set_id = hfi1_ipoib_set_id;

diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index 5129dc95dec99136d67c5ff12dacb6fbea11d9ac..e8dece5593011f3fbbb3eeb049181450e24b963c 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -866,3 +866,32 @@ void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
                (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
        }
 }
+
+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)
+{
+       struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+       struct hfi1_ipoib_txq *txq = &priv->txqs[q];
+       u64 completed = atomic64_read(&txq->complete_txreqs);
+
+       dd_dev_info(priv->dd, "timeout txq %llx q %u stopped %u stops %d no_desc %d ring_full %d\n",
+                   (unsigned long long)txq, q,
+                   __netif_subqueue_stopped(dev, txq->q_idx),
+                   atomic_read(&txq->stops),
+                   atomic_read(&txq->no_desc),
+                   atomic_read(&txq->ring_full));
+       dd_dev_info(priv->dd, "sde %llx engine %u\n",
+                   (unsigned long long)txq->sde,
+                   txq->sde ? txq->sde->this_idx : 0);
+       dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int);
+       dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n",
+                   txq->sent_txreqs, completed, hfi1_ipoib_used(txq));
+       dd_dev_info(priv->dd, "tx_queue_len %u max_items %lu\n",
+                   dev->tx_queue_len, txq->tx_ring.max_items);
+       dd_dev_info(priv->dd, "head %lu tail %lu\n",
+                   txq->tx_ring.head, txq->tx_ring.tail);
+       dd_dev_info(priv->dd, "wait queued %u\n",
+                   !list_empty(&txq->wait.list));
+       dd_dev_info(priv->dd, "tx_list empty %u\n",
+                   list_empty(&txq->tx_list));
+}
+

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 8f769ebaacc6654c295689596d226070f8eaceb2..8ca80e27339ae5c3771141b3096a9ad0f7e41226 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1181,7 +1181,12 @@ unref:
 static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
 {
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
+       struct rdma_netdev *rn = netdev_priv(dev);
 
+       if (rn->tx_timeout) {
+               rn->tx_timeout(dev, txqueue);
+               return;
+       }
        ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
                   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
        ipoib_warn(priv,

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index bed4cfe50554f78becc9e3455b11cfe40f82cabf..c596882893aeb263cad43bb66a21feabc080b305 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2215,6 +2215,8 @@ struct rdma_netdev {
                            int set_qkey, u32 qkey);
        int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
                            union ib_gid *gid, u16 mlid);
+       /* timeout */
+       void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
 };
 
 struct rdma_netdev_alloc_params {