IB/hfi1: Inline sdma_txclean() for verbs pio

Short circuit sdma_txclean() by adding an __sdma_txclean()
that is only called when the tx has sdma mappings.

Convert internal calls to __sdma_txclean().

This removes a call from the critical path.

Reviewed-by: Sebastian Sanchez <sebastian.sanchez@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index fd39bca..7102a07 100644 (file)
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -375,7 +375,7 @@ static inline void complete_tx(struct sdma_engine *sde,
                           sde->head_sn, tx->sn);
        sde->head_sn++;
 #endif
-       sdma_txclean(sde->dd, tx);
+       __sdma_txclean(sde->dd, tx);
        if (complete)
                (*complete)(tx, res);
        if (wait && iowait_sdma_dec(wait))
@@ -1643,7 +1643,7 @@ static inline u8 ahg_mode(struct sdma_txreq *tx)
 }
 
 /**
- * sdma_txclean() - clean tx of mappings, descp *kmalloc's
+ * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
  * @dd: hfi1_devdata for unmapping
  * @tx: tx request to clean
  *
@@ -1653,7 +1653,7 @@ static inline u8 ahg_mode(struct sdma_txreq *tx)
  * The code can be called multiple times without issue.
  *
  */
-void sdma_txclean(
+void __sdma_txclean(
        struct hfi1_devdata *dd,
        struct sdma_txreq *tx)
 {
@@ -3080,7 +3080,7 @@ static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
                tx->descp[i] = tx->descs[i];
        return 0;
 enomem:
-       sdma_txclean(dd, tx);
+       __sdma_txclean(dd, tx);
        return -ENOMEM;
 }
 
@@ -3109,14 +3109,14 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
 
        rval = _extend_sdma_tx_descs(dd, tx);
        if (rval) {
-               sdma_txclean(dd, tx);
+               __sdma_txclean(dd, tx);
                return rval;
        }
 
        /* If coalesce buffer is allocated, copy data into it */
        if (tx->coalesce_buf) {
                if (type == SDMA_MAP_NONE) {
-                       sdma_txclean(dd, tx);
+                       __sdma_txclean(dd, tx);
                        return -EINVAL;
                }
 
@@ -3124,7 +3124,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
                        kvaddr = kmap(page);
                        kvaddr += offset;
                } else if (WARN_ON(!kvaddr)) {
-                       sdma_txclean(dd, tx);
+                       __sdma_txclean(dd, tx);
                        return -EINVAL;
                }
 
@@ -3154,7 +3154,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
                                      DMA_TO_DEVICE);
 
                if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-                       sdma_txclean(dd, tx);
+                       __sdma_txclean(dd, tx);
                        return -ENOSPC;
                }
 
@@ -3196,7 +3196,7 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
        if ((unlikely(tx->num_desc == tx->desc_limit))) {
                rval = _extend_sdma_tx_descs(dd, tx);
                if (rval) {
-                       sdma_txclean(dd, tx);
+                       __sdma_txclean(dd, tx);
                        return rval;
                }
        }

diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 56257ea..21f1e28 100644 (file)
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -667,7 +667,13 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
                           int type, void *kvaddr, struct page *page,
                           unsigned long offset, u16 len);
 int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *);
-void sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *);
+void __sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *);
+
+static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx)
+{
+       if (tx->num_desc)
+               __sdma_txclean(dd, tx);
+}
 
 /* helpers used by public routines */
 static inline void _sdma_close_tx(struct hfi1_devdata *dd,
@@ -753,7 +759,7 @@ static inline int sdma_txadd_page(
                       DMA_TO_DEVICE);
 
        if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-               sdma_txclean(dd, tx);
+               __sdma_txclean(dd, tx);
                return -ENOSPC;
        }
 
@@ -834,7 +840,7 @@ static inline int sdma_txadd_kvaddr(
                       DMA_TO_DEVICE);
 
        if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-               sdma_txclean(dd, tx);
+               __sdma_txclean(dd, tx);
                return -ENOSPC;
        }