octeontx2-pf: Add support for adaptive interrupt coalescing

Added support for adaptive IRQ coalescing. It uses net_dim
algorithm to find the suitable delay/IRQ count based on the
current packet rate.

Signed-off-by: Suman Ghosh <sumang@marvell.com>
Link: https://lore.kernel.org/r/20220517044055.876158-1-sumang@marvell.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig
index 639893d..e1036b0 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/Kconfig
+++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig
@@ -33,6 +33,7 @@ config OCTEONTX2_PF
        select OCTEONTX2_MBOX
        select NET_DEVLINK
        depends on (64BIT && COMPILE_TEST) || ARM64
+       select DIMLIB
        depends on PCI
        depends on PTP_1588_CLOCK_OPTIONAL
        help

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index b9d7601..fb8db58 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -97,11 +97,6 @@ void otx2_get_dev_stats(struct otx2_nic *pfvf)
 {
        struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats;
 
-#define OTX2_GET_RX_STATS(reg) \
-        otx2_read64(pfvf, NIX_LF_RX_STATX(reg))
-#define OTX2_GET_TX_STATS(reg) \
-        otx2_read64(pfvf, NIX_LF_TX_STATX(reg))
-
        dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
        dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP);
        dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST);

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index c587c14..ce27663 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -18,6 +18,7 @@
 #include <net/pkt_cls.h>
 #include <net/devlink.h>
 #include <linux/time64.h>
+#include <linux/dim.h>
 
 #include <mbox.h>
 #include <npc.h>
@@ -54,6 +55,11 @@ enum arua_mapped_qtypes {
 /* Send skid of 2000 packets required for CQ size of 4K CQEs. */
 #define SEND_CQ_SKID   2000
 
+#define OTX2_GET_RX_STATS(reg) \
+       otx2_read64(pfvf, NIX_LF_RX_STATX(reg))
+#define OTX2_GET_TX_STATS(reg) \
+       otx2_read64(pfvf, NIX_LF_TX_STATX(reg))
+
 struct otx2_lmt_info {
        u64 lmt_addr;
        u16 lmt_id;
@@ -351,6 +357,7 @@ struct otx2_nic {
 #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED   BIT_ULL(12)
 #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED  BIT_ULL(13)
 #define OTX2_FLAG_DMACFLTR_SUPPORT             BIT_ULL(14)
+#define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16)
        u64                     flags;
        u64                     *cq_op_addr;
 
@@ -408,6 +415,9 @@ struct otx2_nic {
        u8                      pfc_en;
        u8                      *queue_to_pfc_map;
 #endif
+
+       /* napi event count. It is needed for adaptive irq coalescing. */
+       u32 napi_events;
 };
 
 static inline bool is_otx2_lbkvf(struct pci_dev *pdev)

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
index fc328de..bc614a4 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
@@ -455,6 +455,14 @@ static int otx2_get_coalesce(struct net_device *netdev,
        cmd->rx_max_coalesced_frames = hw->cq_ecount_wait;
        cmd->tx_coalesce_usecs = hw->cq_time_wait;
        cmd->tx_max_coalesced_frames = hw->cq_ecount_wait;
+       if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
+                       OTX2_FLAG_ADPTV_INT_COAL_ENABLED) {
+               cmd->use_adaptive_rx_coalesce = 1;
+               cmd->use_adaptive_tx_coalesce = 1;
+       } else {
+               cmd->use_adaptive_rx_coalesce = 0;
+               cmd->use_adaptive_tx_coalesce = 0;
+       }
 
        return 0;
 }
@@ -466,11 +474,30 @@ static int otx2_set_coalesce(struct net_device *netdev,
 {
        struct otx2_nic *pfvf = netdev_priv(netdev);
        struct otx2_hw *hw = &pfvf->hw;
+       u8 priv_coalesce_status;
        int qidx;
 
        if (!ec->rx_max_coalesced_frames || !ec->tx_max_coalesced_frames)
                return 0;
 
+       if (ec->use_adaptive_rx_coalesce != ec->use_adaptive_tx_coalesce) {
+               netdev_err(netdev,
+                          "adaptive-rx should be same as adaptive-tx");
+               return -EINVAL;
+       }
+
+       /* Check and update coalesce status */
+       if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
+                       OTX2_FLAG_ADPTV_INT_COAL_ENABLED) {
+               priv_coalesce_status = 1;
+               if (!ec->use_adaptive_rx_coalesce)
+                       pfvf->flags &= ~OTX2_FLAG_ADPTV_INT_COAL_ENABLED;
+       } else {
+               priv_coalesce_status = 0;
+               if (ec->use_adaptive_rx_coalesce)
+                       pfvf->flags |= OTX2_FLAG_ADPTV_INT_COAL_ENABLED;
+       }
+
        /* 'cq_time_wait' is 8bit and is in multiple of 100ns,
         * so clamp the user given value to the range of 1 to 25usec.
         */
@@ -494,9 +521,9 @@ static int otx2_set_coalesce(struct net_device *netdev,
         * so clamp the user given value to the range of 1 to 64k.
         */
        ec->rx_max_coalesced_frames = clamp_t(u32, ec->rx_max_coalesced_frames,
-                                             1, U16_MAX);
+                                             1, NAPI_POLL_WEIGHT);
        ec->tx_max_coalesced_frames = clamp_t(u32, ec->tx_max_coalesced_frames,
-                                             1, U16_MAX);
+                                             1, NAPI_POLL_WEIGHT);
 
        /* Rx and Tx are mapped to same CQ, check which one
         * is changed, if both then choose the min.
@@ -509,6 +536,17 @@ static int otx2_set_coalesce(struct net_device *netdev,
                hw->cq_ecount_wait = min_t(u16, ec->rx_max_coalesced_frames,
                                           ec->tx_max_coalesced_frames);
 
+       /* Reset 'cq_time_wait' and 'cq_ecount_wait' to
+        * default values if coalesce status changed from
+        * 'on' to 'off'.
+        */
+       if (priv_coalesce_status &&
+           ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) !=
+            OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) {
+               hw->cq_time_wait = CQ_TIMER_THRESH_DEFAULT;
+               hw->cq_ecount_wait = CQ_CQE_THRESH_DEFAULT;
+       }
+
        if (netif_running(netdev)) {
                for (qidx = 0; qidx < pfvf->hw.cint_cnt; qidx++)
                        otx2_config_irq_coalescing(pfvf, qidx);
@@ -1230,7 +1268,8 @@ end:
 
 static const struct ethtool_ops otx2_ethtool_ops = {
        .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
-                                    ETHTOOL_COALESCE_MAX_FRAMES,
+                                    ETHTOOL_COALESCE_MAX_FRAMES |
+                                    ETHTOOL_COALESCE_USE_ADAPTIVE,
        .supported_ring_params  = ETHTOOL_RING_USE_RX_BUF_LEN |
                                  ETHTOOL_RING_USE_CQE_SIZE,
        .get_link               = otx2_get_link,

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index e193ffb..fe3472e 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -1254,6 +1254,7 @@ static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
        otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
 
        /* Schedule NAPI */
+       pf->napi_events++;
        napi_schedule_irqoff(&cq_poll->napi);
 
        return IRQ_HANDLED;
@@ -1267,6 +1268,7 @@ static void otx2_disable_napi(struct otx2_nic *pf)
 
        for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
                cq_poll = &qset->napi[qidx];
+               cancel_work_sync(&cq_poll->dim.work);
                napi_disable(&cq_poll->napi);
                netif_napi_del(&cq_poll->napi);
        }
@@ -1546,6 +1548,24 @@ static void otx2_do_set_rx_mode(struct otx2_nic *pf)
        mutex_unlock(&pf->mbox.lock);
 }
 
+static void otx2_dim_work(struct work_struct *w)
+{
+       struct dim_cq_moder cur_moder;
+       struct otx2_cq_poll *cq_poll;
+       struct otx2_nic *pfvf;
+       struct dim *dim;
+
+       dim = container_of(w, struct dim, work);
+       cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
+       cq_poll = container_of(dim, struct otx2_cq_poll, dim);
+       pfvf = (struct otx2_nic *)cq_poll->dev;
+       pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
+               CQ_TIMER_THRESH_MAX : cur_moder.usec;
+       pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
+               NAPI_POLL_WEIGHT : cur_moder.pkts;
+       dim->state = DIM_START_MEASURE;
+}
+
 int otx2_open(struct net_device *netdev)
 {
        struct otx2_nic *pf = netdev_priv(netdev);
@@ -1612,6 +1632,8 @@ int otx2_open(struct net_device *netdev)
                        cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
 
                cq_poll->dev = (void *)pf;
+               cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
+               INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
                netif_napi_add(netdev, &cq_poll->napi,
                               otx2_napi_handler, NAPI_POLL_WEIGHT);
                napi_enable(&cq_poll->napi);

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index c26de15..3baeafc 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -484,6 +484,18 @@ process_cqe:
        return 0;
 }
 
+static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
+{
+       struct dim_sample dim_sample;
+       u64 rx_frames, rx_bytes;
+
+       rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
+               OTX2_GET_RX_STATS(RX_UCAST);
+       rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
+       dim_update_sample(pfvf->napi_events, rx_frames, rx_bytes, &dim_sample);
+       net_dim(&cq_poll->dim, dim_sample);
+}
+
 int otx2_napi_handler(struct napi_struct *napi, int budget)
 {
        struct otx2_cq_queue *rx_cq = NULL;
@@ -521,6 +533,17 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
                if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
                        return workdone;
 
+               /* Check for adaptive interrupt coalesce */
+               if (workdone != 0 &&
+                   ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
+                    OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) {
+                       /* Adjust irq coalese using net_dim */
+                       otx2_adjust_adaptive_coalese(pfvf, cq_poll);
+                       /* Update irq coalescing */
+                       for (i = 0; i < pfvf->hw.cint_cnt; i++)
+                               otx2_config_irq_coalescing(pfvf, i);
+               }
+
                /* Re-enable interrupts */
                otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
                             BIT_ULL(0));

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index f1a04cf..c88e8a4 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -109,6 +109,7 @@ struct otx2_cq_poll {
 #define CINT_INVALID_CQ                255
        u8                      cint_idx;
        u8                      cq_ids[CQS_PER_CINT];
+       struct dim              dim;
        struct napi_struct      napi;
 };