From 9d32e4e7e9e11b1dd0ed0d6c294258879ac4d1af Mon Sep 17 00:00:00 2001 From: Yinjun Zhang Date: Mon, 26 Jul 2021 13:16:34 +0200 Subject: [PATCH] nfp: add support for coalesce adaptive feature Use dynamic interrupt moderation library to implement coalesce adaptive feature for nfp driver. Signed-off-by: Yinjun Zhang Signed-off-by: Yu Xiao Signed-off-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/netronome/Kconfig | 1 + drivers/net/ethernet/netronome/nfp/nfp_net.h | 20 ++++ .../net/ethernet/netronome/nfp/nfp_net_common.c | 131 ++++++++++++++++++++- .../net/ethernet/netronome/nfp/nfp_net_ethtool.c | 21 ++-- 4 files changed, 160 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig index b82758d..8844d1a 100644 --- a/drivers/net/ethernet/netronome/Kconfig +++ b/drivers/net/ethernet/netronome/Kconfig @@ -23,6 +23,7 @@ config NFP depends on TLS && TLS_DEVICE || TLS_DEVICE=n select NET_DEVLINK select CRC32 + select DIMLIB help This driver supports the Netronome(R) NFP4000/NFP6000 based cards working as a advanced Ethernet NIC. It works with both diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h index df5b748..df20373 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -360,6 +361,9 @@ struct nfp_net_rx_ring { * @rx_ring: Pointer to RX ring * @xdp_ring: Pointer to an extra TX ring for XDP * @irq_entry: MSI-X table entry (use for talking to the device) + * @event_ctr: Number of interrupt + * @rx_dim: Dynamic interrupt moderation structure for RX + * @tx_dim: Dynamic interrupt moderation structure for TX * @rx_sync: Seqlock for atomic updates of RX stats * @rx_pkts: Number of received packets * @rx_bytes: Number of received bytes @@ -410,6 +414,10 @@ struct nfp_net_r_vector { u16 irq_entry; + u16 event_ctr; + struct dim rx_dim; + struct dim tx_dim; + struct u64_stats_sync rx_sync; u64 rx_pkts; u64 rx_bytes; @@ -571,6 +579,8 @@ struct nfp_net_dp { * mailbox area, crypto TLV * @link_up: Is the link up? * @link_status_lock: Protects @link_* and ensures atomicity with BAR reading + * @rx_coalesce_adapt_on: Is RX interrupt moderation adaptive? + * @tx_coalesce_adapt_on: Is TX interrupt moderation adaptive? * @rx_coalesce_usecs: RX interrupt moderation usecs delay parameter * @rx_coalesce_max_frames: RX interrupt moderation frame count parameter * @tx_coalesce_usecs: TX interrupt moderation usecs delay parameter @@ -654,6 +664,8 @@ struct nfp_net { struct semaphore bar_lock; + bool rx_coalesce_adapt_on; + bool tx_coalesce_adapt_on; u32 rx_coalesce_usecs; u32 rx_coalesce_max_frames; u32 tx_coalesce_usecs; @@ -919,6 +931,14 @@ static inline bool nfp_netdev_is_nfp_net(struct net_device *netdev) return netdev->netdev_ops == &nfp_net_netdev_ops; } +static inline int nfp_net_coalesce_para_check(u32 usecs, u32 pkts) +{ + if ((usecs >= ((1 << 16) - 1)) || (pkts >= ((1 << 16) - 1))) + return -EINVAL; + + return 0; +} + /* Prototypes */ void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver, void __iomem *ctrl_bar); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index ed2ade2..15078f9 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -474,6 +474,12 @@ static irqreturn_t nfp_net_irq_rxtx(int irq, void *data) { struct nfp_net_r_vector *r_vec = data; + /* Currently we cannot tell if it's a rx or tx interrupt, + * since dim does not need accurate event_ctr to calculate, + * we just use this counter for both rx and tx dim. + */ + r_vec->event_ctr++; + napi_schedule_irqoff(&r_vec->napi); /* The FW auto-masks any interrupt, either via the MASK bit in @@ -2061,6 +2067,36 @@ static int nfp_net_poll(struct napi_struct *napi, int budget) if (napi_complete_done(napi, pkts_polled)) nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry); + if (r_vec->nfp_net->rx_coalesce_adapt_on) { + struct dim_sample dim_sample = {}; + unsigned int start; + u64 pkts, bytes; + + do { + start = u64_stats_fetch_begin(&r_vec->rx_sync); + pkts = r_vec->rx_pkts; + bytes = r_vec->rx_bytes; + } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); + + dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); + net_dim(&r_vec->rx_dim, dim_sample); + } + + if (r_vec->nfp_net->tx_coalesce_adapt_on) { + struct dim_sample dim_sample = {}; + unsigned int start; + u64 pkts, bytes; + + do { + start = u64_stats_fetch_begin(&r_vec->tx_sync); + pkts = r_vec->tx_pkts; + bytes = r_vec->tx_bytes; + } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); + + dim_update_sample(r_vec->event_ctr, pkts, bytes, &dim_sample); + net_dim(&r_vec->tx_dim, dim_sample); + } + return pkts_polled; } @@ -2873,6 +2909,7 @@ static int nfp_net_set_config_and_enable(struct nfp_net *nn) */ static void nfp_net_close_stack(struct nfp_net *nn) { + struct nfp_net_r_vector *r_vec; unsigned int r; disable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector); @@ -2880,8 +2917,16 @@ static void nfp_net_close_stack(struct nfp_net *nn) nn->link_up = false; for (r = 0; r < nn->dp.num_r_vecs; r++) { - disable_irq(nn->r_vecs[r].irq_vector); - napi_disable(&nn->r_vecs[r].napi); + r_vec = &nn->r_vecs[r]; + + disable_irq(r_vec->irq_vector); + napi_disable(&r_vec->napi); + + if (r_vec->rx_ring) + cancel_work_sync(&r_vec->rx_dim.work); + + if (r_vec->tx_ring) + cancel_work_sync(&r_vec->tx_dim.work); } netif_tx_disable(nn->dp.netdev); @@ -2948,17 +2993,92 @@ void nfp_ctrl_close(struct nfp_net *nn) rtnl_unlock(); } +static void nfp_net_rx_dim_work(struct work_struct *work) +{ + struct nfp_net_r_vector *r_vec; + unsigned int factor, value; + struct dim_cq_moder moder; + struct nfp_net *nn; + struct dim *dim; + + dim = container_of(work, struct dim, work); + moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + r_vec = container_of(dim, struct nfp_net_r_vector, rx_dim); + nn = r_vec->nfp_net; + + /* Compute factor used to convert coalesce '_usecs' parameters to + * ME timestamp ticks. There are 16 ME clock cycles for each timestamp + * count. + */ + factor = nn->tlv_caps.me_freq_mhz / 16; + if (nfp_net_coalesce_para_check(factor * moder.usec, moder.pkts)) + return; + + /* copy RX interrupt coalesce parameters */ + value = (moder.pkts << 16) | (factor * moder.usec); + rtnl_lock(); + nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(r_vec->rx_ring->idx), value); + (void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD); + rtnl_unlock(); + + dim->state = DIM_START_MEASURE; +} + +static void nfp_net_tx_dim_work(struct work_struct *work) +{ + struct nfp_net_r_vector *r_vec; + unsigned int factor, value; + struct dim_cq_moder moder; + struct nfp_net *nn; + struct dim *dim; + + dim = container_of(work, struct dim, work); + moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix); + r_vec = container_of(dim, struct nfp_net_r_vector, tx_dim); + nn = r_vec->nfp_net; + + /* Compute factor used to convert coalesce '_usecs' parameters to + * ME timestamp ticks. There are 16 ME clock cycles for each timestamp + * count. + */ + factor = nn->tlv_caps.me_freq_mhz / 16; + if (nfp_net_coalesce_para_check(factor * moder.usec, moder.pkts)) + return; + + /* copy TX interrupt coalesce parameters */ + value = (moder.pkts << 16) | (factor * moder.usec); + rtnl_lock(); + nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(r_vec->tx_ring->idx), value); + (void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD); + rtnl_unlock(); + + dim->state = DIM_START_MEASURE; +} + /** * nfp_net_open_stack() - Start the device from stack's perspective * @nn: NFP Net device to reconfigure */ static void nfp_net_open_stack(struct nfp_net *nn) { + struct nfp_net_r_vector *r_vec; unsigned int r; for (r = 0; r < nn->dp.num_r_vecs; r++) { - napi_enable(&nn->r_vecs[r].napi); - enable_irq(nn->r_vecs[r].irq_vector); + r_vec = &nn->r_vecs[r]; + + if (r_vec->rx_ring) { + INIT_WORK(&r_vec->rx_dim.work, nfp_net_rx_dim_work); + r_vec->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } + + if (r_vec->tx_ring) { + INIT_WORK(&r_vec->tx_dim.work, nfp_net_tx_dim_work); + r_vec->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } + + napi_enable(&r_vec->napi); + enable_irq(r_vec->irq_vector); } netif_tx_wake_all_queues(nn->dp.netdev); @@ -3893,6 +4013,9 @@ static void nfp_net_irqmod_init(struct nfp_net *nn) nn->rx_coalesce_max_frames = 64; nn->tx_coalesce_usecs = 50; nn->tx_coalesce_max_frames = 64; + + nn->rx_coalesce_adapt_on = true; + nn->tx_coalesce_adapt_on = true; } static void nfp_net_netdev_init(struct nfp_net *nn) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index 1b48244..a213784 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1083,6 +1083,9 @@ static int nfp_net_get_coalesce(struct net_device *netdev, if (!(nn->cap & NFP_NET_CFG_CTRL_IRQMOD)) return -EINVAL; + ec->use_adaptive_rx_coalesce = nn->rx_coalesce_adapt_on; + ec->use_adaptive_tx_coalesce = nn->tx_coalesce_adapt_on; + ec->rx_coalesce_usecs = nn->rx_coalesce_usecs; ec->rx_max_coalesced_frames = nn->rx_coalesce_max_frames; ec->tx_coalesce_usecs = nn->tx_coalesce_usecs; @@ -1359,19 +1362,18 @@ static int nfp_net_set_coalesce(struct net_device *netdev, if (!ec->tx_coalesce_usecs && !ec->tx_max_coalesced_frames) return -EINVAL; - if (ec->rx_coalesce_usecs * factor >= ((1 << 16) - 1)) - return -EINVAL; - - if (ec->tx_coalesce_usecs * factor >= ((1 << 16) - 1)) + if (nfp_net_coalesce_para_check(ec->rx_coalesce_usecs * factor, + ec->rx_max_coalesced_frames)) return -EINVAL; - if (ec->rx_max_coalesced_frames >= ((1 << 16) - 1)) - return -EINVAL; - - if (ec->tx_max_coalesced_frames >= ((1 << 16) - 1)) + if (nfp_net_coalesce_para_check(ec->tx_coalesce_usecs * factor, + ec->tx_max_coalesced_frames)) return -EINVAL; /* configuration is valid */ + nn->rx_coalesce_adapt_on = !!ec->use_adaptive_rx_coalesce; + nn->tx_coalesce_adapt_on = !!ec->use_adaptive_tx_coalesce; + nn->rx_coalesce_usecs = ec->rx_coalesce_usecs; nn->rx_coalesce_max_frames = ec->rx_max_coalesced_frames; nn->tx_coalesce_usecs = ec->tx_coalesce_usecs; @@ -1443,7 +1445,8 @@ static int nfp_net_set_channels(struct net_device *netdev, static const struct ethtool_ops nfp_net_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_MAX_FRAMES, + ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_USE_ADAPTIVE, .get_drvinfo = nfp_net_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = nfp_net_get_ringparam, -- 2.7.4