liquidio: xmit_more support

Defer ringing the Tx doorbell if skb->xmit_more is set unless the Tx queue
is full or stopped.  To keep latency low, use a deferral limit of 8
packets.  We chose 8 because Octeon can fetch at most 8 packets in a single
PCI read, and our tests show that 8 results in low latency.

Signed-off-by: Intiyaz Basha <intiyaz.basha@cavium.com>
Signed-off-by: Satanand Burla <satananda.burla@cavium.com>
Signed-off-by: Felix Manlunas <felix.manlunas@cavium.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
index 23f6b60..b891d85 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/lio_core.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -91,7 +91,7 @@ void octeon_update_tx_completion_counters(void *buf, int reqtype,
        *bytes_compl += skb->len;
 }
 
-void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
+int octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
 {
        struct octnet_buf_free_info *finfo;
        struct sk_buff *skb;
@@ -112,11 +112,13 @@ void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
                break;
 
        default:
-               return;
+               return 0;
        }
 
        txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
        netdev_tx_sent_queue(txq, skb->len);
+
+       return netif_xmit_stopped(txq);
 }
 
 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)

diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index accd038..8ea24d6 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -2594,7 +2594,8 @@ static void handle_timestamp(struct octeon_device *oct,
  */
 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
                                         struct octnic_data_pkt *ndata,
-                                        struct octnet_buf_free_info *finfo)
+                                        struct octnet_buf_free_info *finfo,
+                                        int xmit_more)
 {
        int retval;
        struct octeon_soft_command *sc;
@@ -2629,7 +2630,7 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
                len = (u32)((struct octeon_instr_ih2 *)
                            (&sc->cmd.cmd2.ih2))->dlengsz;
 
-       ring_doorbell = 1;
+       ring_doorbell = !xmit_more;
 
        retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
                                     sc, len, ndata->reqtype);
@@ -2663,7 +2664,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
        union tx_info *tx_info;
        int status = 0;
        int q_idx = 0, iq_no = 0;
-       int j;
+       int j, xmit_more = 0;
        u64 dptr = 0;
        u32 tag = 0;
 
@@ -2868,17 +2869,19 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
                irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
        }
 
+       xmit_more = skb->xmit_more;
+
        if (unlikely(cmdsetup.s.timestamp))
-               status = send_nic_timestamp_pkt(oct, &ndata, finfo);
+               status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
        else
-               status = octnet_send_nic_data_pkt(oct, &ndata);
+               status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
        if (status == IQ_SEND_FAILED)
                goto lio_xmit_failed;
 
        netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
 
        if (status == IQ_SEND_STOP)
-               stop_q(lio->netdev, q_idx);
+               stop_q(netdev, q_idx);
 
        netif_trans_update(netdev);
 
@@ -2897,6 +2900,9 @@ lio_xmit_failed:
        if (dptr)
                dma_unmap_single(&oct->pci_dev->dev, dptr,
                                 ndata.datasize, DMA_TO_DEVICE);
+
+       octeon_ring_doorbell_locked(oct, iq_no);
+
        tx_buffer_free(skb);
        return NETDEV_TX_OK;
 }

diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
index 4c3b568..00c1930 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
@@ -1690,7 +1690,8 @@ static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf)
  */
 static int send_nic_timestamp_pkt(struct octeon_device *oct,
                                  struct octnic_data_pkt *ndata,
-                                 struct octnet_buf_free_info *finfo)
+                                 struct octnet_buf_free_info *finfo,
+                                 int xmit_more)
 {
        struct octeon_soft_command *sc;
        int ring_doorbell;
@@ -1720,7 +1721,7 @@ static int send_nic_timestamp_pkt(struct octeon_device *oct,
 
        len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz;
 
-       ring_doorbell = 1;
+       ring_doorbell = !xmit_more;
 
        retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
                                     sc, len, ndata->reqtype);
@@ -1752,6 +1753,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
        struct octeon_device *oct;
        int q_idx = 0, iq_no = 0;
        union tx_info *tx_info;
+       int xmit_more = 0;
        struct lio *lio;
        int status = 0;
        u64 dptr = 0;
@@ -1940,10 +1942,12 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
                irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK;
        }
 
+       xmit_more = skb->xmit_more;
+
        if (unlikely(cmdsetup.s.timestamp))
-               status = send_nic_timestamp_pkt(oct, &ndata, finfo);
+               status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
        else
-               status = octnet_send_nic_data_pkt(oct, &ndata);
+               status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
        if (status == IQ_SEND_FAILED)
                goto lio_xmit_failed;
 
@@ -1952,7 +1956,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
        if (status == IQ_SEND_STOP) {
                dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n",
                        iq_no);
-               stop_q(lio->netdev, q_idx);
+               stop_q(netdev, q_idx);
        }
 
        netif_trans_update(netdev);
@@ -1972,6 +1976,9 @@ lio_xmit_failed:
        if (dptr)
                dma_unmap_single(&oct->pci_dev->dev, dptr,
                                 ndata.datasize, DMA_TO_DEVICE);
+
+       octeon_ring_doorbell_locked(oct, iq_no);
+
        tx_buffer_free(skb);
        return NETDEV_TX_OK;
 }

diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
index 63bd9c9..ceac743 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
@@ -37,6 +37,8 @@
 #define   MAX_OCTEON_LINKS            MAX_OCTEON_NICIF
 #define   MAX_OCTEON_MULTICAST_ADDR    32
 
+#define   MAX_OCTEON_FILL_COUNT        8
+
 /* CN6xxx IQ configuration macros */
 #define   CN6XXX_MAX_INPUT_QUEUES      32
 #define   CN6XXX_MAX_IQ_DESCRIPTORS    2048

diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
index 5c3c8da..81c9876 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
@@ -343,6 +343,9 @@ int octeon_delete_instr_queue(struct octeon_device *octeon_dev, u32 iq_no);
 
 int lio_wait_for_instr_fetch(struct octeon_device *oct);
 
+void
+octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no);
+
 int
 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
                                void (*fn)(void *));

diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
index 32ef3a7..c846eec 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
@@ -63,7 +63,7 @@ struct octnet_buf_free_info {
 };
 
 /* BQL-related functions */
-void octeon_report_sent_bytes_to_bql(void *buf, int reqtype);
+int octeon_report_sent_bytes_to_bql(void *buf, int reqtype);
 void octeon_update_tx_completion_counters(void *buf, int reqtype,
                                          unsigned int *pkts_compl,
                                          unsigned int *bytes_compl);

diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
index b457cf2..150609b 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
@@ -82,9 +82,10 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
 }
 
 int octnet_send_nic_data_pkt(struct octeon_device *oct,
-                            struct octnic_data_pkt *ndata)
+                            struct octnic_data_pkt *ndata,
+                            int xmit_more)
 {
-       int ring_doorbell = 1;
+       int ring_doorbell = !xmit_more;
 
        return octeon_send_command(oct, ndata->q_no, ring_doorbell, &ndata->cmd,
                                   ndata->buf, ndata->datasize,

diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
index 6480ef8..de4130d 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
@@ -279,7 +279,8 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
  * queue should be stopped, and IQ_SEND_OK if it sent okay.
  */
 int octnet_send_nic_data_pkt(struct octeon_device *oct,
-                            struct octnic_data_pkt *ndata);
+                            struct octnic_data_pkt *ndata,
+                            int xmit_more);
 
 /** Send a NIC control packet to the device
  * @param oct - octeon device pointer

diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index 1e0fbce..a104597 100644 (file)
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -278,6 +278,18 @@ ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)
        }
 }
 
+void
+octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no)
+{
+       struct octeon_instr_queue *iq;
+
+       iq = oct->instr_queue[iq_no];
+       spin_lock(&iq->post_lock);
+       if (iq->fill_cnt)
+               ring_doorbell(oct, iq);
+       spin_unlock(&iq->post_lock);
+}
+
 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq,
                                      u8 *cmd)
 {
@@ -543,6 +555,7 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no,
                    u32 force_db, void *cmd, void *buf,
                    u32 datasize, u32 reqtype)
 {
+       int xmit_stopped;
        struct iq_post_status st;
        struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 
@@ -554,12 +567,13 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no,
        st = __post_command2(iq, cmd);
 
        if (st.status != IQ_SEND_FAILED) {
-               octeon_report_sent_bytes_to_bql(buf, reqtype);
+               xmit_stopped = octeon_report_sent_bytes_to_bql(buf, reqtype);
                __add_to_request_list(iq, st.index, buf, reqtype);
                INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
                INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
 
-               if (force_db)
+               if (iq->fill_cnt >= MAX_OCTEON_FILL_COUNT || force_db ||
+                   xmit_stopped || st.status == IQ_SEND_STOP)
                        ring_doorbell(oct, iq);
        } else {
                INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);