From c859e21a35ce5604dde0b618169680aa3c7e3bdb Mon Sep 17 00:00:00 2001 From: Intiyaz Basha Date: Thu, 26 Oct 2017 16:18:20 -0700 Subject: [PATCH] liquidio: xmit_more support Defer ringing the Tx doorbell if skb->xmit_more is set unless the Tx queue is full or stopped. To keep latency low, use a deferral limit of 8 packets. We chose 8 because Octeon can fetch at most 8 packets in a single PCI read, and our tests show that 8 results in low latency. Signed-off-by: Intiyaz Basha Signed-off-by: Satanand Burla Signed-off-by: Felix Manlunas Signed-off-by: David S. Miller --- .../net/ethernet/cavium/liquidio/lio_core.c | 6 ++++-- .../net/ethernet/cavium/liquidio/lio_main.c | 18 ++++++++++++------ .../net/ethernet/cavium/liquidio/lio_vf_main.c | 17 ++++++++++++----- .../ethernet/cavium/liquidio/octeon_config.h | 2 ++ .../net/ethernet/cavium/liquidio/octeon_iq.h | 3 +++ .../net/ethernet/cavium/liquidio/octeon_main.h | 2 +- .../net/ethernet/cavium/liquidio/octeon_nic.c | 5 +++-- .../net/ethernet/cavium/liquidio/octeon_nic.h | 3 ++- .../ethernet/cavium/liquidio/request_manager.c | 18 ++++++++++++++++-- 9 files changed, 55 insertions(+), 19 deletions(-) diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c index 23f6b60030c5..b891d858e416 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_core.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c @@ -91,7 +91,7 @@ void octeon_update_tx_completion_counters(void *buf, int reqtype, *bytes_compl += skb->len; } -void octeon_report_sent_bytes_to_bql(void *buf, int reqtype) +int octeon_report_sent_bytes_to_bql(void *buf, int reqtype) { struct octnet_buf_free_info *finfo; struct sk_buff *skb; @@ -112,11 +112,13 @@ void octeon_report_sent_bytes_to_bql(void *buf, int reqtype) break; default: - return; + return 0; } txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb)); netdev_tx_sent_queue(txq, skb->len); + + return netif_xmit_stopped(txq); } void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr) diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index accd038f3f34..8ea24d68e824 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -2594,7 +2594,8 @@ static void handle_timestamp(struct octeon_device *oct, */ static inline int send_nic_timestamp_pkt(struct octeon_device *oct, struct octnic_data_pkt *ndata, - struct octnet_buf_free_info *finfo) + struct octnet_buf_free_info *finfo, + int xmit_more) { int retval; struct octeon_soft_command *sc; @@ -2629,7 +2630,7 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct, len = (u32)((struct octeon_instr_ih2 *) (&sc->cmd.cmd2.ih2))->dlengsz; - ring_doorbell = 1; + ring_doorbell = !xmit_more; retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, sc, len, ndata->reqtype); @@ -2663,7 +2664,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) union tx_info *tx_info; int status = 0; int q_idx = 0, iq_no = 0; - int j; + int j, xmit_more = 0; u64 dptr = 0; u32 tag = 0; @@ -2868,17 +2869,19 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) irh->vlan = skb_vlan_tag_get(skb) & 0xfff; } + xmit_more = skb->xmit_more; + if (unlikely(cmdsetup.s.timestamp)) - status = send_nic_timestamp_pkt(oct, &ndata, finfo); + status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); else - status = octnet_send_nic_data_pkt(oct, &ndata); + status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); if (status == IQ_SEND_FAILED) goto lio_xmit_failed; netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); if (status == IQ_SEND_STOP) - stop_q(lio->netdev, q_idx); + stop_q(netdev, q_idx); netif_trans_update(netdev); @@ -2897,6 +2900,9 @@ lio_xmit_failed: if (dptr) dma_unmap_single(&oct->pci_dev->dev, dptr, ndata.datasize, DMA_TO_DEVICE); + + octeon_ring_doorbell_locked(oct, iq_no); + tx_buffer_free(skb); return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c index 4c3b5688529b..00c19306ecee 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c @@ -1690,7 +1690,8 @@ static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf) */ static int send_nic_timestamp_pkt(struct octeon_device *oct, struct octnic_data_pkt *ndata, - struct octnet_buf_free_info *finfo) + struct octnet_buf_free_info *finfo, + int xmit_more) { struct octeon_soft_command *sc; int ring_doorbell; @@ -1720,7 +1721,7 @@ static int send_nic_timestamp_pkt(struct octeon_device *oct, len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz; - ring_doorbell = 1; + ring_doorbell = !xmit_more; retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, sc, len, ndata->reqtype); @@ -1752,6 +1753,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) struct octeon_device *oct; int q_idx = 0, iq_no = 0; union tx_info *tx_info; + int xmit_more = 0; struct lio *lio; int status = 0; u64 dptr = 0; @@ -1940,10 +1942,12 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK; } + xmit_more = skb->xmit_more; + if (unlikely(cmdsetup.s.timestamp)) - status = send_nic_timestamp_pkt(oct, &ndata, finfo); + status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); else - status = octnet_send_nic_data_pkt(oct, &ndata); + status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); if (status == IQ_SEND_FAILED) goto lio_xmit_failed; @@ -1952,7 +1956,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) if (status == IQ_SEND_STOP) { dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n", iq_no); - stop_q(lio->netdev, q_idx); + stop_q(netdev, q_idx); } netif_trans_update(netdev); @@ -1972,6 +1976,9 @@ lio_xmit_failed: if (dptr) dma_unmap_single(&oct->pci_dev->dev, dptr, ndata.datasize, DMA_TO_DEVICE); + + octeon_ring_doorbell_locked(oct, iq_no); + tx_buffer_free(skb); return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h index 63bd9c94e547..ceac74388e09 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h +++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h @@ -37,6 +37,8 @@ #define MAX_OCTEON_LINKS MAX_OCTEON_NICIF #define MAX_OCTEON_MULTICAST_ADDR 32 +#define MAX_OCTEON_FILL_COUNT 8 + /* CN6xxx IQ configuration macros */ #define CN6XXX_MAX_INPUT_QUEUES 32 #define CN6XXX_MAX_IQ_DESCRIPTORS 2048 diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h index 5c3c8da976f7..81c987682941 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h +++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h @@ -343,6 +343,9 @@ int octeon_delete_instr_queue(struct octeon_device *octeon_dev, u32 iq_no); int lio_wait_for_instr_fetch(struct octeon_device *oct); +void +octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no); + int octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype, void (*fn)(void *)); diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h index 32ef3a7d88d8..c846eec11a45 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h +++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h @@ -63,7 +63,7 @@ struct octnet_buf_free_info { }; /* BQL-related functions */ -void octeon_report_sent_bytes_to_bql(void *buf, int reqtype); +int octeon_report_sent_bytes_to_bql(void *buf, int reqtype); void octeon_update_tx_completion_counters(void *buf, int reqtype, unsigned int *pkts_compl, unsigned int *bytes_compl); diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c index b457cf23fce6..150609bd8849 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c +++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c @@ -82,9 +82,10 @@ octeon_alloc_soft_command_resp(struct octeon_device *oct, } int octnet_send_nic_data_pkt(struct octeon_device *oct, - struct octnic_data_pkt *ndata) + struct octnic_data_pkt *ndata, + int xmit_more) { - int ring_doorbell = 1; + int ring_doorbell = !xmit_more; return octeon_send_command(oct, ndata->q_no, ring_doorbell, &ndata->cmd, ndata->buf, ndata->datasize, diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h index 6480ef863441..de4130d26a98 100644 --- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h +++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h @@ -279,7 +279,8 @@ octeon_alloc_soft_command_resp(struct octeon_device *oct, * queue should be stopped, and IQ_SEND_OK if it sent okay. */ int octnet_send_nic_data_pkt(struct octeon_device *oct, - struct octnic_data_pkt *ndata); + struct octnic_data_pkt *ndata, + int xmit_more); /** Send a NIC control packet to the device * @param oct - octeon device pointer diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c index 1e0fbce86d60..a10459742ae4 100644 --- a/drivers/net/ethernet/cavium/liquidio/request_manager.c +++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c @@ -278,6 +278,18 @@ ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq) } } +void +octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no) +{ + struct octeon_instr_queue *iq; + + iq = oct->instr_queue[iq_no]; + spin_lock(&iq->post_lock); + if (iq->fill_cnt) + ring_doorbell(oct, iq); + spin_unlock(&iq->post_lock); +} + static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq, u8 *cmd) { @@ -543,6 +555,7 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no, u32 force_db, void *cmd, void *buf, u32 datasize, u32 reqtype) { + int xmit_stopped; struct iq_post_status st; struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; @@ -554,12 +567,13 @@ octeon_send_command(struct octeon_device *oct, u32 iq_no, st = __post_command2(iq, cmd); if (st.status != IQ_SEND_FAILED) { - octeon_report_sent_bytes_to_bql(buf, reqtype); + xmit_stopped = octeon_report_sent_bytes_to_bql(buf, reqtype); __add_to_request_list(iq, st.index, buf, reqtype); INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); - if (force_db) + if (iq->fill_cnt >= MAX_OCTEON_FILL_COUNT || force_db || + xmit_stopped || st.status == IQ_SEND_STOP) ring_doorbell(oct, iq); } else { INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); -- 2.30.2