Discussion:
[dpdk-dev] [PATCH 0/5] interrupt mode for fm10k
(too old to reply)
Shaopeng He
2015-09-25 05:37:00 UTC
Permalink
This patch series contains four major parts.

1. implement rx_descriptor_done function in fm10k
2. make sure default VID available in dev_init in fm10k
3. fix a memory leak for non-ip packet in l3fwd-power
4. add rx interrupt support in fm10k PF and VF

The patch set is developed based on one previous patch set
"[PATCH v1 00/11] interrupt mode for i40e"
http://www.dpdk.org/ml/archives/dev/2015-September/023903.html

Shaopeng He (5):
fm10k: implement rx_descriptor_done function
fm10k: enable Rx queue interrupts for PF and VF
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc: release note update for fm10k intr mode

doc/guides/rel_notes/release_2_2.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 163 +++++++++++++++++++++++++++++++++--
drivers/net/fm10k/fm10k_rxtx.c | 25 ++++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 188 insertions(+), 8 deletions(-)
--
1.9.3
Shaopeng He
2015-09-25 05:37:01 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index c089882..4fc5e8e 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -311,6 +311,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
#endif
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a69c990..a82cd59 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2049,6 +2049,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index d3f7b89..5c56fc2 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -354,6 +354,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Shaopeng He
2015-09-25 05:37:02 UTC
Permalink
The patch does below things for fm10k PF and VF:
- Setup NIC to generate MSI-X interrupts
- Set the RXINT register to map interrupt causes to vectors
- Implement interrupt enable/disable functions

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 147 +++++++++++++++++++++++++++++++++++++--
1 file changed, 140 insertions(+), 7 deletions(-)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a82cd59..6648934 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -52,6 +52,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -67,6 +69,8 @@ static void
fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
static void fm10k_tx_queue_release(void *queue);
static void fm10k_rx_queue_release(void *queue);
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -406,6 +410,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -413,10 +418,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -741,6 +759,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -804,6 +825,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -815,6 +838,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}

static void
@@ -1778,6 +1821,94 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs interrupt for mailbox
+ * so igb_uio is not supported for rx interrupt
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 + RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2050,6 +2181,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
@@ -2060,7 +2193,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2177,7 +2310,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2185,7 +2318,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2291,7 +2423,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Chen, Jing D
2015-10-22 06:52:28 UTC
Permalink
Hi,

Best Regards,
Mark
-----Original Message-----
Sent: Friday, September 25, 2015 1:37 PM
Subject: [dpdk-dev] [PATCH 2/5] fm10k: enable Rx queue interrupts for PF
and VF
- Setup NIC to generate MSI-X interrupts
- Set the RXINT register to map interrupt causes to vectors
- Implement interrupt enable/disable functions
The description is too simple, can you help to extend?
Besides that, there are complicated changes in this patch.
Can you help you split it to several smaller ones for better understanding?
---
drivers/net/fm10k/fm10k_ethdev.c | 147
+++++++++++++++++++++++++++++++++++++--
1 file changed, 140 insertions(+), 7 deletions(-)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c
b/drivers/net/fm10k/fm10k_ethdev.c
index a82cd59..6648934 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
In previous enable function, you'll use rte_intr_enable() to enable interrupt, but
You needn't disable it in this function?
+ return 0;
+}
+
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs interrupt for mailbox
+ * so igb_uio is not supported for rx interrupt
+ */
I guess you'll support both igb_uio and VFIO, RX interrupt mode only enabled in case VFIO is used.
I suggest you add more comments here for better understanding.
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = RX_VEC_START;
+ queue_id < dev->data-
Post by Shaopeng He
nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 +
RX_VEC_START)
+ vec++;
No "else" to handle exceptional case?
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d
rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2050,6 +2181,8 @@ static const struct eth_dev_ops
fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
@@ -2060,7 +2193,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;
PMD_INIT_FUNC_TRACE();
@@ -2177,7 +2310,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}
- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));
hw->mac.ops.update_int_moderator(hw);
@@ -2185,7 +2318,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;
for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2291,7 +2423,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
He, Shaopeng
2015-10-23 07:53:51 UTC
Permalink
Hi, Mark

Thanks for the comments, please find my reply inline.

Thanks,
--Shaopeng
-----Original Message-----
From: Chen, Jing D
Sent: Thursday, October 22, 2015 2:52 PM
Subject: RE: [dpdk-dev] [PATCH 2/5] fm10k: enable Rx queue interrupts for
PF and VF
Hi,
Best Regards,
Mark
-----Original Message-----
Sent: Friday, September 25, 2015 1:37 PM
Subject: [dpdk-dev] [PATCH 2/5] fm10k: enable Rx queue interrupts for
PF and VF
- Setup NIC to generate MSI-X interrupts
- Set the RXINT register to map interrupt causes to vectors
- Implement interrupt enable/disable functions
The description is too simple, can you help to extend?
Besides that, there are complicated changes in this patch.
Can you help you split it to several smaller ones for better understanding?
Sure, I will send v2 to split into smaller ones.
---
drivers/net/fm10k/fm10k_ethdev.c | 147
+++++++++++++++++++++++++++++++++++++--
1 file changed, 140 insertions(+), 7 deletions(-)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c
b/drivers/net/fm10k/fm10k_ethdev.c
index a82cd59..6648934 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
In previous enable function, you'll use rte_intr_enable() to enable interrupt,
but You needn't disable it in this function?
rte_intr_enable() is needed in fm10k_dev_rx_queue_intr_enable to re-enable
the interrupt in port level. To disable individual queue interrupt, currently only
need to disable the hardware side by register setting.
+ return 0;
+}
+
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) {
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs interrupt for mailbox
+ * so igb_uio is not supported for rx interrupt
+ */
I guess you'll support both igb_uio and VFIO, RX interrupt mode only enabled
in case VFIO is used.
I suggest you add more comments here for better understanding.
You are right, current igb_uio only has one interrupt vector, and mailbox
needs that, so igb_uio cannot support RX interrupt mode. I will add more
comments here in next version.
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = RX_VEC_START;
+ queue_id < dev->data-
Post by Shaopeng He
nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 +
RX_VEC_START)
+ vec++;
No "else" to handle exceptional case?
For the "else" case, when all available efd (event fd) are used, currently we
just reuse the last efd, keep the same value for "vec", so no "else" is needed.
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d
rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) {
struct fm10k_fault fault;
@@ -2050,6 +2181,8 @@ static const struct eth_dev_ops
fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
@@ -2060,7 +2193,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev) {
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
Post by Shaopeng He
dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;
PMD_INIT_FUNC_TRACE();
@@ -2177,7 +2310,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}
- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));
hw->mac.ops.update_int_moderator(hw);
@@ -2185,7 +2318,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;
for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2291,7 +2423,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Shaopeng He
2015-09-25 05:37:03 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 6648934..d5e18bc 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2354,6 +2354,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true);
--
1.9.3
Shaopeng He
2015-09-25 05:37:04 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

Signed-off-by: Shaopeng He <***@intel.com>
---
examples/l3fwd-power/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 8bb88ce..ecdff73 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
De Lara Guarch, Pablo
2015-09-25 07:19:24 UTC
Permalink
Hi Shaopeng,
-----Original Message-----
Sent: Friday, September 25, 2015 6:37 AM
Subject: [dpdk-dev] [PATCH 4/5] l3fwd-power: fix a memory leak for non-ip
packet
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.
Is this patch related to the rest of the patchset? Something where fm10k is involved?
Because it does not look like.

Thanks,
Pablo
---
examples/l3fwd-power/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 8bb88ce..ecdff73 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);
}
--
1.9.3
He, Shaopeng
2015-09-25 09:00:33 UTC
Permalink
Hi, Pablo
-----Original Message-----
From: De Lara Guarch, Pablo
Sent: Friday, September 25, 2015 3:19 PM
Subject: RE: [dpdk-dev] [PATCH 4/5] l3fwd-power: fix a memory leak for
non-ip packet
Hi Shaopeng,
-----Original Message-----
Sent: Friday, September 25, 2015 6:37 AM
Subject: [dpdk-dev] [PATCH 4/5] l3fwd-power: fix a memory leak for
non-ip packet
Previous l3fwd-power only processes IP and IPv6 packet, other packet's
mbuf is not released, and causes a memory leak.
This patch fixes this issue.
Is this patch related to the rest of the patchset? Something where fm10k is involved?
Because it does not look like.
Thanks for the comments.

This bug almost happens every time with fm10k when testing this interrupt mode.
But for other NIC cards, it rarely occurs. It is because of that right now only fm10k
has the switch core, by that, other types of packets sneak into the testing.
I include this bug-fixing in the patch set, hope that people can test this feature
more easily.

Thanks,
--Shaopeng
Thanks,
Pablo
---
examples/l3fwd-power/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-
power/main.c
index 8bb88ce..ecdff73 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);
}
--
1.9.3
Shaopeng He
2015-09-25 05:37:05 UTC
Permalink
Signed-off-by: Shaopeng He <***@intel.com>
---
doc/guides/rel_notes/release_2_2.rst | 2 ++
1 file changed, 2 insertions(+)

diff --git a/doc/guides/rel_notes/release_2_2.rst b/doc/guides/rel_notes/release_2_2.rst
index 73dba47..44b3aea 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -5,11 +5,13 @@ New Features
------------

* Support interrupt mode on i40e
+* Support interrupt mode on fm10k

Resolved Issues
---------------

* Fix ixgbe/igb rx interrupt compatible issue with mbox
+* Fix l3fwd-power memory leak for non-ip packet

Known Issues
------------
--
1.9.3
Shaopeng He
2015-10-26 03:47:38 UTC
Permalink
This patch series adds interrupt mode support for fm10k,
contains four major parts:

1. implement rx_descriptor_done function in fm10k
2. make sure default VID available in dev_init in fm10k
3. fix a memory leak for non-ip packet in l3fwd-power
4. add rx interrupt support in fm10k PF and VF

The patch set is developed based on one previous patch set
"[PATCH v1 00/11] interrupt mode for i40e"
http://www.dpdk.org/ml/archives/dev/2015-September/023903.html

Shaopeng He (7):
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc: release note update for fm10k intr mode

doc/guides/rel_notes/release_2_2.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 164 +++++++++++++++++++++++++++++++++--
drivers/net/fm10k/fm10k_rxtx.c | 25 ++++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 189 insertions(+), 8 deletions(-)
--
1.9.3
Shaopeng He
2015-10-26 03:47:39 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index c089882..4fc5e8e 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -311,6 +311,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
#endif
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a69c990..a82cd59 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2049,6 +2049,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index d3f7b89..5c56fc2 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -354,6 +354,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Shaopeng He
2015-10-26 03:47:40 UTC
Permalink
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 93 +++++++++++++++++++++++++++++++++++++---
1 file changed, 86 insertions(+), 7 deletions(-)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a82cd59..68ae1ba 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -52,6 +52,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -67,6 +69,8 @@ static void
fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
static void fm10k_tx_queue_release(void *queue);
static void fm10k_rx_queue_release(void *queue);
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -406,6 +410,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -413,10 +418,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -741,6 +759,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -1778,6 +1799,64 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 + RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2060,7 +2139,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2177,7 +2256,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2185,7 +2264,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2291,7 +2369,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Shaopeng He
2015-10-26 03:47:43 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index ce4d916..4823816 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2355,6 +2355,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true);
--
1.9.3
Shaopeng He
2015-10-26 03:47:41 UTC
Permalink
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 68ae1ba..45571f2 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -825,6 +825,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -836,6 +838,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}

static void
--
1.9.3
Shaopeng He
2015-10-26 03:47:42 UTC
Permalink
Interrupt mode framework has enable/disable functions for individual
rx queue, this patch implements these two functions.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 45571f2..ce4d916 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1821,6 +1821,37 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -2151,6 +2182,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
--
1.9.3
Shaopeng He
2015-10-26 03:47:44 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

Signed-off-by: Shaopeng He <***@intel.com>
---
examples/l3fwd-power/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 8bb88ce..ecdff73 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
Shaopeng He
2015-10-26 03:47:45 UTC
Permalink
Signed-off-by: Shaopeng He <***@intel.com>
---
doc/guides/rel_notes/release_2_2.rst | 2 ++
1 file changed, 2 insertions(+)

diff --git a/doc/guides/rel_notes/release_2_2.rst b/doc/guides/rel_notes/release_2_2.rst
index 73dba47..44b3aea 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -5,11 +5,13 @@ New Features
------------

* Support interrupt mode on i40e
+* Support interrupt mode on fm10k

Resolved Issues
---------------

* Fix ixgbe/igb rx interrupt compatible issue with mbox
+* Fix l3fwd-power memory leak for non-ip packet

Known Issues
------------
--
1.9.3
Qiu, Michael
2015-12-22 07:00:15 UTC
Permalink
Post by Shaopeng He
---
doc/guides/rel_notes/release_2_2.rst | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/guides/rel_notes/release_2_2.rst b/doc/guides/rel_notes/release_2_2.rst
index 73dba47..44b3aea 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -5,11 +5,13 @@ New Features
------------
* Support interrupt mode on i40e
+* Support interrupt mode on fm10k
Do we need to declare the limitation like only VFIO support for RRC, and
other limitations like could not work on the management port for
BR(maybe I'm wrong, but need BAR4 to run Testpoint)
Post by Shaopeng He
Resolved Issues
---------------
* Fix ixgbe/igb rx interrupt compatible issue with mbox
+* Fix l3fwd-power memory leak for non-ip packet
Known Issues
------------
He, Shaopeng
2015-12-22 12:07:25 UTC
Permalink
-----Original Message-----
From: Qiu, Michael
Sent: Tuesday, December 22, 2015 3:00 PM
Cc: Chen, Jing D
Subject: Re: [PATCH v2 7/7] doc: release note update for fm10k intr mode
Post by Shaopeng He
---
doc/guides/rel_notes/release_2_2.rst | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/guides/rel_notes/release_2_2.rst
b/doc/guides/rel_notes/release_2_2.rst
index 73dba47..44b3aea 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -5,11 +5,13 @@ New Features
------------
* Support interrupt mode on i40e
+* Support interrupt mode on fm10k
Do we need to declare the limitation like only VFIO support for RRC, and
other limitations like could not work on the management port for BR(maybe
I'm wrong, but need BAR4 to run Testpoint)
As we discussed, we will update limitations together in nics/fm10k.rst.
Post by Shaopeng He
Resolved Issues
---------------
* Fix ixgbe/igb rx interrupt compatible issue with mbox
+* Fix l3fwd-power memory leak for non-ip packet
Known Issues
------------
Chen, Jing D
2015-10-28 07:34:42 UTC
Permalink
-----Original Message-----
From: He, Shaopeng
Sent: Monday, October 26, 2015 11:48 AM
Cc: Chen, Jing D; Qiu, Michael; He, Shaopeng
Subject: [PATCH v2 0/7] interrupt mode for fm10k
This patch series adds interrupt mode support for fm10k,
1. implement rx_descriptor_done function in fm10k
2. make sure default VID available in dev_init in fm10k
3. fix a memory leak for non-ip packet in l3fwd-power
4. add rx interrupt support in fm10k PF and VF
The patch set is developed based on one previous patch set
"[PATCH v1 00/11] interrupt mode for i40e"
http://www.dpdk.org/ml/archives/dev/2015-September/023903.html
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc: release note update for fm10k intr mode
doc/guides/rel_notes/release_2_2.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 164
+++++++++++++++++++++++++++++++++--
drivers/net/fm10k/fm10k_rxtx.c | 25 ++++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 189 insertions(+), 8 deletions(-)
--
1.9.3
Acked-by : Jing Chen <***@intel.com>
He, Shaopeng
2015-10-30 10:40:41 UTC
Permalink
Hi, Thomas

The previous patch set which this one is based on, has a v2 sent today as
"[PATCH v2 00/11] interrupt mode for i40e"
http://dpdk.org/ml/archives/dev/2015-October/026710.html.

I have verified that will not affect this patch set. Because this patch set only depends
on the framework part change in that patch set, most likely, later version of that patch
set (if have any) will not affect this patch set too.

Could you please help to apply this patch set after you merge the final version of
" interrupt mode for i40e"?

Thank you very much!
--Shaopeng
-----Original Message-----
From: Chen, Jing D
Sent: Wednesday, October 28, 2015 3:35 PM
Cc: Qiu, Michael
Subject: RE: [PATCH v2 0/7] interrupt mode for fm10k
-----Original Message-----
From: He, Shaopeng
Sent: Monday, October 26, 2015 11:48 AM
Cc: Chen, Jing D; Qiu, Michael; He, Shaopeng
Subject: [PATCH v2 0/7] interrupt mode for fm10k
This patch series adds interrupt mode support for fm10k,
1. implement rx_descriptor_done function in fm10k
2. make sure default VID available in dev_init in fm10k
3. fix a memory leak for non-ip packet in l3fwd-power
4. add rx interrupt support in fm10k PF and VF
The patch set is developed based on one previous patch set
"[PATCH v1 00/11] interrupt mode for i40e"
http://www.dpdk.org/ml/archives/dev/2015-September/023903.html
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc: release note update for fm10k intr mode
doc/guides/rel_notes/release_2_2.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 164
+++++++++++++++++++++++++++++++++--
drivers/net/fm10k/fm10k_rxtx.c | 25 ++++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 189 insertions(+), 8 deletions(-)
--
1.9.3
Thomas Monjalon
2015-10-30 13:54:27 UTC
Permalink
Hi,

The problem is that i40e interrupt patches are not merge-able.
They are not enough reviewed especially for igb/ixgbe and vfio changes.
Post by He, Shaopeng
Hi, Thomas
The previous patch set which this one is based on, has a v2 sent today as
"[PATCH v2 00/11] interrupt mode for i40e"
http://dpdk.org/ml/archives/dev/2015-October/026710.html.
I have verified that will not affect this patch set. Because this patch set only depends
on the framework part change in that patch set, most likely, later version of that patch
set (if have any) will not affect this patch set too.
Could you please help to apply this patch set after you merge the final version of
" interrupt mode for i40e"?
Thank you very much!
--Shaopeng
He, Shaopeng
2015-10-31 03:06:05 UTC
Permalink
Hi, Thomas
-----Original Message-----
Sent: Friday, October 30, 2015 9:54 PM
To: He, Shaopeng
Subject: Re: [PATCH v2 0/7] interrupt mode for fm10k
Hi,
The problem is that i40e interrupt patches are not merge-able.
They are not enough reviewed especially for igb/ixgbe and vfio changes.
Thanks for your time, understood.

Best regards,
--Shaopeng
Post by He, Shaopeng
Hi, Thomas
The previous patch set which this one is based on, has a v2 sent today
as "[PATCH v2 00/11] interrupt mode for i40e"
http://dpdk.org/ml/archives/dev/2015-October/026710.html.
I have verified that will not affect this patch set. Because this
patch set only depends on the framework part change in that patch set,
most likely, later version of that patch set (if have any) will not affect this
patch set too.
Post by He, Shaopeng
Could you please help to apply this patch set after you merge the
final version of " interrupt mode for i40e"?
Thank you very much!
--Shaopeng
Thomas Monjalon
2015-11-04 14:52:03 UTC
Permalink
Post by He, Shaopeng
Post by Thomas Monjalon
The problem is that i40e interrupt patches are not merge-able.
They are not enough reviewed especially for igb/ixgbe and vfio changes.
Thanks for your time, understood.
Now that i40e interrupt is integrated, it would be helpful to have a v3
rebased please.
Shaopeng He
2015-11-05 05:12:02 UTC
Permalink
This patch series adds interrupt mode support for fm10k,
contains four major parts:

1. implement rx_descriptor_done function in fm10k
2. make sure default VID available in dev_init in fm10k
3. fix a memory leak for non-ip packet in l3fwd-power
4. add rx interrupt support in fm10k PF and VF

Changes in v3:
- Rebase to latest code

Changes in v2:
- Reword some comments and commit messages
- Split one big patch into three smaller ones

Shaopeng He (7):
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc: release note update for fm10k intr mode

doc/guides/rel_notes/release_2_2.rst | 7 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 172 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 201 insertions(+), 12 deletions(-)
--
1.9.3
Shaopeng He
2015-11-05 05:12:03 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index 754aa6a..65002c9 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -343,6 +343,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index cf7ada7..9a32cf8 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2416,6 +2416,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index 1bac28d..805bd7d 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -354,6 +354,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Shaopeng He
2015-11-05 05:12:04 UTC
Permalink
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 ++
drivers/net/fm10k/fm10k_ethdev.c | 101 ++++++++++++++++++++++++++++++++++-----
2 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index 65002c9..6729f7f 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4

+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
struct fm10k_macvlan_filter_info {
uint16_t vlan_num; /* Total VLAN number */
uint16_t mac_num; /* Total mac number */
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 9a32cf8..d5cb233 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
ETH_TXQ_FLAGS_NOOFFLOADS)
@@ -112,6 +114,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {

#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -632,6 +636,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -639,10 +644,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -998,6 +1016,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -2053,7 +2074,7 @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2084,7 +2105,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2115,7 +2136,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2132,7 +2153,7 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2143,6 +2164,64 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 + FM10K_RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2502,7 +2581,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2608,7 +2687,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2616,7 +2695,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2723,7 +2801,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Shaopeng He
2015-11-05 05:12:05 UTC
Permalink
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index d5cb233..4a973e8 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1070,6 +1070,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -1081,6 +1083,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}

static void
--
1.9.3
Shaopeng He
2015-11-05 05:12:06 UTC
Permalink
Interrupt mode framework has enable/disable functions for individual
rx queue, this patch implements these two functions.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 4a973e8..9b210d7 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2186,6 +2186,37 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -2518,6 +2549,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
--
1.9.3
Shaopeng He
2015-11-05 05:12:07 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 9b210d7..2cb0b94 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2786,6 +2786,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true,
MAIN_VSI_POOL_NUMBER);
--
1.9.3
Shaopeng He
2015-11-05 05:12:08 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
examples/l3fwd-power/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 9175989..9c76d9e 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
Shaopeng He
2015-11-05 05:12:09 UTC
Permalink
Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
doc/guides/rel_notes/release_2_2.rst | 7 +++++++
1 file changed, 7 insertions(+)

diff --git a/doc/guides/rel_notes/release_2_2.rst b/doc/guides/rel_notes/release_2_2.rst
index 59dda59..c6d255e 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -65,6 +65,8 @@ New Features

* **Added fm10k VMDQ support.**

+* **Added fm10k Rx interrupt support.**
+
* **New NIC Boulder Rapid support.**

Boulder Rapid is a new NIC of Intel's fm10k family.
@@ -187,6 +189,11 @@ Libraries
Examples
~~~~~~~~

+* **l3fwd-power: Fixed memory leak for non-ip packet.**
+
+ Fixed issue in l3fwd-power where, recieving other packet than
+ types of IP and IPv6, the mbuf was not released, and caused
+ a memory leak.

Other
~~~~~
--
1.9.3
Shaopeng He
2015-12-21 10:19:46 UTC
Permalink
This patch series adds interrupt mode support for fm10k,
contains four major parts:

1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.

Changes in v4:
- Rebase to latest code
- Update release 2.3 note in each patch

Changes in v3:
- Rebase to latest code

Changes in v2:
- Reword some comments and commit messages
- Split one big patch into three smaller ones

Shaopeng He (6):
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet

doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 172 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
5 files changed, 202 insertions(+), 12 deletions(-)
--
1.9.3
Shaopeng He
2015-12-21 10:19:47 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index cd38af2..e2f677a 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -345,6 +345,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index e4aed94..d39c33b 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2435,6 +2435,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index e958865..36d3002 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -369,6 +369,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Qiu, Michael
2015-12-22 06:50:54 UTC
Permalink
Post by Shaopeng He
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index cd38af2..e2f677a 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -345,6 +345,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index e4aed94..d39c33b 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2435,6 +2435,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index e958865..36d3002 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -369,6 +369,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}
+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
Sorry, here makes my confuse: offset for RX queue id?
Post by Shaopeng He
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
He, Shaopeng
2015-12-22 11:55:29 UTC
Permalink
-----Original Message-----
From: Qiu, Michael
Sent: Tuesday, December 22, 2015 2:51 PM
Subject: Re: [dpdk-dev] [PATCH v4 1/6] fm10k: implement
rx_descriptor_done function
Post by Shaopeng He
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend, then
l3fwd-power will adjust the cpu frequency according to the result.
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index cd38af2..e2f677a 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -345,6 +345,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct
rte_mbuf **rx_pkts, uint16_t fm10k_recv_scattered_pkts(void
*rx_queue,
Post by Shaopeng He
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
diff --git a/drivers/net/fm10k/fm10k_ethdev.c
b/drivers/net/fm10k/fm10k_ethdev.c
index e4aed94..d39c33b 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2435,6 +2435,7 @@ static const struct eth_dev_ops
fm10k_eth_dev_ops = {
Post by Shaopeng He
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c
b/drivers/net/fm10k/fm10k_rxtx.c index e958865..36d3002 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -369,6 +369,31 @@ fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts,
Post by Shaopeng He
return nb_rcv;
}
+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset) {
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
Sorry, here makes my confuse: offset for RX queue id?
Good catch, will fix it in next version
Post by Shaopeng He
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q) {
uint16_t next_rs, count = 0;
Shaopeng He
2015-12-21 10:19:48 UTC
Permalink
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
doc/guides/rel_notes/release_2_3.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 ++
drivers/net/fm10k/fm10k_ethdev.c | 101 +++++++++++++++++++++++++++++++----
3 files changed, 95 insertions(+), 11 deletions(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..2cb5ebd 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,8 @@ DPDK Release 2.3
New Features
------------

+* **Added fm10k Rx interrupt support.**
+

Resolved Issues
---------------
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index e2f677a..770d6ba 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4

+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
ETH_TXQ_FLAGS_NOOFFLOADS)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index d39c33b..a34c5e2 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -109,6 +111,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {

#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -687,6 +691,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -694,10 +699,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -1053,6 +1071,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -2072,7 +2093,7 @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2103,7 +2124,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2134,7 +2155,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2151,7 +2172,7 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2162,6 +2183,64 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 + FM10K_RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2531,7 +2610,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2637,7 +2716,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2645,7 +2724,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2752,7 +2830,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Qiu, Michael
2015-12-22 07:28:09 UTC
Permalink
Post by Shaopeng He
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.
---
doc/guides/rel_notes/release_2_3.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 ++
drivers/net/fm10k/fm10k_ethdev.c | 101 +++++++++++++++++++++++++++++++----
3 files changed, 95 insertions(+), 11 deletions(-)
diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..2cb5ebd 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,8 @@ DPDK Release 2.3
New Features
------------
+* **Added fm10k Rx interrupt support.**
+
Resolved Issues
---------------
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index e2f677a..770d6ba 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4
+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
ETH_TXQ_FLAGS_NOOFFLOADS)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index d39c33b..a34c5e2 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -109,6 +111,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -687,6 +691,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -694,10 +699,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;
- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);
/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -1053,6 +1071,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}
+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -2072,7 +2093,7 @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2103,7 +2124,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2134,7 +2155,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2151,7 +2172,7 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2162,6 +2183,64 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}
static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 + FM10K_RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
Here if failure, should we call rte_intr_efd_disable()?

Thanks,
Michael
Post by Shaopeng He
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2531,7 +2610,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;
PMD_INIT_FUNC_TRACE();
@@ -2637,7 +2716,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}
- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));
hw->mac.ops.update_int_moderator(hw);
@@ -2645,7 +2724,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;
for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2752,7 +2830,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
He, Shaopeng
2015-12-23 01:57:53 UTC
Permalink
-----Original Message-----
From: Qiu, Michael
Sent: Tuesday, December 22, 2015 3:28 PM
Subject: Re: [dpdk-dev] [PATCH v4 2/6] fm10k: setup rx queue interrupts for
PF and VF
Post by Shaopeng He
In interrupt mode, each rx queue can have one interrupt to notify the
up layer application when packets are available in that queue. Some
queues also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only
those drivers which support multiple interrupt vectors e.g. vfio-pci
can work in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.
---
doc/guides/rel_notes/release_2_3.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 ++
drivers/net/fm10k/fm10k_ethdev.c | 101
+++++++++++++++++++++++++++++++----
Post by Shaopeng He
3 files changed, 95 insertions(+), 11 deletions(-)
diff --git a/doc/guides/rel_notes/release_2_3.rst
b/doc/guides/rel_notes/release_2_3.rst
index 99de186..2cb5ebd 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,8 @@ DPDK Release 2.3
New Features
------------
+* **Added fm10k Rx interrupt support.**
+
Resolved Issues
---------------
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index e2f677a..770d6ba 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4
+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
#define FM10K_SIMPLE_TX_FLAG
((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
Post by Shaopeng He
ETH_TXQ_FLAGS_NOOFFLOADS)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c
b/drivers/net/fm10k/fm10k_ethdev.c
index d39c33b..a34c5e2 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */ #define CHARS_PER_UINT32
(sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 <<
CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */ #define
+Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
static void fm10k_close_mbx_service(struct fm10k_hw *hw); static
109,6
=
Post by Shaopeng He
{
#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS +
FM10K_MAX_QUEUES_PF * \
Post by Shaopeng He
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
static void
int fm10k_dev_rx_init(struct rte_eth_dev *dev) {
struct fm10k_hw *hw =
FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -694,10 +699,23 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;
- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev,
i));
Post by Shaopeng He
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw,
FM10K_ITR(Q2V(dev, i)),
Post by Shaopeng He
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
Post by Shaopeng He
+ else
+ FM10K_WRITE_REG(hw,
FM10K_VFITR(Q2V(dev, i)),
Post by Shaopeng He
+ FM10K_ITR_AUTOMASK |
FM10K_ITR_MASK_CLEAR);
Post by Shaopeng He
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);
/* Setup RX queues */
@@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}
+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
+2093,7
Post by Shaopeng He
@@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox),
int_map);
Post by Shaopeng He
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault),
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox),
int_map);
Post by Shaopeng He
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault),
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
/* Only INT 0 available, other 15 are reserved. */
fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;
/* Only INT 0 available, other 15 are reserved. */
+2183,64
Post by Shaopeng He
@@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev) }
static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) {
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data-
nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1 +
FM10K_RX_VEC_START)
Post by Shaopeng He
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d
rx_queues"
Post by Shaopeng He
+ " intr_vec", dev->data->nb_rx_queues);
Here if failure, should we call rte_intr_efd_disable()?
Thanks for the comments, you are right, better to do more clean up here, will update in next version

Thanks,
--Shaopeng
Thanks,
Michael
Post by Shaopeng He
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) {
struct fm10k_fault fault;
@@ -2531,7 +2610,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev) {
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;
PMD_INIT_FUNC_TRACE();
@@ -2637,7 +2716,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}
- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));
hw->mac.ops.update_int_moderator(hw);
@@ -2645,7 +2724,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;
for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2752,7 +2830,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_DETACHABLE,
Post by Shaopeng He
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
RTE_PCI_DRV_INTR_LSC |
Post by Shaopeng He
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
Shaopeng He
2015-12-21 10:19:49 UTC
Permalink
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a34c5e2..b5b809c 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1125,6 +1125,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -1136,6 +1138,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}

static void
--
1.9.3
Qiu, Michael
2015-12-22 06:54:51 UTC
Permalink
Post by Shaopeng He
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.
---
drivers/net/fm10k/fm10k_ethdev.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index a34c5e2..b5b809c 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1125,6 +1125,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;
PMD_INIT_FUNC_TRACE();
@@ -1136,6 +1138,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
This line could be removed, because rte_free already do the check, see
below:
void rte_free(void *addr)
{
if (addr == NULL) return;
if (malloc_elem_free(malloc_elem_from_data(addr)) < 0)
rte_panic("Fatal error: Invalid memory\n");
}
Post by Shaopeng He
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}
static void
He, Shaopeng
2015-12-22 12:03:27 UTC
Permalink
-----Original Message-----
From: Qiu, Michael
Sent: Tuesday, December 22, 2015 2:55 PM
Subject: Re: [dpdk-dev] [PATCH v4 3/6] fm10k: remove rx queue interrupts
when dev stops
Post by Shaopeng He
Previous dev_stop function stops the rx/tx queues. This patch adds
logic to disable rx queue interrupt, clean the datapath event and
queue/vec map.
Post by Shaopeng He
---
drivers/net/fm10k/fm10k_ethdev.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/drivers/net/fm10k/fm10k_ethdev.c
b/drivers/net/fm10k/fm10k_ethdev.c
index a34c5e2..b5b809c 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1125,6 +1125,8 @@ fm10k_dev_start(struct rte_eth_dev *dev) static
void fm10k_dev_stop(struct rte_eth_dev *dev) {
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data-
dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;
PMD_INIT_FUNC_TRACE();
@@ -1136,6 +1138,26 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw,
FM10K_ITR(Q2V(dev, i)),
Post by Shaopeng He
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw,
FM10K_VFITR(Q2V(dev, i)),
Post by Shaopeng He
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ if (intr_handle->intr_vec != NULL) {
This line could be removed, because rte_free already do the check, see
void rte_free(void *addr)
{
if (addr == NULL) return;
if (malloc_elem_free(malloc_elem_from_data(addr)) < 0)
rte_panic("Fatal error: Invalid memory\n"); }
Yes, it could be removed. Thanks.
Post by Shaopeng He
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
+ }
}
static void
Shaopeng He
2015-12-21 10:19:50 UTC
Permalink
Interrupt mode framework has enable/disable functions for individual
rx queue, this patch implements these two functions.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index b5b809c..83e2f65 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2205,6 +2205,37 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -2537,6 +2568,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
--
1.9.3
Shaopeng He
2015-12-21 10:19:51 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 83e2f65..08d4ea9 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2815,6 +2815,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true,
MAIN_VSI_POOL_NUMBER);
--
1.9.3
Shaopeng He
2015-12-21 10:19:52 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
---
doc/guides/rel_notes/release_2_3.rst | 6 ++++++
examples/l3fwd-power/main.c | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 2cb5ebd..fc871ab 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -25,6 +25,12 @@ Libraries
Examples
~~~~~~~~

+* **l3fwd-power: Fixed memory leak for non-ip packet.**
+
+ Fixed issue in l3fwd-power where, recieving other packet than
+ types of IP and IPv6, the mbuf was not released, and caused
+ a memory leak.
+

Other
~~~~~
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 828c18a..d9cd848 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
Shaopeng He
2015-12-23 07:37:07 UTC
Permalink
This patch series adds interrupt mode support for fm10k,
contains four major parts:

1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.

v5 changes:
- remove one unnecessary NULL check for rte_free
- fix a wrong error message
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst

v4 changes:
- rebase to latest code
- update release 2.3 note in corresponding patches

v3 changes:
- rebase to latest code
- macro renaming according to the EAL change

v2 changes:
- reword some comments and commit messages
- split one big patch into three smaller ones

Shaopeng He (6):
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet

doc/guides/nics/fm10k.rst | 7 ++
doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 174 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
6 files changed, 211 insertions(+), 12 deletions(-)
--
1.9.3
Shaopeng He
2015-12-23 07:37:08 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

v5 change:
- fix a wrong error message

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index cd38af2..e2f677a 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -345,6 +345,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index e4aed94..d39c33b 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2435,6 +2435,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index e958865..0002f09 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -369,6 +369,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX descriptor offset %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Shaopeng He
2015-12-23 07:37:09 UTC
Permalink
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.

v5 changes:
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst

v4 change:
- update release note inside the patch

v3 change:
- macro renaming according to the EAL change

v2 changes:
- split one big patch into three smaller ones
- reword some comments and commit messages

Signed-off-by: Shaopeng He <***@intel.com>
---
doc/guides/nics/fm10k.rst | 7 +++
doc/guides/rel_notes/release_2_3.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 105 +++++++++++++++++++++++++++++++----
4 files changed, 106 insertions(+), 11 deletions(-)

diff --git a/doc/guides/nics/fm10k.rst b/doc/guides/nics/fm10k.rst
index 4206b7f..dc5cb6e 100644
--- a/doc/guides/nics/fm10k.rst
+++ b/doc/guides/nics/fm10k.rst
@@ -65,3 +65,10 @@ The FM10000 family of NICS support a maximum of a 15K jumbo frame. The value
is fixed and cannot be changed. So, even when the ``rxmode.max_rx_pkt_len``
member of ``struct rte_eth_conf`` is set to a value lower than 15364, frames
up to 15364 bytes can still reach the host interface.
+
+Interrupt mode
+~~~~~~~~~~~~~~~~~~~~~
+
+The FM10000 family of NICS need one separate interrupt for mailbox. So only
+drivers which support multiple interrupt vectors e.g. vfio-pci can work
+for fm10k interrupt mode.
diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..2cb5ebd 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,8 @@ DPDK Release 2.3
New Features
------------

+* **Added fm10k Rx interrupt support.**
+

Resolved Issues
---------------
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index e2f677a..770d6ba 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4

+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
ETH_TXQ_FLAGS_NOOFFLOADS)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index d39c33b..583335a 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -109,6 +111,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {

#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -687,6 +691,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -694,10 +699,25 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK |
+ FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK |
+ FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -1053,6 +1073,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -2072,7 +2095,7 @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2103,7 +2126,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2134,7 +2157,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2151,7 +2174,7 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2162,6 +2185,66 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1
+ + FM10K_RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ rte_intr_efd_disable(intr_handle);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2531,7 +2614,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2637,7 +2720,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2645,7 +2728,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2752,7 +2834,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Shaopeng He
2015-12-23 07:37:10 UTC
Permalink
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.

v5 changes:
- remove one unnecessary NULL check for rte_free

v2 changes:
- split one big patch into three smaller ones

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 583335a..da78389 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1127,6 +1127,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -1138,6 +1140,24 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
}

static void
--
1.9.3
Shaopeng He
2015-12-23 07:37:11 UTC
Permalink
Interrupt mode framework has enable/disable functions for individual
rx queue, this patch implements these two functions.

v2 changes:
- split one big patch into three smaller ones

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index da78389..06bfffd 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2205,6 +2205,37 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -2539,6 +2570,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
--
1.9.3
Shaopeng He
2015-12-23 07:37:12 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 06bfffd..832a3fe 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2817,6 +2817,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true,
MAIN_VSI_POOL_NUMBER);
--
1.9.3
Shaopeng He
2015-12-23 07:37:13 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

v4 change:
- update release note inside the patch

Signed-off-by: Shaopeng He <***@intel.com>
---
doc/guides/rel_notes/release_2_3.rst | 6 ++++++
examples/l3fwd-power/main.c | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 2cb5ebd..fc871ab 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -25,6 +25,12 @@ Libraries
Examples
~~~~~~~~

+* **l3fwd-power: Fixed memory leak for non-ip packet.**
+
+ Fixed issue in l3fwd-power where, recieving other packet than
+ types of IP and IPv6, the mbuf was not released, and caused
+ a memory leak.
+

Other
~~~~~
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 828c18a..d9cd848 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
Qiu, Michael
2015-12-24 08:04:11 UTC
Permalink
Post by Shaopeng He
This patch series adds interrupt mode support for fm10k,
1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.
- remove one unnecessary NULL check for rte_free
- fix a wrong error message
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst
- rebase to latest code
- update release 2.3 note in corresponding patches
- rebase to latest code
- macro renaming according to the EAL change
- reword some comments and commit messages
- split one big patch into three smaller ones
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc/guides/nics/fm10k.rst | 7 ++
doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 174 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
6 files changed, 211 insertions(+), 12 deletions(-)
Acked-by: Michael Qiu <***@intel.com>
Shaopeng He
2016-02-05 04:57:44 UTC
Permalink
This patch series adds interrupt mode support for fm10k,
contains four major parts:

1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.

v6 changes:
- add fixes line
- relocate version change message in individual patch
after the --- separator

v5 changes:
- remove one unnecessary NULL check for rte_free
- fix a wrong error message
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst

v4 changes:
- rebase to latest code
- update release 2.3 note in corresponding patches

v3 changes:
- rebase to latest code
- macro renaming according to the EAL change

v2 changes:
- reword some comments and commit messages
- split one big patch into three smaller ones

Shaopeng He (6):
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet

doc/guides/nics/fm10k.rst | 7 ++
doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 174 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
6 files changed, 211 insertions(+), 12 deletions(-)
--
1.9.3
Shaopeng He
2016-02-05 04:57:45 UTC
Permalink
rx_descriptor_done is used by interrupt mode example application
(l3fwd-power) to check rxd DD bit to decide the RX trend,
then l3fwd-power will adjust the cpu frequency according to
the result.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
v5 change:
- fix a wrong error message
---
drivers/net/fm10k/fm10k.h | 3 +++
drivers/net/fm10k/fm10k_ethdev.c | 1 +
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++++++++++++++++++++++
3 files changed, 29 insertions(+)

diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index cd38af2..e2f677a 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -345,6 +345,9 @@ uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+
uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index e4aed94..d39c33b 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2435,6 +2435,7 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.rx_queue_release = fm10k_rx_queue_release,
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
+ .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index e958865..0002f09 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -369,6 +369,31 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
return nb_rcv;
}

+int
+fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ volatile union fm10k_rx_desc *rxdp;
+ struct fm10k_rx_queue *rxq = rx_queue;
+ uint16_t desc;
+ int ret;
+
+ if (unlikely(offset >= rxq->nb_desc)) {
+ PMD_DRV_LOG(ERR, "Invalid RX descriptor offset %u", offset);
+ return 0;
+ }
+
+ desc = rxq->next_dd + offset;
+ if (desc >= rxq->nb_desc)
+ desc -= rxq->nb_desc;
+
+ rxdp = &rxq->hw_ring[desc];
+
+ ret = !!(rxdp->w.status &
+ rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+ return ret;
+}
+
static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
{
uint16_t next_rs, count = 0;
--
1.9.3
Shaopeng He
2016-02-05 04:57:46 UTC
Permalink
In interrupt mode, each rx queue can have one interrupt to notify the up
layer application when packets are available in that queue. Some queues
also can share one interrupt.
Currently, fm10k needs one separate interrupt for mailbox. So, only those
drivers which support multiple interrupt vectors e.g. vfio-pci can work
in fm10k interrupt mode.
This patch uses the RXINT/INT_MAP registers to map interrupt causes
(rx queue and other events) to vectors, and enable these interrupts
through kernel drivers like vfio-pci.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
v5 changes:
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst

v4 change:
- update release note inside the patch

v3 change:
- macro renaming according to the EAL change

v2 changes:
- split one big patch into three smaller ones
- reword some comments and commit messages
---
doc/guides/nics/fm10k.rst | 7 +++
doc/guides/rel_notes/release_2_3.rst | 2 +
drivers/net/fm10k/fm10k.h | 3 +
drivers/net/fm10k/fm10k_ethdev.c | 105 +++++++++++++++++++++++++++++++----
4 files changed, 106 insertions(+), 11 deletions(-)

diff --git a/doc/guides/nics/fm10k.rst b/doc/guides/nics/fm10k.rst
index 4206b7f..dc5cb6e 100644
--- a/doc/guides/nics/fm10k.rst
+++ b/doc/guides/nics/fm10k.rst
@@ -65,3 +65,10 @@ The FM10000 family of NICS support a maximum of a 15K jumbo frame. The value
is fixed and cannot be changed. So, even when the ``rxmode.max_rx_pkt_len``
member of ``struct rte_eth_conf`` is set to a value lower than 15364, frames
up to 15364 bytes can still reach the host interface.
+
+Interrupt mode
+~~~~~~~~~~~~~~~~~~~~~
+
+The FM10000 family of NICS need one separate interrupt for mailbox. So only
+drivers which support multiple interrupt vectors e.g. vfio-pci can work
+for fm10k interrupt mode.
diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..2cb5ebd 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,8 @@ DPDK Release 2.3
New Features
------------

+* **Added fm10k Rx interrupt support.**
+

Resolved Issues
---------------
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index e2f677a..770d6ba 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -129,6 +129,9 @@
#define RTE_FM10K_TX_MAX_FREE_BUF_SZ 64
#define RTE_FM10K_DESCS_PER_LOOP 4

+#define FM10K_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
+#define FM10K_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
+
#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
ETH_TXQ_FLAGS_NOOFFLOADS)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index d39c33b..583335a 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -54,6 +54,8 @@
/* Number of chars per uint32 type */
#define CHARS_PER_UINT32 (sizeof(uint32_t))
#define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
+/* default 1:1 map from queue ID to interrupt vector ID */
+#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])

static void fm10k_close_mbx_service(struct fm10k_hw *hw);
static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
@@ -109,6 +111,8 @@ struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {

#define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
(FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
+static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);

static void
fm10k_mbx_initlock(struct fm10k_hw *hw)
@@ -687,6 +691,7 @@ static int
fm10k_dev_rx_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i, ret;
struct fm10k_rx_queue *rxq;
uint64_t base_addr;
@@ -694,10 +699,25 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev)
uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
uint16_t buf_size;

- /* Disable RXINT to avoid possible interrupt */
- for (i = 0; i < hw->mac.max_queues; i++)
+ /* enable RXINT for interrupt mode */
+ i = 0;
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK |
+ FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_AUTOMASK |
+ FM10K_ITR_MASK_CLEAR);
+ }
+ }
+ /* Disable other RXINT to avoid possible interrupt */
+ for (; i < hw->mac.max_queues; i++)
FM10K_WRITE_REG(hw, FM10K_RXINT(i),
- 3 << FM10K_RXINT_TIMER_SHIFT);
+ 3 << FM10K_RXINT_TIMER_SHIFT);

/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; ++i) {
@@ -1053,6 +1073,9 @@ fm10k_dev_start(struct rte_eth_dev *dev)
return diag;
}

+ if (fm10k_dev_rxq_interrupt_setup(dev))
+ return -EIO;
+
diag = fm10k_dev_rx_init(dev);
if (diag) {
PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
@@ -2072,7 +2095,7 @@ fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2103,7 +2126,7 @@ fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
@@ -2134,7 +2157,7 @@ fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;

/* Bind all local non-queue interrupt to vector 0 */
- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2151,7 +2174,7 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
uint32_t int_map = FM10K_INT_MAP_DISABLE;

- int_map |= 0;
+ int_map |= FM10K_MISC_VEC_ID;

/* Only INT 0 available, other 15 are reserved. */
FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
@@ -2162,6 +2185,66 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
+ uint32_t intr_vector, vec;
+ uint16_t queue_id;
+ int result = 0;
+
+ /* fm10k needs one separate interrupt for mailbox,
+ * so only drivers which support multiple interrupt vectors
+ * e.g. vfio-pci can work for fm10k interrupt mode
+ */
+ if (!rte_intr_cap_multiple(intr_handle) ||
+ dev->data->dev_conf.intr_conf.rxq == 0)
+ return result;
+
+ intr_vector = dev->data->nb_rx_queues;
+
+ /* disable interrupt first */
+ rte_intr_disable(&dev->pci_dev->intr_handle);
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_disable_intr_pf(dev);
+ else
+ fm10k_dev_disable_intr_vf(dev);
+
+ if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+ PMD_INIT_LOG(ERR, "Failed to init event fd");
+ result = -EIO;
+ }
+
+ if (rte_intr_dp_is_en(intr_handle) && !result) {
+ intr_handle->intr_vec = rte_zmalloc("intr_vec",
+ dev->data->nb_rx_queues * sizeof(int), 0);
+ if (intr_handle->intr_vec) {
+ for (queue_id = 0, vec = FM10K_RX_VEC_START;
+ queue_id < dev->data->nb_rx_queues;
+ queue_id++) {
+ intr_handle->intr_vec[queue_id] = vec;
+ if (vec < intr_handle->nb_efd - 1
+ + FM10K_RX_VEC_START)
+ vec++;
+ }
+ } else {
+ PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+ " intr_vec", dev->data->nb_rx_queues);
+ rte_intr_efd_disable(intr_handle);
+ result = -ENOMEM;
+ }
+ }
+
+ if (hw->mac.type == fm10k_mac_pf)
+ fm10k_dev_enable_intr_pf(dev);
+ else
+ fm10k_dev_enable_intr_vf(dev);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ hw->mac.ops.update_int_moderator(hw);
+ return result;
+}
+
+static int
fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
{
struct fm10k_fault fault;
@@ -2531,7 +2614,7 @@ static int
eth_fm10k_dev_init(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
- int diag;
+ int diag, i;
struct fm10k_macvlan_filter_info *macvlan;

PMD_INIT_FUNC_TRACE();
@@ -2637,7 +2720,7 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
fm10k_dev_enable_intr_vf(dev);
}

- /* Enable uio intr after callback registered */
+ /* Enable intr after callback registered */
rte_intr_enable(&(dev->pci_dev->intr_handle));

hw->mac.ops.update_int_moderator(hw);
@@ -2645,7 +2728,6 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)
/* Make sure Switch Manager is ready before going forward. */
if (hw->mac.type == fm10k_mac_pf) {
int switch_ready = 0;
- int i;

for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
fm10k_mbx_lock(hw);
@@ -2752,7 +2834,8 @@ static struct eth_driver rte_pmd_fm10k = {
.pci_drv = {
.name = "rte_pmd_fm10k",
.id_table = pci_id_fm10k_map,
- .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+ RTE_PCI_DRV_DETACHABLE,
},
.eth_dev_init = eth_fm10k_dev_init,
.eth_dev_uninit = eth_fm10k_dev_uninit,
--
1.9.3
Shaopeng He
2016-02-05 04:57:49 UTC
Permalink
When PF establishes a connection with Switch Manager, it receives
a logic port range from SM, and registers certain logic ports from
that range, then a default VID will be send back from SM. This whole
transaction needs to be finished in dev_init, otherwise, in dev_start
the interrupt setting will be changed according to RX queue number,
and probably will cause this transaction failed.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 06bfffd..832a3fe 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2817,6 +2817,21 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev)

fm10k_mbx_unlock(hw);

+ /* Make sure default VID is ready before going forward. */
+ if (hw->mac.type == fm10k_mac_pf) {
+ for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
+ if (hw->mac.default_vid)
+ break;
+ /* Delay some time to acquire async port VLAN info. */
+ rte_delay_us(WAIT_SWITCH_MSG_US);
+ }
+
+ if (!hw->mac.default_vid) {
+ PMD_INIT_LOG(ERR, "default VID is not ready");
+ return -1;
+ }
+ }
+
/* Add default mac address */
fm10k_MAC_filter_set(dev, hw->mac.addr, true,
MAIN_VSI_POOL_NUMBER);
--
1.9.3
Shaopeng He
2016-02-05 04:57:47 UTC
Permalink
Previous dev_stop function stops the rx/tx queues. This patch adds logic
to disable rx queue interrupt, clean the datapath event and queue/vec map.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
v5 change:
- remove one unnecessary NULL check for rte_free

v2 change:
- split one big patch into three smaller ones
---
drivers/net/fm10k/fm10k_ethdev.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 583335a..da78389 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -1127,6 +1127,8 @@ fm10k_dev_start(struct rte_eth_dev *dev)
static void
fm10k_dev_stop(struct rte_eth_dev *dev)
{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
int i;

PMD_INIT_FUNC_TRACE();
@@ -1138,6 +1140,24 @@ fm10k_dev_stop(struct rte_eth_dev *dev)
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
fm10k_dev_rx_queue_stop(dev, i);
+
+ /* Disable datapath event */
+ if (rte_intr_dp_is_en(intr_handle)) {
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ FM10K_WRITE_REG(hw, FM10K_RXINT(i),
+ 3 << FM10K_RXINT_TIMER_SHIFT);
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
+ FM10K_ITR_MASK_SET);
+ }
+ }
+ /* Clean datapath event and queue/vec mapping */
+ rte_intr_efd_disable(intr_handle);
+ rte_free(intr_handle->intr_vec);
+ intr_handle->intr_vec = NULL;
}

static void
--
1.9.3
Shaopeng He
2016-02-05 04:57:48 UTC
Permalink
Interrupt mode framework has enable/disable functions for individual
rx queue, this patch implements these two functions.

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
v2 change:
- split one big patch into three smaller ones
---
drivers/net/fm10k/fm10k_ethdev.c | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index da78389..06bfffd 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -2205,6 +2205,37 @@ fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
}

static int
+fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Enable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
+ rte_intr_enable(&dev->pci_dev->intr_handle);
+ return 0;
+}
+
+static int
+fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+ struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* Disable ITR */
+ if (hw->mac.type == fm10k_mac_pf)
+ FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ else
+ FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
+ FM10K_ITR_MASK_SET);
+ return 0;
+}
+
+static int
fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
{
struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -2539,6 +2570,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
.tx_queue_setup = fm10k_tx_queue_setup,
.tx_queue_release = fm10k_tx_queue_release,
.rx_descriptor_done = fm10k_dev_rx_descriptor_done,
+ .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
.reta_update = fm10k_reta_update,
.reta_query = fm10k_reta_query,
.rss_hash_update = fm10k_rss_hash_update,
--
1.9.3
Shaopeng He
2016-02-05 04:57:50 UTC
Permalink
Previous l3fwd-power only processes IP and IPv6 packet, other
packet's mbuf is not released, and causes a memory leak.
This patch fixes this issue.

Fixes: 3c0184cc ("examples: replace some offload flags with packet type")

Signed-off-by: Shaopeng He <***@intel.com>
Acked-by: Jing Chen <***@intel.com>
Acked-by: Michael Qiu <***@intel.com>
---
v6 change:
- add fixes line
v4 change:
- update release note inside the patch
---
doc/guides/rel_notes/release_2_3.rst | 6 ++++++
examples/l3fwd-power/main.c | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 2cb5ebd..fc871ab 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -25,6 +25,12 @@ Libraries
Examples
~~~~~~~~

+* **l3fwd-power: Fixed memory leak for non-ip packet.**
+
+ Fixed issue in l3fwd-power where, recieving other packet than
+ types of IP and IPv6, the mbuf was not released, and caused
+ a memory leak.
+

Other
~~~~~
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 828c18a..d9cd848 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -714,7 +714,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
/* We don't currently handle IPv6 packets in LPM mode. */
rte_pktmbuf_free(m);
#endif
- }
+ } else
+ rte_pktmbuf_free(m);

}
--
1.9.3
Bruce Richardson
2016-02-05 16:34:50 UTC
Permalink
Post by Shaopeng He
This patch series adds interrupt mode support for fm10k,
1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.
- add fixes line
- relocate version change message in individual patch
after the --- separator
- remove one unnecessary NULL check for rte_free
- fix a wrong error message
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst
- rebase to latest code
- update release 2.3 note in corresponding patches
- rebase to latest code
- macro renaming according to the EAL change
- reword some comments and commit messages
- split one big patch into three smaller ones
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc/guides/nics/fm10k.rst | 7 ++
doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 174 ++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
6 files changed, 211 insertions(+), 12 deletions(-)
--
1.9.3
Applied to dpdk-next-net/rel_16_04.

Thanks,
/Bruce
Liu, Yong
2016-03-03 08:36:10 UTC
Permalink
Hi Thomas,
In patchwork website, this patch set seem that has been applied.
http://dpdk.org/dev/patchwork/patch/10381/

But I can't find it neither in dpdk repo nor in dpdk-next-net repo.
Could you check with that?
-----Original Message-----
Sent: Friday, February 05, 2016 12:58 PM
Subject: [dpdk-dev] [PATCH v6 0/6] interrupt mode for fm10k
This patch series adds interrupt mode support for fm10k,
1. implement rx_descriptor_done function in fm10k
2. add rx interrupt support in fm10k PF and VF
3. make sure default VID available in dev_init in fm10k
4. fix a memory leak for non-ip packet in l3fwd-power,
which happens mostly when testing fm10k interrupt mode.
- add fixes line
- relocate version change message in individual patch
after the --- separator
- remove one unnecessary NULL check for rte_free
- fix a wrong error message
- add more clean up when memory allocation fails
- split line over 80 characters to 2 lines
- update interrupt mode limitation in fm10k.rst
- rebase to latest code
- update release 2.3 note in corresponding patches
- rebase to latest code
- macro renaming according to the EAL change
- reword some comments and commit messages
- split one big patch into three smaller ones
fm10k: implement rx_descriptor_done function
fm10k: setup rx queue interrupts for PF and VF
fm10k: remove rx queue interrupts when dev stops
fm10k: add rx queue interrupt en/dis functions
fm10k: make sure default VID available in dev_init
l3fwd-power: fix a memory leak for non-ip packet
doc/guides/nics/fm10k.rst | 7 ++
doc/guides/rel_notes/release_2_3.rst | 8 ++
drivers/net/fm10k/fm10k.h | 6 ++
drivers/net/fm10k/fm10k_ethdev.c | 174
++++++++++++++++++++++++++++++++---
drivers/net/fm10k/fm10k_rxtx.c | 25 +++++
examples/l3fwd-power/main.c | 3 +-
6 files changed, 211 insertions(+), 12 deletions(-)
--
1.9.3
Thomas Monjalon
2016-03-03 09:10:48 UTC
Permalink
Post by Liu, Yong
Hi Thomas,
In patchwork website, this patch set seem that has been applied.
http://dpdk.org/dev/patchwork/patch/10381/
But I can't find it neither in dpdk repo nor in dpdk-next-net repo.
Could you check with that?
It's here:
http://dpdk.org/browse/next/dpdk-next-net/log/?h=rel_16_04&ofs=50
Liu, Yong
2016-03-03 09:19:25 UTC
Permalink
Thanks Thomas, found it.
-----Original Message-----
Sent: Thursday, March 03, 2016 5:11 PM
To: Liu, Yong
Subject: Re: [dpdk-dev] [PATCH v6 0/6] interrupt mode for fm10k
Post by Liu, Yong
Hi Thomas,
In patchwork website, this patch set seem that has been applied.
http://dpdk.org/dev/patchwork/patch/10381/
But I can't find it neither in dpdk repo nor in dpdk-next-net repo.
Could you check with that?
http://dpdk.org/browse/next/dpdk-next-net/log/?h=rel_16_04&ofs=50
Continue reading on narkive:
Loading...