Discussion:
[dpdk-dev] [PATCH 11/22] net/fm10k: enable port detach on secondary process
(too old to reply)
Qi Zhang
2018-06-07 12:38:38 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/fm10k/fm10k_ethdev.c | 9 +++++++++
1 file changed, 9 insertions(+)

diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 3ff1b0e0f..c7042be4e 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -3264,6 +3264,15 @@ static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,

static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
{
+ struct rte_eth_dev *ethdev =
+ rte_eth_dev_allocated(pci_dev->device.name);
+
+ if (!ethdev)
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
}
--
2.13.6
Qi Zhang
2018-06-07 12:38:43 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/null/rte_eth_null.c | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/drivers/net/null/rte_eth_null.c b/drivers/net/null/rte_eth_null.c
index 1d2e6b9e9..e5b8d2f03 100644
--- a/drivers/net/null/rte_eth_null.c
+++ b/drivers/net/null/rte_eth_null.c
@@ -623,6 +623,7 @@ rte_pmd_null_probe(struct rte_vdev_device *dev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &ops;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -667,18 +668,31 @@ static int
rte_pmd_null_remove(struct rte_vdev_device *dev)
{
struct rte_eth_dev *eth_dev = NULL;
+ const char *name;

if (!dev)
return -EINVAL;

+ name = rte_vdev_device_name(dev);
+
PMD_LOG(INFO, "Closing null ethdev on numa socket %u",
rte_socket_id());

/* find the ethdev entry */
- eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev));
+ eth_dev = rte_eth_dev_allocated(name);
if (eth_dev == NULL)
return -1;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
rte_free(eth_dev->data->dev_private);

rte_eth_dev_release_port(eth_dev);
--
2.13.6
Qi Zhang
2018-06-07 12:38:34 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/i40e/i40e_ethdev.c | 2 ++
drivers/net/i40e/i40e_ethdev_vf.c | 9 +++++++++
2 files changed, 11 insertions(+)

diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 13c5d3296..1de3c1499 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -678,6 +678,8 @@ static int eth_i40e_pci_remove(struct rte_pci_device *pci_dev)
if (!ethdev)
return -ENODEV;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);

if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
return rte_eth_dev_destroy(ethdev, i40e_vf_representor_uninit);
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 804e44530..2b1ece851 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1500,6 +1500,15 @@ static int eth_i40evf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,

static int eth_i40evf_pci_remove(struct rte_pci_device *pci_dev)
{
+ struct rte_eth_dev *ethdev;
+ ethdev = rte_eth_dev_allocated(pci_dev->device.name);
+
+ if (!ethdev)
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
return rte_eth_dev_pci_generic_remove(pci_dev, i40evf_dev_uninit);
}
--
2.13.6
Qi Zhang
2018-06-07 12:38:47 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/tap/rte_eth_tap.c | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 5531fe9d9..56d3b6cc9 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1759,6 +1759,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &ops;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -1827,12 +1828,24 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
{
struct rte_eth_dev *eth_dev = NULL;
struct pmd_internals *internals;
+ const char *name;
int i;

+ name = rte_vdev_device_name(dev);
/* find the ethdev entry */
- eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev));
+ eth_dev = rte_eth_dev_allocated(name);
if (!eth_dev)
- return 0;
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }

internals = eth_dev->data->dev_private;
--
2.13.6
Qi Zhang
2018-06-07 12:38:42 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/kni/rte_eth_kni.c | 11 +++++++++++
1 file changed, 11 insertions(+)

diff --git a/drivers/net/kni/rte_eth_kni.c b/drivers/net/kni/rte_eth_kni.c
index ab63ea427..3ee527ab2 100644
--- a/drivers/net/kni/rte_eth_kni.c
+++ b/drivers/net/kni/rte_eth_kni.c
@@ -419,6 +419,7 @@ eth_kni_probe(struct rte_vdev_device *vdev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &eth_kni_ops;
+ eth_dev->device = &vdev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -463,6 +464,16 @@ eth_kni_remove(struct rte_vdev_device *vdev)
if (eth_dev == NULL)
return -1;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(vdev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
eth_kni_dev_stop(eth_dev);

internals = eth_dev->data->dev_private;
--
2.13.6
Qi Zhang
2018-06-07 12:38:31 UTC
Permalink
The patch introduce the solution to handle different hotplug cases in
multi-process situation, it include below scenario:

1. Attach a share device from primary
2. Detach a share device from primary
3. Attach a share device from secondary
4. Detach a share device from secondary
5. Attach a private device from secondary
6. Detach a private device from secondary
7. Detach a share device from secondary privately
8. Attach a share device from secondary privately

In primary-secondary process model, we assume device is shared by default.
that means attach or detach a device on any process will broadcast to
all other processes through mp channel then device information will be
synchronized on all processes.

Any failure during attaching process will cause inconsistent status
between processes, so proper rollback action should be considered.
Also it is not safe to detach a share device when other process still use
it, so a handshake mechanism is introduced, it will be implemented in
following separate patch.

Scenario for Case 1, 2:

attach device
a) primary attach the new device if failed goto h).
b) primary send attach sync request to all secondary.
c) secondary receive request and attach device and send reply.
d) primary check the reply if all success go to i).
e) primary send attach rollback sync request to all secondary.
f) secondary receive the request and detach device and send reply.
g) primary receive the reply and detach device as rollback action.
h) attach fail
i) attach success

detach device
a) primary perform pre-detach check, if device is locked, goto i).
b) primary send pre-detach sync request to all secondary.
c) secondary perform pre-detach check and send reply.
d) primary check the reply if any fail goto i).
e) primary send detach sync request to all secondary
f) secondary detach the device and send reply (assume no fail)
g) primary detach the device.
h) detach success
i) detach failed

Case 3, 4:
This will be implemented in following patch.

Case 5, 6:
Secondary process can attach private device which only visible to itself,
in this case no IPC is involved, primary process is not allowd to have
private device so far.

Case 7, 8:
Secondary process can also temporally to detach a share device "privately"
then attach it back later, this action also not impact other processes.

APIs chenages:

rte_eth_dev_attach and rte_eth_dev_attach are extended to support
share device attach/detach in primary-secondary process model, it will
be called in case 1,2,3,4.

New API rte_eth_dev_attach_private and rte_eth_dev_detach_private are
introduced to cover case 5,6,7,8, this API can only be invoked in secondary
process.

Signed-off-by: Qi Zhang <***@intel.com>
---
lib/librte_eal/common/eal_private.h | 8 ++
lib/librte_eal/linuxapp/eal/eal.c | 6 ++
lib/librte_ethdev/Makefile | 1 +
lib/librte_ethdev/rte_ethdev.c | 183 ++++++++++++++++++++++++++++---
lib/librte_ethdev/rte_ethdev.h | 37 +++++++
lib/librte_ethdev/rte_ethdev_core.h | 5 +
lib/librte_ethdev/rte_ethdev_driver.h | 27 +++++
lib/librte_ethdev/rte_ethdev_mp.c | 195 ++++++++++++++++++++++++++++++++++
lib/librte_ethdev/rte_ethdev_mp.h | 44 ++++++++
9 files changed, 489 insertions(+), 17 deletions(-)
create mode 100644 lib/librte_ethdev/rte_ethdev_mp.c
create mode 100644 lib/librte_ethdev/rte_ethdev_mp.h

diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index bdadc4d50..92fa59bed 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -258,4 +258,12 @@ int rte_mp_channel_init(void);
*/
void dev_callback_process(char *device_name, enum rte_dev_event_type event);

+/**
+ * Register mp channel callback functions of ethdev layer.
+ *
+ * @return
+ * 0 on success.
+ * (<0) on failure.
+ */
+int rte_eth_dev_mp_init(void);
#endif /* _EAL_PRIVATE_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 8655b8691..b276e1caa 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -1041,6 +1041,12 @@ rte_eal_init(int argc, char **argv)

rte_eal_mcfg_complete();

+ if (rte_eth_dev_mp_init()) {
+ rte_eal_init_alert("rte_eth_dev_mp_init() failed\n");
+ rte_errno = ENOEXEC;
+ return -1;
+ }
+
return fctret;
}

diff --git a/lib/librte_ethdev/Makefile b/lib/librte_ethdev/Makefile
index c2f2f7d82..04e93f337 100644
--- a/lib/librte_ethdev/Makefile
+++ b/lib/librte_ethdev/Makefile
@@ -19,6 +19,7 @@ EXPORT_MAP := rte_ethdev_version.map
LIBABIVER := 9

SRCS-y += rte_ethdev.c
+SRCS-y += rte_ethdev_mp.c
SRCS-y += rte_flow.c
SRCS-y += rte_tm.c
SRCS-y += rte_mtr.c
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index ec14adb91..24360f522 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -41,11 +41,12 @@
#include "rte_ethdev.h"
#include "rte_ethdev_driver.h"
#include "ethdev_profile.h"
+#include "rte_ethdev_mp.h"

-static int ethdev_logtype;
+int ethdev_logtype;

-#define ethdev_log(level, fmt, ...) \
- rte_log(RTE_LOG_ ## level, ethdev_logtype, fmt "\n", ## __VA_ARGS__)
+#define RTE_ETH_MP_ACTION_REQUEST "rte_eth_mp_request"
+#define RTE_ETH_MP_ACTION_RESPONSE "rte_eth_mp_response"

static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
@@ -656,9 +657,8 @@ eth_err(uint16_t port_id, int ret)
return ret;
}

-/* attach the new device, then store port_id of the device */
int
-rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
+do_eth_dev_attach(const char *devargs, uint16_t *port_id)
{
int current = rte_eth_dev_count_total();
struct rte_devargs da;
@@ -703,14 +703,104 @@ rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
return ret;
}

-/* detach the device, then store the name of the device */
int
-rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
+do_eth_dev_detach(uint16_t port_id)
{
struct rte_device *dev;
struct rte_bus *bus;
+ int ret = 0;
+
+ dev = rte_eth_devices[port_id].device;
+ if (dev == NULL)
+ return -EINVAL;
+
+ bus = rte_bus_find_by_device(dev);
+ if (bus == NULL)
+ return -ENOENT;
+
+ ret = rte_eal_hotplug_remove(bus->name, dev->name);
+ if (ret < 0)
+ return ret;
+
+ rte_eth_dev_release_port(&rte_eth_devices[port_id]);
+ return ret;
+
+}
+
+/* attach the new device, then store port_id of the device */
+int
+rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
+{
+ struct eth_dev_mp_req req;
+ int ret;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+
+ /**
+ * If secondary process, we just send request to primray
+ * to start the process.
+ */
+ req.t = REQ_TYPE_ATTACH;
+ strlcpy(req.devargs, devargs, MAX_DEV_ARGS_LEN);
+
+ ret = rte_eth_dev_request_to_primary(&req);
+ if (ret) {
+ ethdev_log(ERR, "Failed to send device attach request to primary\n");
+ return ret;
+ }
+
+ *port_id = req.port_id;
+ return req.result;
+ }
+
+ ret = do_eth_dev_attach(devargs, port_id);
+ if (ret)
+ return ret;
+
+ /* send attach request to seoncary */
+ req.t = REQ_TYPE_ATTACH;
+ strlcpy(req.devargs, devargs, MAX_DEV_ARGS_LEN);
+ req.port_id = *port_id;
+ ret = rte_eth_dev_request_to_secondary(&req);
+ if (ret) {
+ ethdev_log(ERR, "Failed to send device attach request to secondary\n");
+ goto rollback;
+ }
+
+ if (req.result)
+ goto rollback;
+
+ return 0;
+
+rollback:
+ /* send rollback request to secondary since some one fail to attach */
+ req.t = REQ_TYPE_ATTACH_ROLLBACK;
+ req.port_id = *port_id;
+ rte_eth_dev_request_to_secondary(&req);
+
+ do_eth_dev_detach(*port_id);
+
+ return -ENODEV;
+}
+
+/* attach the new device, then store port_id of the device */
+int
+rte_eth_dev_attach_private(const char *devargs, uint16_t *port_id)
+{
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ return -ENOTSUP;
+
+ return do_eth_dev_attach(devargs, port_id);
+}
+
+/* detach the device, then store the name of the device */
+int
+rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
+{
+ struct eth_dev_mp_req req = {0};
+ int ret;
uint32_t dev_flags;
- int ret = -1;

RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);

@@ -721,22 +811,81 @@ rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
return -ENOTSUP;
}

- dev = rte_eth_devices[port_id].device;
- if (dev == NULL)
- return -EINVAL;
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ req.t = REQ_TYPE_DETACH;
+ req.port_id = port_id;

- bus = rte_bus_find_by_device(dev);
- if (bus == NULL)
- return -ENOENT;
+ /**
+ * If secondary process, we just send request to primray
+ * to start the process.
+ */
+ ret = rte_eth_dev_request_to_primary(&req);
+ if (ret) {
+ ethdev_log(ERR, "Failed to send device detach request to primary\n");
+ return ret;
+ }

- ret = rte_eal_hotplug_remove(bus->name, dev->name);
- if (ret < 0)
+ return req.result;
+ }
+
+ /* check pre_detach */
+ req.t = REQ_TYPE_PRE_DETACH;
+ req.port_id = port_id;
+ ret = rte_eth_dev_request_to_secondary(&req);
+ if (ret) {
+ ethdev_log(ERR, "Failed to send device pre-detach request to secondary\n");
+ return ret;
+ }
+
+ if (req.result) {
+ ethdev_log(ERR, "Device is busy on secondary, can't be detached\n");
+ return req.result;
+ }
+
+ /* detach on seconary first */
+ req.t = REQ_TYPE_DETACH;
+ ret = rte_eth_dev_request_to_secondary(&req);
+ if (ret) {
+ ethdev_log(ERR, "Failed to send device detach request to secondary\n");
+ return ret;
+ }
+
+ if (req.result)
+ /**
+ * this should rarely happen, something wrong in secondary
+ * process, will not block primary detach.
+ */
+ ethdev_log(ERR, "Failed to detach device on secondary process\n");
+
+ /* detach on primary */
+ ret = do_eth_dev_detach(port_id);
+ if (ret)
return ret;

- rte_eth_dev_release_port(&rte_eth_devices[port_id]);
return 0;
}

+/* detach the device, then store the name of the device */
+int
+rte_eth_dev_detach_private(uint16_t port_id, char *name __rte_unused)
+{
+ uint32_t dev_flags;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ return -ENOTSUP;
+
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
+
+ dev_flags = rte_eth_devices[port_id].data->dev_flags;
+ if (dev_flags & RTE_ETH_DEV_BONDED_SLAVE) {
+ ethdev_log(ERR,
+ "Port %" PRIu16 " is bonded, cannot detach", port_id);
+ return -ENOTSUP;
+ }
+
+ return do_eth_dev_detach(port_id);
+}
+
static int
rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
{
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 36e3984ea..bb03d613b 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -1462,6 +1462,9 @@ uint16_t __rte_experimental rte_eth_dev_count_total(void);

/**
* Attach a new Ethernet device specified by arguments.
+ * In multi-process mode, it will sync with other process
+ * to make sure all processes attach the device, any
+ * failure on other process will rollback the action.
*
* @param devargs
* A pointer to a strings array describing the new device
@@ -1475,9 +1478,27 @@ uint16_t __rte_experimental rte_eth_dev_count_total(void);
int rte_eth_dev_attach(const char *devargs, uint16_t *port_id);

/**
+ * Attach a private Ethernet device specified by arguments.
+ * A private device is invisible to other process.
+ * Can only be invoked in secondary process.
+ *
+ * @param devargs
+ * A pointer to a strings array describing the new device
+ * to be attached. The strings should be a pci address like
+ * '0000:01:00.0' or virtual device name like 'net_pcap0'.
+ * @param port_id
+ * A pointer to a port identifier actually attached.
+ * @return
+ * 0 on success and port_id is filled, negative on error
+ */
+int rte_eth_dev_attach_private(const char *devargs, uint16_t *port_id);
+
+/**
* Detach a Ethernet device specified by port identifier.
* This function must be called when the device is in the
* closed state.
+ * In multi-process mode, it will sync with other process
+ * to detach the device.
*
* @param port_id
* The port identifier of the device to detach.
@@ -1490,6 +1511,22 @@ int rte_eth_dev_attach(const char *devargs, uint16_t *port_id);
int rte_eth_dev_detach(uint16_t port_id, char *devname);

/**
+ * Detach a private Ethernet device specified by port identifier
+ * This function must be called when the device is in the
+ * closed state.
+ * Can only be invoked in secondary process.
+ *
+ * @param port_id
+ * The port identifier of the device to detach.
+ * @param devname
+ * A pointer to a buffer that will be filled with the device name.
+ * This buffer must be at least RTE_DEV_NAME_MAX_LEN long.
+ * @return
+ * 0 on success and devname is filled, negative on error
+ */
+int rte_eth_dev_detach_private(uint16_t port_id, char *devname);
+
+/**
* Convert a numerical speed in Mbps to a bitmap flag that can be used in
* the bitmap link_speeds of the struct rte_eth_conf
*
diff --git a/lib/librte_ethdev/rte_ethdev_core.h b/lib/librte_ethdev/rte_ethdev_core.h
index 33d12b3a2..2cb6de745 100644
--- a/lib/librte_ethdev/rte_ethdev_core.h
+++ b/lib/librte_ethdev/rte_ethdev_core.h
@@ -622,4 +622,9 @@ struct rte_eth_dev_data {
*/
extern struct rte_eth_dev rte_eth_devices[];

+extern int ethdev_logtype;
+#define ethdev_log(level, fmt, ...) \
+ rte_log(RTE_LOG_ ## level, ethdev_logtype, fmt "\n", ## __VA_ARGS__)
+
+
#endif /* _RTE_ETHDEV_CORE_H_ */
diff --git a/lib/librte_ethdev/rte_ethdev_driver.h b/lib/librte_ethdev/rte_ethdev_driver.h
index 261335426..616add313 100644
--- a/lib/librte_ethdev/rte_ethdev_driver.h
+++ b/lib/librte_ethdev/rte_ethdev_driver.h
@@ -338,6 +338,33 @@ typedef int (*ethdev_uninit_t)(struct rte_eth_dev *ethdev);
int __rte_experimental
rte_eth_dev_destroy(struct rte_eth_dev *ethdev, ethdev_uninit_t ethdev_uninit);

+/**
+ * Attach a new Ethernet device in current process.
+ *
+ * @param devargs
+ * A pointer to a strings array describing the new device
+ * to be attached. The strings should be a pci address like
+ * '0000:01:00.0' or virtual device name like 'net_pcap0'.
+ * @param port_id
+ * A pointer to a port identifier actually attached.
+ * @return
+ * 0 on success and port_id is filled, negative on error
+ */
+int do_eth_dev_attach(const char *devargs, uint16_t *port_id);
+
+/**
+ * Detach a Ethernet device in current process.
+ *
+ * @param port_id
+ * The port identifier of the device to detach.
+ * @param devname
+ * A pointer to a buffer that will be filled with the device name.
+ * This buffer must be at least RTE_DEV_NAME_MAX_LEN long.
+ * @return
+ * 0 on success and devname is filled, negative on error
+ */
+int do_eth_dev_detach(uint16_t port_id);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_ethdev/rte_ethdev_mp.c b/lib/librte_ethdev/rte_ethdev_mp.c
new file mode 100644
index 000000000..8ede8151d
--- /dev/null
+++ b/lib/librte_ethdev/rte_ethdev_mp.c
@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include "rte_ethdev_driver.h"
+#include "rte_ethdev_mp.h"
+
+static int detach_on_secondary(uint16_t port_id)
+{
+ struct rte_device *dev;
+ struct rte_bus *bus;
+ int ret = 0;
+
+ if (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED) {
+ ethdev_log(ERR, "detach on secondary: invalid port %d\n",
+ port_id);
+ return -ENODEV;
+ }
+
+ dev = rte_eth_devices[port_id].device;
+ if (dev == NULL)
+ return -EINVAL;
+
+ bus = rte_bus_find_by_device(dev);
+ if (bus == NULL)
+ return -ENOENT;
+
+ ret = rte_eal_hotplug_remove(bus->name, dev->name);
+ if (ret) {
+ ethdev_log(ERR, "failed to hot unplug bus: %s, device:%s\n",
+ bus->name, dev->name);
+ return ret;
+ }
+
+ rte_eth_dev_release_port_local(&rte_eth_devices[port_id]);
+ return ret;
+}
+
+static int attach_on_secondary(const char *devargs, uint16_t port_id)
+{
+ struct rte_devargs da;
+ int ret;
+
+ if (rte_eth_devices[port_id].state != RTE_ETH_DEV_UNUSED) {
+ ethdev_log(ERR, "port %d already in used, failed to attach\n",
+ port_id);
+ return -EINVAL;
+ }
+
+ memset(&da, 0, sizeof(da));
+
+ if (rte_devargs_parse(&da, "%s", devargs)) {
+ ethdev_log(ERR, "failed to parse devargs %s\n", devargs);
+ return -EINVAL;
+ }
+
+ ret = rte_eal_hotplug_add(da.bus->name, da.name, "");
+ if (ret) {
+ ethdev_log(ERR, "failed to hotplug bus:%s, device:%s\n",
+ da.bus->name, da.name);
+ free(da.args);
+ return ret;
+ }
+
+ if (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED) {
+ ethdev_log(ERR, "failed to attach to port %d, this is a pmd issue\n",
+ port_id);
+ return -ENODEV;
+ }
+ free(da.args);
+ return 0;
+}
+
+static int handle_secondary_request(const struct rte_mp_msg *msg, const void *peer)
+{
+ (void)msg;
+ (void)(peer);
+ return -ENOTSUP;
+}
+
+static int handle_primary_response(const struct rte_mp_msg *msg, const void *peer)
+{
+ (void)msg;
+ (void)(peer);
+ return -ENOTSUP;
+}
+
+static int handle_primary_request(const struct rte_mp_msg *msg, const void *peer)
+{
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)msg->param;
+ struct rte_mp_msg mp_resp = {0};
+ struct eth_dev_mp_req *resp =
+ (struct eth_dev_mp_req *)mp_resp.param;
+ int ret = 0;
+
+ memset(&mp_resp, 0, sizeof(mp_resp));
+
+ switch (req->t) {
+ case REQ_TYPE_ATTACH:
+ ret = attach_on_secondary(req->devargs, req->port_id);
+ break;
+ case REQ_TYPE_PRE_DETACH:
+ ret = 0;
+ break;
+ case REQ_TYPE_DETACH:
+ case REQ_TYPE_ATTACH_ROLLBACK:
+ ret = detach_on_secondary(req->port_id);
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ strcpy(mp_resp.name, ETH_DEV_MP_ACTION_REQUEST);
+ mp_resp.len_param = sizeof(*req);
+ memcpy(resp, req, sizeof(*resp));
+ resp->result = ret;
+ if (rte_mp_reply(&mp_resp, peer) < 0) {
+ ethdev_log(ERR, "failed to send reply to primary request\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+int rte_eth_dev_request_to_primary(struct eth_dev_mp_req *req)
+{
+ (void)req;
+ return -ENOTSUP;
+}
+
+/**
+ * Request from primary to secondary.
+ *
+ * Be invoked when try to attach or detach a share device
+ * from primary process.
+ */
+int rte_eth_dev_request_to_secondary(struct eth_dev_mp_req *req)
+{
+ struct rte_mp_msg mp_req = {0};
+ struct rte_mp_reply mp_reply;
+ struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
+ int ret;
+ int i;
+
+ memcpy(mp_req.param, req, sizeof(*req));
+ mp_req.len_param = sizeof(*req);
+ strcpy(mp_req.name, ETH_DEV_MP_ACTION_REQUEST);
+
+ ret = rte_mp_request_sync(&mp_req, &mp_reply, &ts);
+ if (ret) {
+ ethdev_log(ERR, "rte_mp_request_sync failed\n");
+ return ret;
+ }
+
+ req->result = 0;
+ for (i = 0; i < mp_reply.nb_received; i++) {
+ struct eth_dev_mp_req *resp =
+ (struct eth_dev_mp_req *)mp_reply.msgs[i].param;
+ if (resp->result) {
+ req->result = resp->result;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+int rte_eth_dev_mp_init(void)
+{
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ if (rte_mp_action_register(ETH_DEV_MP_ACTION_REQUEST,
+ handle_secondary_request)) {
+ RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
+ ETH_DEV_MP_ACTION_REQUEST);
+ return -1;
+ }
+ } else {
+ if (rte_mp_action_register(ETH_DEV_MP_ACTION_RESPONSE,
+ handle_primary_response)) {
+ RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
+ ETH_DEV_MP_ACTION_RESPONSE);
+ return -1;
+ }
+ if (rte_mp_action_register(ETH_DEV_MP_ACTION_REQUEST,
+ handle_primary_request)) {
+ RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
+ ETH_DEV_MP_ACTION_REQUEST);
+ }
+ }
+
+ return 0;
+}
+
diff --git a/lib/librte_ethdev/rte_ethdev_mp.h b/lib/librte_ethdev/rte_ethdev_mp.h
new file mode 100644
index 000000000..c3e55dfec
--- /dev/null
+++ b/lib/librte_ethdev/rte_ethdev_mp.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _RTE_ETHDEV_MP_H_
+#define _RTE_ETHDEV_MP_H_
+
+#define MAX_DEV_ARGS_LEN 0x80
+
+#define ETH_DEV_MP_ACTION_REQUEST "eth_dev_mp_request"
+#define ETH_DEV_MP_ACTION_RESPONSE "eth_dev_mp_response"
+
+enum eth_dev_req_type {
+ REQ_TYPE_ATTACH,
+ REQ_TYPE_PRE_DETACH,
+ REQ_TYPE_DETACH,
+ REQ_TYPE_ATTACH_ROLLBACK,
+};
+
+struct eth_dev_mp_req {
+ enum eth_dev_req_type t;
+ char devargs[MAX_DEV_ARGS_LEN];
+ uint16_t port_id;
+ int result;
+};
+
+/**
+ * this is a synchronous wrapper for secondary process send
+ * request to primary process, this is invoked when an attach
+ * or detach request issued from primary.
+ */
+int rte_eth_dev_request_to_primary(struct eth_dev_mp_req *req);
+
+/**
+ * this is a synchronous wrapper for primary process send
+ * request to secondary process, this is invoked when an attach
+ * or detach request issued from secondary process.
+ */
+int rte_eth_dev_request_to_secondary(struct eth_dev_mp_req *req);
+
+/* Register mp channel callback functions of ethdev layer.*/
+int rte_eth_dev_mp_init(void);
+
+#endif
--
2.13.6
Qi Zhang
2018-06-07 12:38:32 UTC
Permalink
Introduce API rte_eth_dev_lock and rte_eth_dev_unlock to let
application lock or unlock on specific ethdev, a locked device
can't be detached, this help applicaiton to prevent unexpected
device detaching, especially in multi-process envrionment.

Aslo the new API let application to register a callback function
which will be invoked before a device is going to be detached,
the return value of the function will decide if device will continue
be detached or not, this support application to do condition check
at runtime.

Signed-off-by: Qi Zhang <***@intel.com>
---
lib/librte_ethdev/Makefile | 1 +
lib/librte_ethdev/rte_ethdev.c | 41 ++++++++++++++-
lib/librte_ethdev/rte_ethdev.h | 64 ++++++++++++++++++++++
lib/librte_ethdev/rte_ethdev_lock.c | 102 ++++++++++++++++++++++++++++++++++++
lib/librte_ethdev/rte_ethdev_lock.h | 23 ++++++++
lib/librte_ethdev/rte_ethdev_mp.c | 3 +-
6 files changed, 232 insertions(+), 2 deletions(-)
create mode 100644 lib/librte_ethdev/rte_ethdev_lock.c
create mode 100644 lib/librte_ethdev/rte_ethdev_lock.h

diff --git a/lib/librte_ethdev/Makefile b/lib/librte_ethdev/Makefile
index 04e93f337..5c4646469 100644
--- a/lib/librte_ethdev/Makefile
+++ b/lib/librte_ethdev/Makefile
@@ -20,6 +20,7 @@ LIBABIVER := 9

SRCS-y += rte_ethdev.c
SRCS-y += rte_ethdev_mp.c
+SRCS-y += rte_ethdev_lock.c
SRCS-y += rte_flow.c
SRCS-y += rte_tm.c
SRCS-y += rte_mtr.c
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 24360f522..6494e71a4 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -42,6 +42,7 @@
#include "rte_ethdev_driver.h"
#include "ethdev_profile.h"
#include "rte_ethdev_mp.h"
+#include "rte_ethdev_lock.h"

int ethdev_logtype;

@@ -787,7 +788,6 @@ rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
int
rte_eth_dev_attach_private(const char *devargs, uint16_t *port_id)
{
-
if (rte_eal_process_type() == RTE_PROC_PRIMARY)
return -ENOTSUP;

@@ -828,6 +828,10 @@ rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
return req.result;
}

+ ret = process_lock_callbacks(port_id);
+ if (ret)
+ return ret;
+
/* check pre_detach */
req.t = REQ_TYPE_PRE_DETACH;
req.port_id = port_id;
@@ -870,6 +874,7 @@ int
rte_eth_dev_detach_private(uint16_t port_id, char *name __rte_unused)
{
uint32_t dev_flags;
+ int ret;

if (rte_eal_process_type() == RTE_PROC_PRIMARY)
return -ENOTSUP;
@@ -883,6 +888,10 @@ rte_eth_dev_detach_private(uint16_t port_id, char *name __rte_unused)
return -ENOTSUP;
}

+ ret = process_lock_callbacks(port_id);
+ if (ret)
+ return ret;
+
return do_eth_dev_detach(port_id);
}

@@ -4683,6 +4692,36 @@ rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
return result;
}

+static int
+dev_is_busy(uint16_t port_id __rte_unused, void *user_args __rte_unused)
+{
+ return -EBUSY;
+}
+
+int
+rte_eth_dev_lock(uint16_t port_id, rte_eth_dev_lock_callback_t callback,
+ void *user_args)
+{
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ if (callback == NULL)
+ return register_lock_callback(port_id, dev_is_busy, NULL);
+ else
+ return register_lock_callback(port_id, callback, user_args);
+}
+
+int
+rte_eth_dev_unlock(uint16_t port_id, rte_eth_dev_lock_callback_t callback,
+ void *user_args)
+{
+ RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+ if (callback == NULL)
+ return unregister_lock_callback(port_id, dev_is_busy, NULL);
+ else
+ return unregister_lock_callback(port_id, callback, user_args);
+}
+
RTE_INIT(ethdev_init_log);
static void
ethdev_init_log(void)
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index bb03d613b..506b6acdd 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -4356,6 +4356,70 @@ rte_eth_tx_buffer(uint16_t port_id, uint16_t queue_id,
return rte_eth_tx_buffer_flush(port_id, queue_id, buffer);
}

+/**
+ * Callback function before device is detached.
+ *
+ * This type of function will be added into a function list, and will be
+ * invoked before device be detached. Application can register a callback
+ * function so it can be notified and do some cleanup before detach happen.
+ * Also, any callback function return !0 value will prevent device be
+ * detached(ref. rte_eth_dev_lock and rte_eth_dev_unlock).
+ *
+ * @param port_id
+ * The port identifier of the Ethernet device.
+ * @param user_args
+ * This is parameter "user_args" be saved when callback function is
+ * registered(rte_dev_eth_lock).
+ *
+ * @return
+ * 0 device is allowed be detached.
+ * !0 device is not allowed be detached.
+ */
+typedef int (*rte_eth_dev_lock_callback_t)(uint16_t port_id, void *user_args);
+
+/**
+ * Lock an Ethernet Device directly or register a callback function
+ * for condition check at runtime, this help application to prevent
+ * a device be detached unexpectly.
+ * NOTE: Lock a device mutliple times with same parmeter will increase
+ * a ref_count, and coresponding unlock decrease the ref_count, the
+ * device will be unlocked when ref_count reach 0.
+ *
+ * @param port_id
+ * The port identifier of the Ethernet device.
+ * @param callback
+ * !NULL the callback function will be added into a pre-detach list,
+ * it will be invoked when a device is going to be detached. The
+ * return value will decide if continue detach the device or not.
+ * NULL lock the device directly, basically this just regiter a empty
+ * callback function(dev_is_busy) that return -EBUSY, so we can
+ * handle the pre-detach check in unified way.
+ * @param user_args
+ * parameter will be parsed to callback function, only valid when
+ * callback != NULL.
+ * @return
+ * 0 on success, negative on error.
+ */
+int rte_eth_dev_lock(uint16_t port_id, rte_eth_dev_lock_callback_t callback,
+ void *user_args);
+
+/**
+ * Reverse operation of rte_eth_dev_lock.
+ *
+ * @param port_id
+ * The port identifier of the Ethernet device.
+ * @param callback
+ * NULL decrease the ref_count of default callback function.
+ * !NULL decrease the ref_count of specific callback with matched
+ * user_args.
+ * @param user_args
+ * parameter to match, only valid when callback != NULL.
+ * @return
+ * 0 on success, negative on error.
+ */
+int rte_eth_dev_unlock(uint16_t port_id, rte_eth_dev_lock_callback_t callback,
+ void *user_args);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_ethdev/rte_ethdev_lock.c b/lib/librte_ethdev/rte_ethdev_lock.c
new file mode 100644
index 000000000..688d1d70a
--- /dev/null
+++ b/lib/librte_ethdev/rte_ethdev_lock.c
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#include "rte_ethdev_lock.h"
+
+struct lock_entry {
+ TAILQ_ENTRY(lock_entry) next;
+ rte_eth_dev_lock_callback_t callback;
+ uint16_t port_id;
+ void *user_args;
+ int ref_count;
+};
+
+TAILQ_HEAD(lock_entry_list, lock_entry);
+static struct lock_entry_list lock_entry_list =
+ TAILQ_HEAD_INITIALIZER(lock_entry_list);
+static rte_spinlock_t lock_entry_lock = RTE_SPINLOCK_INITIALIZER;
+
+int
+register_lock_callback(uint16_t port_id,
+ rte_eth_dev_lock_callback_t callback,
+ void *user_args)
+{
+ struct lock_entry *le;
+
+ rte_spinlock_lock(&lock_entry_lock);
+
+ TAILQ_FOREACH(le, &lock_entry_list, next) {
+ if (le->port_id == port_id &&
+ le->callback == callback &&
+ le->user_args == user_args)
+ break;
+ }
+
+ if (!le) {
+ le = calloc(1, sizeof(struct lock_entry));
+ if (!le) {
+ rte_spinlock_unlock(&lock_entry_lock);
+ return -ENOMEM;
+ }
+ le->callback = callback;
+ le->port_id = port_id;
+ le->user_args = user_args;
+ TAILQ_INSERT_TAIL(&lock_entry_list, le, next);
+ }
+ le->ref_count++;
+
+ rte_spinlock_unlock(&lock_entry_lock);
+ return 0;
+}
+
+int
+unregister_lock_callback(uint16_t port_id,
+ rte_eth_dev_lock_callback_t callback,
+ void *user_args)
+{
+ struct lock_entry *le;
+ int ret = 0;
+
+ rte_spinlock_lock(&lock_entry_lock);
+
+ TAILQ_FOREACH(le, &lock_entry_list, next) {
+ if (le->port_id == port_id &&
+ le->callback == callback &&
+ le->user_args == user_args)
+ break;
+ }
+
+ if (le) {
+ le->ref_count--;
+ if (!le->ref_count) {
+ TAILQ_REMOVE(&lock_entry_list, le, next);
+ free(le);
+ }
+ } else {
+ ret = -ENOENT;
+ }
+
+ rte_spinlock_unlock(&lock_entry_lock);
+ return ret;
+}
+
+int
+process_lock_callbacks(uint16_t port_id)
+{
+ struct lock_entry *le;
+
+ rte_spinlock_lock(&lock_entry_lock);
+
+ TAILQ_FOREACH(le, &lock_entry_list, next) {
+ if (le->port_id != port_id)
+ continue;
+
+ if (le->callback(port_id, le->user_args)) {
+ rte_spinlock_unlock(&lock_entry_lock);
+ return -EBUSY;
+ }
+ }
+
+ rte_spinlock_unlock(&lock_entry_lock);
+ return 0;
+}
diff --git a/lib/librte_ethdev/rte_ethdev_lock.h b/lib/librte_ethdev/rte_ethdev_lock.h
new file mode 100644
index 000000000..7b370c926
--- /dev/null
+++ b/lib/librte_ethdev/rte_ethdev_lock.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _RTE_ETHDEV_LOCK_H_
+#define _RTE_ETHDEV_LOCK_H_
+
+#include "rte_ethdev.h"
+
+int
+register_lock_callback(uint16_t port_id,
+ rte_eth_dev_lock_callback_t callback,
+ void *user_args);
+
+int
+unregister_lock_callback(uint16_t port_id,
+ rte_eth_dev_lock_callback_t callback,
+ void *user_args);
+
+int
+process_lock_callbacks(uint16_t port_id);
+
+#endif
diff --git a/lib/librte_ethdev/rte_ethdev_mp.c b/lib/librte_ethdev/rte_ethdev_mp.c
index 8ede8151d..e23c8b010 100644
--- a/lib/librte_ethdev/rte_ethdev_mp.c
+++ b/lib/librte_ethdev/rte_ethdev_mp.c
@@ -4,6 +4,7 @@

#include "rte_ethdev_driver.h"
#include "rte_ethdev_mp.h"
+#include "rte_ethdev_lock.h"

static int detach_on_secondary(uint16_t port_id)
{
@@ -101,7 +102,7 @@ static int handle_primary_request(const struct rte_mp_msg *msg, const void *peer
ret = attach_on_secondary(req->devargs, req->port_id);
break;
case REQ_TYPE_PRE_DETACH:
- ret = 0;
+ ret = process_lock_callbacks(req->port_id);
break;
case REQ_TYPE_DETACH:
case REQ_TYPE_ATTACH_ROLLBACK:
--
2.13.6
Qi Zhang
2018-06-07 12:38:30 UTC
Permalink
Add driver API rte_eth_release_port_local to support the requirement
that an ethdev only be released on secondary process, so only local
state be set to unused , share data will not be reset so primary
process can still use it.

Signed-off-by: Qi Zhang <***@intel.com>
---
lib/librte_ethdev/rte_ethdev.c | 24 +++++++++++++++++++++---
lib/librte_ethdev/rte_ethdev_driver.h | 13 +++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index cd4bfd3c6..ec14adb91 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -359,6 +359,23 @@ rte_eth_dev_attach_secondary(const char *name)
}

int
+rte_eth_dev_release_port_local(struct rte_eth_dev *eth_dev)
+{
+ if (eth_dev == NULL)
+ return -EINVAL;
+
+ _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_DESTROY, NULL);
+
+ rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
+
+ eth_dev->state = RTE_ETH_DEV_UNUSED;
+
+ rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
+
+ return 0;
+}
+
+int
rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
{
if (eth_dev == NULL)
@@ -370,9 +387,10 @@ rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)

rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);

- eth_dev->state = RTE_ETH_DEV_UNUSED;
-
- memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
+ if (eth_dev->state != RTE_ETH_DEV_UNUSED) {
+ eth_dev->state = RTE_ETH_DEV_UNUSED;
+ memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
+ }

rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);

diff --git a/lib/librte_ethdev/rte_ethdev_driver.h b/lib/librte_ethdev/rte_ethdev_driver.h
index c9c825e3f..261335426 100644
--- a/lib/librte_ethdev/rte_ethdev_driver.h
+++ b/lib/librte_ethdev/rte_ethdev_driver.h
@@ -70,6 +70,19 @@ int rte_eth_dev_release_port(struct rte_eth_dev *eth_dev);

/**
* @internal
+ * Release the specified ethdev port in local process, only set to ethdev
+ * state to unused, but not reset share data since it assume other process
+ * is still using it, typically it is called by secondary process.
+ *
+ * @param eth_dev
+ * The *eth_dev* pointer is the address of the *rte_eth_dev* structure.
+ * @return
+ * - 0 on success, negative on error
+ */
+int rte_eth_dev_release_port_local(struct rte_eth_dev *eth_dev);
+
+/**
+ * @internal
* Release device queues and clear its configuration to force the user
* application to reconfigure it. It is for internal use only.
*
--
2.13.6
Qi Zhang
2018-06-07 12:38:37 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/e1000/igb_ethdev.c | 9 +++++++++
1 file changed, 9 insertions(+)

diff --git a/drivers/net/e1000/igb_ethdev.c b/drivers/net/e1000/igb_ethdev.c
index edc7be319..bd2b2d218 100644
--- a/drivers/net/e1000/igb_ethdev.c
+++ b/drivers/net/e1000/igb_ethdev.c
@@ -1089,6 +1089,15 @@ static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,

static int eth_igb_pci_remove(struct rte_pci_device *pci_dev)
{
+ struct rte_eth_dev *ethdev =
+ rte_eth_dev_allocated(pci_dev->device.name);
+
+ if (!ethdev)
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit);
}
--
2.13.6
Qi Zhang
2018-06-07 12:38:45 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/pcap/rte_eth_pcap.c | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/drivers/net/pcap/rte_eth_pcap.c b/drivers/net/pcap/rte_eth_pcap.c
index 6bd4a7d79..4c366a92b 100644
--- a/drivers/net/pcap/rte_eth_pcap.c
+++ b/drivers/net/pcap/rte_eth_pcap.c
@@ -925,6 +925,7 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &ops;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -1016,6 +1017,7 @@ static int
pmd_pcap_remove(struct rte_vdev_device *dev)
{
struct rte_eth_dev *eth_dev = NULL;
+ const char *name;

PMD_LOG(INFO, "Closing pcap ethdev on numa socket %d",
rte_socket_id());
@@ -1023,11 +1025,22 @@ pmd_pcap_remove(struct rte_vdev_device *dev)
if (!dev)
return -1;

+ name = rte_vdev_device_name(dev);
/* reserve an ethdev entry */
- eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev));
+ eth_dev = rte_eth_dev_allocated(name);
if (eth_dev == NULL)
return -1;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
rte_free(eth_dev->data->dev_private);

rte_eth_dev_release_port(eth_dev);
--
2.13.6
Qi Zhang
2018-06-07 12:38:33 UTC
Permalink
This patch cover the multi-process hotplug case when a share device
attach/detach request be issued from secondary process, the implementation
references malloc_mp.c.

device attach on secondary:
a) seconary send asycn request to primary and wait on a condition
which will be released by matched response from primary.
b) primary receive the request and attach the new device if failed
goto i).
c) primary forward attach request to all secondary as async request
(because this in mp thread context, use sync request will deadlock)
d) secondary receive request and attach device and send reply.
e) primary check the reply if all success go to j).
f) primary send attach rollback async request to all secondary.
g) secondary receive the request and detach device and send reply.
h) primary receive the reply and detach device as rollback action.
i) send fail response to secondary, goto k).
j) send success response to secondary.
k) secondary process receive response and return.

device detach on secondary:
a) secondary send async request to primary and wait on a condition
which will be released by matched response from primary.
b) primary receive the request and perform pre-detach check, if device
is locked, goto j).
c) primary send pre-detach async request to all secondary.
d) secondary perform pre-detach check and send reply.
e) primary check the reply if any fail goto j).
f) primary send detach async request to all secondary
g) secondary detach the device and send reply
h) primary detach the device.
i) send success response to secondary, goto k).
j) send fail response to secondary.
k) secondary process receive response and return.

Signed-off-by: Qi Zhang <***@intel.com>
---
lib/librte_ethdev/rte_ethdev_mp.c | 477 +++++++++++++++++++++++++++++++++++++-
lib/librte_ethdev/rte_ethdev_mp.h | 1 +
2 files changed, 468 insertions(+), 10 deletions(-)

diff --git a/lib/librte_ethdev/rte_ethdev_mp.c b/lib/librte_ethdev/rte_ethdev_mp.c
index e23c8b010..6dbd23fa5 100644
--- a/lib/librte_ethdev/rte_ethdev_mp.c
+++ b/lib/librte_ethdev/rte_ethdev_mp.c
@@ -2,10 +2,69 @@
* Copyright(c) 2010-2018 Intel Corporation
*/

+#include <sys/time.h>
+
#include "rte_ethdev_driver.h"
#include "rte_ethdev_mp.h"
#include "rte_ethdev_lock.h"

+enum req_state {
+ REQ_STATE_INACTIVE = 0,
+ REQ_STATE_ACTIVE,
+ REQ_STATE_COMPLETE
+};
+
+struct mp_request {
+ TAILQ_ENTRY(mp_request) next;
+ struct eth_dev_mp_req user_req; /**< contents of request */
+ pthread_cond_t cond; /**< variable we use to time out on this request */
+ enum req_state state; /**< indicate status of this request */
+};
+
+/*
+ * We could've used just a single request, but it may be possible for
+ * secondaries to timeout earlier than the primary, and send a new request while
+ * primary is still expecting replies to the old one. Therefore, each new
+ * request will get assigned a new ID, which is how we will distinguish between
+ * expected and unexpected messages.
+ */
+TAILQ_HEAD(mp_request_list, mp_request);
+static struct {
+ struct mp_request_list list;
+ pthread_mutex_t lock;
+} mp_request_list = {
+ .list = TAILQ_HEAD_INITIALIZER(mp_request_list.list),
+ .lock = PTHREAD_MUTEX_INITIALIZER
+};
+
+#define MP_TIMEOUT_S 5 /**< 5 seconds timeouts */
+
+static struct mp_request *
+find_request_by_id(uint64_t id)
+{
+ struct mp_request *req;
+
+ TAILQ_FOREACH(req, &mp_request_list.list, next) {
+ if (req->user_req.id == id)
+ break;
+ }
+ return req;
+}
+
+static uint64_t
+get_unique_id(void)
+{
+ uint64_t id;
+
+ do {
+ id = rte_rand();
+ } while (find_request_by_id(id) != NULL);
+ return id;
+}
+
+static int
+send_request_to_secondary_async(const struct eth_dev_mp_req *req);
+
static int detach_on_secondary(uint16_t port_id)
{
struct rte_device *dev;
@@ -72,18 +131,325 @@ static int attach_on_secondary(const char *devargs, uint16_t port_id)
return 0;
}

-static int handle_secondary_request(const struct rte_mp_msg *msg, const void *peer)
+static int
+check_reply(const struct eth_dev_mp_req *req,
+ const struct rte_mp_reply *reply)
+{
+ struct eth_dev_mp_req *resp;
+ int i;
+
+ if (reply->nb_received != reply->nb_sent)
+ return -EINVAL;
+
+ for (i = 0; i < reply->nb_received; i++) {
+ resp = (struct eth_dev_mp_req *)reply->msgs[i].param;
+
+ if (resp->t != req->t) {
+ ethdev_log(ERR, "Unexpected response to async request\n");
+ return -EINVAL;
+ }
+
+ if (resp->id != req->id) {
+ ethdev_log(ERR, "response to wrong async request\n");
+ return -EINVAL;
+ }
+
+ if (resp->result)
+ return resp->result;
+ }
+
+ return 0;
+}
+
+static int
+send_response_to_secondary(const struct eth_dev_mp_req *req, int result)
+{
+ struct rte_mp_msg resp_msg = {0};
+ struct eth_dev_mp_req *resp =
+ (struct eth_dev_mp_req *)resp_msg.param;
+ int ret = 0;
+
+ resp_msg.len_param = sizeof(*resp);
+ strcpy(resp_msg.name, ETH_DEV_MP_ACTION_RESPONSE);
+ memcpy(resp, req, sizeof(*req));
+ resp->result = result;
+
+ ret = rte_mp_sendmsg(&resp_msg);
+ if (ret)
+ ethdev_log(ERR, "failed to send response to secondary\n");
+
+ return ret;
+}
+
+static int
+handle_async_attach_response(const struct rte_mp_msg *request,
+ const struct rte_mp_reply *reply)
+{
+ struct mp_request *entry;
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)request->param;
+ struct eth_dev_mp_req tmp_req;
+ int ret = 0;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (!entry) {
+ ethdev_log(ERR, "wrong request ID\n");
+ ret = -EINVAL;
+ goto finish;
+ }
+
+ ret = check_reply(req, reply);
+ if (ret) {
+ tmp_req = *req;
+ tmp_req.t = REQ_TYPE_ATTACH_ROLLBACK;
+
+ ret = send_request_to_secondary_async(&tmp_req);
+ if (ret) {
+ ethdev_log(ERR, "couldn't send async request\n");
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+ }
+ } else {
+ send_response_to_secondary(req, 0);
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+ }
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ return ret;
+}
+
+static int
+handle_async_detach_response(const struct rte_mp_msg *request,
+ const struct rte_mp_reply *reply)
+{
+ struct mp_request *entry;
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)request->param;
+ int ret = 0;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (!entry) {
+ ethdev_log(ERR, "wrong request ID\n");
+ ret = -EINVAL;
+ goto finish;
+ }
+
+ ret = check_reply(req, reply);
+ if (ret) {
+ send_response_to_secondary(req, ret);
+ } else {
+ do_eth_dev_detach(req->port_id);
+ send_response_to_secondary(req, 0);
+ }
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ return ret;
+}
+
+static int
+handle_async_pre_detach_response(const struct rte_mp_msg *request,
+ const struct rte_mp_reply *reply)
+{
+ struct mp_request *entry;
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)request->param;
+ struct eth_dev_mp_req tmp_req;
+ int ret = 0;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (!entry) {
+ ethdev_log(ERR, "wrong request ID\n");
+ ret = -EINVAL;
+ goto finish;
+ }
+
+ ret = check_reply(req, reply);
+ if (!ret) {
+ tmp_req = *req;
+ tmp_req.t = REQ_TYPE_DETACH;
+
+ ret = send_request_to_secondary_async(&tmp_req);
+ if (ret) {
+ ethdev_log(ERR, "couldn't send async request\n");
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+ }
+ } else {
+ send_response_to_secondary(req, ret);
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+ }
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ return 0;
+}
+
+static int
+handle_async_rollback_response(const struct rte_mp_msg *request,
+ const struct rte_mp_reply *reply __rte_unused)
+{
+ struct mp_request *entry;
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)request->param;
+ int ret = 0;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (!entry) {
+ ethdev_log(ERR, "wrong request ID\n");
+ ret = -EINVAL;
+ goto finish;
+ }
+
+ /* we have nothing to do if rollback still fail, just detach */
+ do_eth_dev_detach(req->port_id);
+ /* send response to secondary with the reason of rollback */
+ send_response_to_secondary(req, req->result);
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+ free(entry);
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ return ret;
+}
+
+static int
+send_request_to_secondary_async(const struct eth_dev_mp_req *req)
{
- (void)msg;
- (void)(peer);
- return -ENOTSUP;
+ struct rte_mp_msg mp_req = {0};
+ struct timespec ts = {.tv_sec = MP_TIMEOUT_S, .tv_nsec = 0};
+ rte_mp_async_reply_t clb;
+ struct mp_request *entry;
+ int ret = 0;
+
+ memcpy(mp_req.param, req, sizeof(*req));
+ mp_req.len_param = sizeof(*req);
+ strcpy(mp_req.name, ETH_DEV_MP_ACTION_REQUEST);
+
+ if (req->t == REQ_TYPE_ATTACH)
+ clb = handle_async_attach_response;
+ else if (req->t == REQ_TYPE_PRE_DETACH)
+ clb = handle_async_pre_detach_response;
+ else if (req->t == REQ_TYPE_DETACH)
+ clb = handle_async_detach_response;
+ else if (req->t == REQ_TYPE_ATTACH_ROLLBACK)
+ clb = handle_async_rollback_response;
+ else
+ return -1;
+ do {
+ ret = rte_mp_request_async(&mp_req, &ts, clb);
+ } while (ret != 0 && rte_errno == EEXIST);
+
+ if (ret)
+ ethdev_log(ERR, "couldn't send async request\n");
+ entry = find_request_by_id(req->id);
+ (void)entry;
+ return ret;
+}
+
+static int handle_secondary_request(const struct rte_mp_msg *msg,
+ const void *peer __rte_unused)
+{
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)msg->param;
+ struct eth_dev_mp_req tmp_req;
+ struct mp_request *entry;
+ uint16_t port_id;
+ int ret = 0;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (entry) {
+ ethdev_log(ERR, "duplicate request id\n");
+ ret = -EEXIST;
+ goto finish;
+ }
+
+ entry = malloc(sizeof(*entry));
+ if (entry == NULL) {
+ ethdev_log(ERR, "not enough memory to allocate request entry\n");
+ ret = -ENOMEM;
+ goto finish;
+ }
+
+ if (req->t == REQ_TYPE_ATTACH) {
+ ret = do_eth_dev_attach(req->devargs, &port_id);
+ if (!ret) {
+ tmp_req = *req;
+ tmp_req.port_id = port_id;
+ ret = send_request_to_secondary_async(&tmp_req);
+ }
+ } else if (req->t == REQ_TYPE_DETACH) {
+ if (!rte_eth_dev_is_valid_port(req->port_id))
+ ret = -EINVAL;
+ if (!ret)
+ ret = process_lock_callbacks(req->port_id);
+ if (!ret) {
+ tmp_req = *req;
+ tmp_req.t = REQ_TYPE_PRE_DETACH;
+ ret = send_request_to_secondary_async(&tmp_req);
+ }
+ } else {
+ ethdev_log(ERR, "unsupported secondary to primary request\n");
+ ret = -ENOTSUP;
+ goto finish;
+ }
+
+ if (ret) {
+ ret = send_response_to_secondary(req, ret);
+ if (ret) {
+ ethdev_log(ERR, "failed to send response to secondary\n");
+ goto finish;
+ }
+ } else {
+ memcpy(&entry->user_req, req, sizeof(*req));
+ entry->state = REQ_STATE_ACTIVE;
+ TAILQ_INSERT_TAIL(&mp_request_list.list, entry, next);
+ entry = NULL;
+ }
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ if (entry)
+ free(entry);
+ return ret;
}

-static int handle_primary_response(const struct rte_mp_msg *msg, const void *peer)
+static int handle_primary_response(const struct rte_mp_msg *msg,
+ const void *peer __rte_unused)
{
- (void)msg;
- (void)(peer);
- return -ENOTSUP;
+ const struct eth_dev_mp_req *req =
+ (const struct eth_dev_mp_req *)msg->param;
+ struct mp_request *entry;
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ entry = find_request_by_id(req->id);
+ if (entry) {
+ entry->user_req.result = req->result;
+ entry->user_req.port_id = req->port_id;
+ entry->state = REQ_STATE_COMPLETE;
+
+ pthread_cond_signal(&entry->cond);
+ }
+
+ pthread_mutex_unlock(&mp_request_list.lock);
+
+ return 0;
}

static int handle_primary_request(const struct rte_mp_msg *msg, const void *peer)
@@ -124,10 +490,101 @@ static int handle_primary_request(const struct rte_mp_msg *msg, const void *peer
return 0;
}

+/**
+ * secondary to primary request.
+ *
+ * device attach:
+ * a) seconary send request to primary.
+ * b) primary attach the new device if failed goto i).
+ * c) primary forward attach request to all secondary.
+ * d) secondary receive request and attach device and send reply.
+ * e) primary check the reply if all success go to j).
+ * f) primary send attach rollback request to all secondary.
+ * g) secondary receive the request and detach device and send reply.
+ * h) primary receive the reply and detach device as rollback action.
+ * i) send fail response to secondary, goto k).
+ * j) send success response to secondary.
+ * k) end.
+
+ * device detach:
+ * a) secondary send request to primary.
+ * b) primary perform pre-detach check, if device is locked, got j).
+ * c) primary send pre-detach check request to all secondary.
+ * d) secondary perform pre-detach check and send reply.
+ * e) primary check the reply if any fail goto j).
+ * f) primary send detach request to all secondary
+ * g) secondary detach the device and send reply
+ * h) primary detach the device.
+ * i) send success response to secondary, goto k).
+ * j) send fail response to secondary.
+ * k) end.
+ */
int rte_eth_dev_request_to_primary(struct eth_dev_mp_req *req)
{
- (void)req;
- return -ENOTSUP;
+ struct rte_mp_msg msg = {0};
+ struct eth_dev_mp_req *msg_req = (struct eth_dev_mp_req *)msg.param;
+ struct mp_request *entry;
+ struct timespec ts = {0};
+ struct timeval now;
+ int ret = 0;
+
+ entry = malloc(sizeof(*entry));
+ if (entry == NULL) {
+ ethdev_log(ERR, "not enough memory to allocate request entry\n");
+ return -ENOMEM;
+ }
+
+ pthread_mutex_lock(&mp_request_list.lock);
+
+ ret = gettimeofday(&now, NULL);
+ if (ret) {
+ ethdev_log(ERR, "cannot get current time\n");
+ ret = -EINVAL;
+ goto finish;
+ }
+
+ ts.tv_nsec = (now.tv_usec * 1000) % 1000000000;
+ ts.tv_sec = now.tv_sec + MP_TIMEOUT_S +
+ (now.tv_usec * 1000) / 1000000000;
+
+ pthread_cond_init(&entry->cond, NULL);
+
+ msg.len_param = sizeof(*req);
+ strcpy(msg.name, ETH_DEV_MP_ACTION_REQUEST);
+
+ req->id = get_unique_id();
+
+ memcpy(msg_req, req, sizeof(*req));
+
+ ret = rte_mp_sendmsg(&msg);
+ if (ret) {
+ ethdev_log(ERR, "cannot send message to primary");
+ goto finish;
+ }
+
+ memcpy(&entry->user_req, req, sizeof(*req));
+
+ entry->state = REQ_STATE_ACTIVE;
+
+ TAILQ_INSERT_TAIL(&mp_request_list.list, entry, next);
+
+ do {
+ ret = pthread_cond_timedwait(&entry->cond,
+ &mp_request_list.lock, &ts);
+ } while (ret != 0 && ret != ETIMEDOUT);
+
+ if (entry->state != REQ_STATE_COMPLETE) {
+ RTE_LOG(ERR, EAL, "request time out\n");
+ ret = -ETIMEDOUT;
+ } else {
+ req->result = entry->user_req.result;
+ }
+ TAILQ_REMOVE(&mp_request_list.list, entry, next);
+
+finish:
+ pthread_mutex_unlock(&mp_request_list.lock);
+ free(entry);
+ return ret;
}

/**
diff --git a/lib/librte_ethdev/rte_ethdev_mp.h b/lib/librte_ethdev/rte_ethdev_mp.h
index c3e55dfec..6d10dfdad 100644
--- a/lib/librte_ethdev/rte_ethdev_mp.h
+++ b/lib/librte_ethdev/rte_ethdev_mp.h
@@ -18,6 +18,7 @@ enum eth_dev_req_type {
};

struct eth_dev_mp_req {
+ uint64_t id;
enum eth_dev_req_type t;
char devargs[MAX_DEV_ARGS_LEN];
uint16_t port_id;
--
2.13.6
Qi Zhang
2018-06-07 12:38:35 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/ixgbe/ixgbe_ethdev.c | 12 ++++++++++++
1 file changed, 12 insertions(+)

diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 87d2ad090..260640e50 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -1792,6 +1792,9 @@ static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev)
if (!ethdev)
return -ENODEV;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
return rte_eth_dev_destroy(ethdev, ixgbe_vf_representor_uninit);
else
@@ -1809,6 +1812,15 @@ static struct rte_pci_driver rte_ixgbe_pmd = {
static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct rte_pci_device *pci_dev)
{
+ struct rte_eth_dev *ethdev;
+
+ ethdev = rte_eth_dev_allocated(pci_dev->device.name);
+ if (!ethdev)
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
return rte_eth_dev_pci_generic_probe(pci_dev,
sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init);
}
--
2.13.6
Qi Zhang
2018-06-07 12:38:40 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/bonding/rte_eth_bond_pmd.c | 11 +++++++++++
1 file changed, 11 insertions(+)

diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c
index 02d94b1b1..1221f62b2 100644
--- a/drivers/net/bonding/rte_eth_bond_pmd.c
+++ b/drivers/net/bonding/rte_eth_bond_pmd.c
@@ -3065,6 +3065,7 @@ bond_probe(struct rte_vdev_device *dev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &default_dev_ops;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -3171,6 +3172,16 @@ bond_remove(struct rte_vdev_device *dev)
if (eth_dev == NULL)
return -ENODEV;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
RTE_ASSERT(eth_dev->device == &dev->device);

internals = eth_dev->data->dev_private;
--
2.13.6
Qi Zhang
2018-06-07 12:38:29 UTC
Permalink
Implemented the bus ops scan_one, besides this improve the scan
efficiency in hotplug case, it aslo avoid sync IPC invoke (which
happens in vdev->scan on secondary process). The benifit is it
removes the potiential deadlock in the case when secondary process
receive a request from primary process to attach a new device, since
vdev->scan will be invoked on mp thread itself at this case.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/bus/vdev/vdev.c | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)

diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index 6139dd551..cdbd77df0 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -467,6 +467,35 @@ vdev_scan(void)
return 0;
}

+static struct rte_device *vdev_scan_one(struct rte_devargs *devargs)
+{
+ struct rte_vdev_device *dev = NULL;
+
+ dev = calloc(1, sizeof(*dev));
+ if (!dev) {
+ VDEV_LOG(ERR, "failed to allocate memory for new device");
+ return NULL;
+ }
+
+ rte_spinlock_recursive_lock(&vdev_device_list_lock);
+
+ if (find_vdev(devargs->name)) {
+ VDEV_LOG(ERR, "device %s already exist", devargs->name);
+ free(dev);
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+ return NULL;
+ }
+
+ dev->device.devargs = devargs;
+ dev->device.numa_node = SOCKET_ID_ANY;
+ dev->device.name = devargs->name;
+ TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
+
+ rte_spinlock_recursive_unlock(&vdev_device_list_lock);
+
+ return &dev->device;
+}
+
static int
vdev_probe(void)
{
@@ -531,6 +560,7 @@ vdev_unplug(struct rte_device *dev)

static struct rte_bus rte_vdev_bus = {
.scan = vdev_scan,
+ .scan_one = vdev_scan_one,
.probe = vdev_probe,
.find_device = vdev_find_device,
.plug = vdev_plug,
--
2.13.6
Qi Zhang
2018-06-07 12:38:44 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/octeontx/octeontx_ethdev.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)

diff --git a/drivers/net/octeontx/octeontx_ethdev.c b/drivers/net/octeontx/octeontx_ethdev.c
index 1eb453b21..b42b69896 100644
--- a/drivers/net/octeontx/octeontx_ethdev.c
+++ b/drivers/net/octeontx/octeontx_ethdev.c
@@ -1016,6 +1016,7 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,

eth_dev->tx_pkt_burst = octeontx_xmit_pkts;
eth_dev->rx_pkt_burst = octeontx_recv_pkts;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -1138,6 +1139,18 @@ octeontx_remove(struct rte_vdev_device *dev)
if (eth_dev == NULL)
return -ENODEV;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0) {
+ rte_eth_dev_release_port_local(eth_dev);
+ continue;
+ }
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
nic = octeontx_pmd_priv(eth_dev);
rte_event_dev_stop(nic->evdev);
PMD_INIT_LOG(INFO, "Closing octeontx device %s", octtx_name);
@@ -1148,6 +1161,9 @@ octeontx_remove(struct rte_vdev_device *dev)
rte_event_dev_close(nic->evdev);
}

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return 0;
+
/* Free FC resource */
octeontx_pko_fc_free();
--
2.13.6
Qi Zhang
2018-06-07 12:38:49 UTC
Permalink
The sample code demonstrate device (ethdev only) management
at multi-process envrionment. User can attach/detach a device
on primary process and see it is synced on secondary process
automatically, also user can lock a device to prevent it be
detached or unlock it to go back to default behaviour.

How to start?
./devmgm_mp --proc-type=auto
help
list
/* attach a af_packet vdev */
attach net_af_packet,iface=eth0
/* detach port 0 */
detach 0
/* attach a private af_packet vdev (secondary process only)*/
attachp net_af_packet,iface=eth0
/* detach a private device (secondary process only) */
detachp 0
/* lock port 0 */
lock 0
/* unlock port 0 */
unlock 0
Signed-off-by: Qi Zhang <***@intel.com>
---
examples/devmgm_mp/Makefile | 64 +++++++
examples/devmgm_mp/commands.c | 383 +++++++++++++++++++++++++++++++++++++++++
examples/devmgm_mp/commands.h | 10 ++
examples/devmgm_mp/main.c | 41 +++++
examples/devmgm_mp/meson.build | 11 ++
5 files changed, 509 insertions(+)
create mode 100644 examples/devmgm_mp/Makefile
create mode 100644 examples/devmgm_mp/commands.c
create mode 100644 examples/devmgm_mp/commands.h
create mode 100644 examples/devmgm_mp/main.c
create mode 100644 examples/devmgm_mp/meson.build

diff --git a/examples/devmgm_mp/Makefile b/examples/devmgm_mp/Makefile
new file mode 100644
index 000000000..e6c0cb0c5
--- /dev/null
+++ b/examples/devmgm_mp/Makefile
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+
+# binary name
+APP = devmgm_mp
+
+# all source are stored in SRCS-y
+SRCS-y := main.c commands.c
+
+# Build using pkg-config variables if possible
+$(shell pkg-config --exists libdpdk)
+ifeq ($(.SHELLSTATUS),0)
+
+all: shared
+.PHONY: shared static
+shared: build/$(APP)-shared
+ ln -sf $(APP)-shared build/$(APP)
+static: build/$(APP)-static
+ ln -sf $(APP)-static build/$(APP)
+
+PC_FILE := $(shell pkg-config --path libdpdk)
+CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
+LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
+LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
+
+build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
+ $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
+
+build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
+ $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
+
+build:
+ @mkdir -p $@
+
+.PHONY: clean
+clean:
+ rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
+ rmdir --ignore-fail-on-non-empty build
+
+else
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = devmgm_mp
+
+# all source are stored in SRCS-y
+SRCS-y := main.c commands.c
+
+CFLAGS += -O3
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS_parse_obj_list.o := -D_GNU_SOURCE
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
diff --git a/examples/devmgm_mp/commands.c b/examples/devmgm_mp/commands.c
new file mode 100644
index 000000000..145cb766e
--- /dev/null
+++ b/examples/devmgm_mp/commands.c
@@ -0,0 +1,383 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation.
+ * Copyright (c) 2009, Olivier MATZ <***@droids-corp.org>
+ * All rights reserved.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <netinet/in.h>
+#include <termios.h>
+#ifndef __linux__
+ #ifdef __FreeBSD__
+ #include <sys/socket.h>
+ #else
+ #include <net/socket.h>
+ #endif
+#endif
+
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_parse_ipaddr.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+#include <rte_ethdev.h>
+
+/**********************************************************/
+
+struct cmd_help_result {
+ cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ cmdline_printf(cl,
+ "commands:\n"
+ "- attach <devargs>\n"
+ "- detach <port_id>\n"
+ "- attachp <devargs>\n"
+ "- detachp <port_id>\n"
+ "- lock <port_id>\n"
+ "- unlock <port_id>\n"
+ "- list\n\n");
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+ TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+ .f = cmd_help_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "show help",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_help_help,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_quit_result {
+ cmdline_fixed_string_t quit;
+};
+
+static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ cmdline_quit(cl);
+}
+
+cmdline_parse_token_string_t cmd_quit_quit =
+ TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
+
+cmdline_parse_inst_t cmd_quit = {
+ .f = cmd_quit_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "quit",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_quit_quit,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_list_result {
+ cmdline_fixed_string_t list;
+};
+
+static void cmd_list_parsed(__attribute__((unused)) void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ uint16_t port_id;
+ char dev_name[RTE_DEV_NAME_MAX_LEN];
+
+ cmdline_printf(cl, "list all etherdev\n");
+
+ RTE_ETH_FOREACH_DEV(port_id) {
+ rte_eth_dev_get_name_by_port(port_id, dev_name);
+ /* Secondary process's ethdev->state may not be
+ * updated after detach on primary process, but
+ * ethdev->data should already be reset, so
+ * use strlen(dev_name) == 0 to know the port is
+ * not used.
+ *
+ * TODO: Secondary process should be informed when a
+ * port is released on primary through mp channel.
+ */
+ if (strlen(dev_name) > 0)
+ cmdline_printf(cl, "%d\t%s\n", port_id, dev_name);
+ else
+ printf("empty dev_name is not expected!\n");
+ }
+}
+
+cmdline_parse_token_string_t cmd_list_list =
+ TOKEN_STRING_INITIALIZER(struct cmd_list_result, list, "list");
+
+cmdline_parse_inst_t cmd_list = {
+ .f = cmd_list_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "list all devices",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_list_list,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_attach_result {
+ cmdline_fixed_string_t attach;
+ cmdline_fixed_string_t device;
+};
+
+static void cmd_dev_attach_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_attach_result *res = parsed_result;
+ uint16_t port_id;
+
+ if (!rte_eth_dev_attach(res->device, &port_id))
+ cmdline_printf(cl, "attached device %s at port %d\n",
+ res->device, port_id);
+ else
+ cmdline_printf(cl, "failed to attached device %s\n",
+ res->device);
+}
+
+cmdline_parse_token_string_t cmd_dev_attach_attach =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_attach_result, attach,
+ "attach");
+cmdline_parse_token_string_t cmd_dev_attach_device =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_attach_result, device, NULL);
+
+cmdline_parse_inst_t cmd_attach_device = {
+ .f = cmd_dev_attach_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "attach a device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_attach_attach,
+ (void *)&cmd_dev_attach_device,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_attachp_result {
+ cmdline_fixed_string_t attachp;
+ cmdline_fixed_string_t device;
+};
+
+static void cmd_dev_attachp_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_attachp_result *res = parsed_result;
+ uint16_t port_id;
+
+ if (!rte_eth_dev_attach_private(res->device, &port_id))
+ cmdline_printf(cl, "attached prviate device %s at port %d\n",
+ res->device, port_id);
+ else
+ cmdline_printf(cl, "failed to attached private device %s\n",
+ res->device);
+}
+
+cmdline_parse_token_string_t cmd_dev_attachp_attachp =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_attachp_result, attachp,
+ "attachp");
+cmdline_parse_token_string_t cmd_dev_attachp_device =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_attachp_result, device, NULL);
+
+cmdline_parse_inst_t cmd_attachp_device = {
+ .f = cmd_dev_attachp_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "attach a private device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_attachp_attachp,
+ (void *)&cmd_dev_attachp_device,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_detach_result {
+ cmdline_fixed_string_t detach;
+ uint16_t port_id;
+};
+
+static void cmd_dev_detach_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_detach_result *res = parsed_result;
+ uint16_t port_id = res->port_id;
+ char dev_name[RTE_DEV_NAME_MAX_LEN];
+
+ printf("detaching...\n");
+ if (!rte_eth_dev_detach(port_id, dev_name))
+ cmdline_printf(cl, "detached device at port %d\n",
+ port_id);
+ else
+ cmdline_printf(cl, "failed to dettached at port %d\n",
+ port_id);
+}
+
+cmdline_parse_token_string_t cmd_dev_detach_detach =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_detach_result, detach,
+ "detach");
+cmdline_parse_token_num_t cmd_dev_detach_port_id =
+ TOKEN_NUM_INITIALIZER(struct cmd_dev_detach_result, port_id, UINT16);
+
+cmdline_parse_inst_t cmd_detach_device = {
+ .f = cmd_dev_detach_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "detach a device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_detach_detach,
+ (void *)&cmd_dev_detach_port_id,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_detachp_result {
+ cmdline_fixed_string_t detachp;
+ uint16_t port_id;
+};
+
+static void cmd_dev_detachp_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_detachp_result *res = parsed_result;
+ uint16_t port_id = res->port_id;
+ char dev_name[RTE_DEV_NAME_MAX_LEN];
+
+ printf("detaching...\n");
+ if (!rte_eth_dev_detach_private(port_id, dev_name))
+ cmdline_printf(cl, "detached private device at port %d\n",
+ port_id);
+ else
+ cmdline_printf(cl, "failed to detach private device at port %d\n",
+ port_id);
+}
+
+cmdline_parse_token_string_t cmd_dev_detachp_detachp =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_detachp_result, detachp,
+ "detachp");
+cmdline_parse_token_num_t cmd_dev_detachp_port_id =
+ TOKEN_NUM_INITIALIZER(struct cmd_dev_detachp_result, port_id, UINT16);
+
+cmdline_parse_inst_t cmd_detachp_device = {
+ .f = cmd_dev_detachp_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "detach a private device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_detachp_detachp,
+ (void *)&cmd_dev_detachp_port_id,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_lock_result {
+ cmdline_fixed_string_t lock;
+ uint16_t port_id;
+};
+
+static void cmd_dev_lock_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_lock_result *res = parsed_result;
+ uint16_t port_id = res->port_id;
+ int ret = 0;
+
+ ret = rte_eth_dev_lock(res->port_id, NULL, NULL);
+ cmdline_printf(cl, "lock port %d, ret = %d\n", port_id, ret);
+}
+
+cmdline_parse_token_string_t cmd_dev_lock_lock =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_lock_result, lock, "lock");
+cmdline_parse_token_num_t cmd_dev_lock_port_id =
+ TOKEN_NUM_INITIALIZER(struct cmd_dev_lock_result, port_id, UINT16);
+
+cmdline_parse_inst_t cmd_lock_device = {
+ .f = cmd_dev_lock_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "lock a device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_lock_lock,
+ (void *)&cmd_dev_lock_port_id,
+ NULL,
+ },
+};
+
+/**********************************************************/
+
+struct cmd_dev_unlock_result {
+ cmdline_fixed_string_t unlock;
+ uint16_t port_id;
+};
+
+static void cmd_dev_unlock_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_dev_unlock_result *res = parsed_result;
+ uint16_t port_id = res->port_id;
+ int ret = 0;
+
+ ret = rte_eth_dev_unlock(res->port_id, NULL, NULL);
+ cmdline_printf(cl, "unlock port %d, ret = %d\n", port_id, ret);
+}
+
+cmdline_parse_token_string_t cmd_dev_unlock_unlock =
+ TOKEN_STRING_INITIALIZER(struct cmd_dev_unlock_result, unlock,
+ "unlock");
+cmdline_parse_token_num_t cmd_dev_unlock_port_id =
+ TOKEN_NUM_INITIALIZER(struct cmd_dev_unlock_result, port_id, UINT16);
+
+cmdline_parse_inst_t cmd_unlock_device = {
+ .f = cmd_dev_unlock_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "unlock a device",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_dev_unlock_unlock,
+ (void *)&cmd_dev_unlock_port_id,
+ NULL,
+ },
+};
+
+/**********************************************************/
+/**********************************************************/
+/****** CONTEXT (list of instruction) */
+
+cmdline_parse_ctx_t main_ctx[] = {
+ (cmdline_parse_inst_t *)&cmd_help,
+ (cmdline_parse_inst_t *)&cmd_quit,
+ (cmdline_parse_inst_t *)&cmd_list,
+ (cmdline_parse_inst_t *)&cmd_attach_device,
+ (cmdline_parse_inst_t *)&cmd_detach_device,
+ (cmdline_parse_inst_t *)&cmd_attachp_device,
+ (cmdline_parse_inst_t *)&cmd_detachp_device,
+ (cmdline_parse_inst_t *)&cmd_lock_device,
+ (cmdline_parse_inst_t *)&cmd_unlock_device,
+ NULL,
+};
diff --git a/examples/devmgm_mp/commands.h b/examples/devmgm_mp/commands.h
new file mode 100644
index 000000000..791204547
--- /dev/null
+++ b/examples/devmgm_mp/commands.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#ifndef _COMMANDS_H_
+#define _COMMANDS_H_
+
+extern cmdline_parse_ctx_t main_ctx[];
+
+#endif /* _COMMANDS_H_ */
diff --git a/examples/devmgm_mp/main.c b/examples/devmgm_mp/main.c
new file mode 100644
index 000000000..f2f2e5a2f
--- /dev/null
+++ b/examples/devmgm_mp/main.c
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation.
+ * Copyright (c) 2009, Olivier MATZ <***@droids-corp.org>
+ * All rights reserved.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <termios.h>
+#include <sys/queue.h>
+
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline.h>
+
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_debug.h>
+
+#include "commands.h"
+
+int main(int argc, char **argv)
+{
+ int ret;
+ struct cmdline *cl;
+
+ ret = rte_eal_init(argc, argv);
+ if (ret < 0)
+ rte_panic("Cannot init EAL\n");
+
+ cl = cmdline_stdin_new(main_ctx, "example> ");
+ if (cl == NULL)
+ rte_panic("Cannot create cmdline instance\n");
+ cmdline_interact(cl);
+ cmdline_stdin_exit(cl);
+
+ return 0;
+}
diff --git a/examples/devmgm_mp/meson.build b/examples/devmgm_mp/meson.build
new file mode 100644
index 000000000..f916eb9af
--- /dev/null
+++ b/examples/devmgm_mp/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+sources = files(
+ 'commands.c', 'main.c'
+)
--
2.13.6
Qi Zhang
2018-06-07 12:38:41 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/failsafe/failsafe.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)

diff --git a/drivers/net/failsafe/failsafe.c b/drivers/net/failsafe/failsafe.c
index eafbb75df..aa676069d 100644
--- a/drivers/net/failsafe/failsafe.c
+++ b/drivers/net/failsafe/failsafe.c
@@ -328,6 +328,7 @@ rte_pmd_failsafe_probe(struct rte_vdev_device *vdev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &failsafe_ops;
+ eth_dev->device = &vdev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -338,10 +339,25 @@ rte_pmd_failsafe_probe(struct rte_vdev_device *vdev)
static int
rte_pmd_failsafe_remove(struct rte_vdev_device *vdev)
{
+ struct rte_eth_dev *eth_dev;
const char *name;

name = rte_vdev_device_name(vdev);
INFO("Uninitializing " FAILSAFE_DRIVER_NAME " for %s", name);
+
+ eth_dev = rte_eth_dev_allocated(name);
+ if (!eth_dev)
+ return -ENODEV;
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(vdev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario.
+ */
+ }
+
return fs_rte_eth_free(name);
}
--
2.13.6
Qi Zhang
2018-06-07 12:38:39 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/af_packet/rte_eth_af_packet.c | 11 +++++++++++
1 file changed, 11 insertions(+)

diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c
index ea47abbf8..e1afbfc14 100644
--- a/drivers/net/af_packet/rte_eth_af_packet.c
+++ b/drivers/net/af_packet/rte_eth_af_packet.c
@@ -935,6 +935,7 @@ rte_pmd_af_packet_probe(struct rte_vdev_device *dev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &ops;
+ eth_dev->device = &dev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -986,6 +987,16 @@ rte_pmd_af_packet_remove(struct rte_vdev_device *dev)
if (eth_dev == NULL)
return -1;

+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(dev)) == 0)
+ return rte_eth_dev_release_port_local(eth_dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
internals = eth_dev->data->dev_private;
for (q = 0; q < internals->nb_queues; q++) {
rte_free(internals->rx_queue[q].rd);
--
2.13.6
Qi Zhang
2018-06-07 12:38:46 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/softnic/rte_eth_softnic.c | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/drivers/net/softnic/rte_eth_softnic.c b/drivers/net/softnic/rte_eth_softnic.c
index 6b3c13e5c..fdb2f0825 100644
--- a/drivers/net/softnic/rte_eth_softnic.c
+++ b/drivers/net/softnic/rte_eth_softnic.c
@@ -750,6 +750,7 @@ pmd_probe(struct rte_vdev_device *vdev)
}
/* TODO: request info from primary to set up Rx and Tx */
eth_dev->dev_ops = &pmd_ops;
+ eth_dev->device = &vdev->device;
rte_eth_dev_probing_finish(eth_dev);
return 0;
}
@@ -803,17 +804,29 @@ pmd_remove(struct rte_vdev_device *vdev)
{
struct rte_eth_dev *dev = NULL;
struct pmd_internals *p;
+ const char *name;

if (!vdev)
return -EINVAL;

- PMD_LOG(INFO, "Removing device \"%s\"",
- rte_vdev_device_name(vdev));
+ name = rte_vdev_device_name(vdev);
+ PMD_LOG(INFO, "Removing device \"%s\"", name);

/* Find the ethdev entry */
- dev = rte_eth_dev_allocated(rte_vdev_device_name(vdev));
+ dev = rte_eth_dev_allocated(name);
if (dev == NULL)
return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* detach device on local pprocess only */
+ if (strlen(rte_vdev_device_args(vdev)) == 0)
+ return rte_eth_dev_release_port_local(dev);
+ /**
+ * else this is a private device for current process
+ * so continue with normal detach scenario
+ */
+ }
+
p = dev->data->dev_private;

/* Free device data structures*/
--
2.13.6
Qi Zhang
2018-06-07 12:38:36 UTC
Permalink
Previously, detach port on secondary process will mess primary
process and cause same device can't be attached again, by take
advantage of rte_eth_release_port_local, we can support this with
minor change.

Signed-off-by: Qi Zhang <***@intel.com>
---
drivers/net/e1000/em_ethdev.c | 9 +++++++++
1 file changed, 9 insertions(+)

diff --git a/drivers/net/e1000/em_ethdev.c b/drivers/net/e1000/em_ethdev.c
index 7039dc100..e626cb10c 100644
--- a/drivers/net/e1000/em_ethdev.c
+++ b/drivers/net/e1000/em_ethdev.c
@@ -349,6 +349,15 @@ static int eth_em_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,

static int eth_em_pci_remove(struct rte_pci_device *pci_dev)
{
+ struct rte_eth_dev *ethdev =
+ rte_eth_dev_allocated(pci_dev->device.name);
+
+ if (!ethdev)
+ return -ENODEV;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return rte_eth_dev_release_port_local(ethdev);
+
return rte_eth_dev_pci_generic_remove(pci_dev, eth_em_dev_uninit);
}
--
2.13.6
Loading...