mir/ovs
2
0
Fork 0
mirror of https://github.com/openvswitch/ovs synced 2025-08-30 22:05:19 +00:00
Code Issues Releases Wiki Activity

netdev-dpdk: vHost client mode and reconnect

Browse Source 
Until now, vHost ports in OVS have only been able to operate in 'server'
mode whereby OVS creates and manages the vHost socket and essentially
acts as the vHost 'server'. With this commit a new mode, 'client' mode,
is available. In this mode, OVS acts as the vHost 'client' and connects
to the socket created and managed by QEMU which now acts as the vHost
'server'. This mode allows for reconnect capability, which allows a
vHost port to resume normal connectivity in event of switch reset.

By default dpdkvhostuser ports still operate in 'server' mode. That is
unless a valid 'vhost-server-path' is specified for a device like so:

ovs-vsctl set Interface dpdkvhostuser0
options:vhost-server-path=/path/to/socket

'vhost-server-path' represents the full path of the vhost user socket
that has been or will be created by QEMU. Once specified, the port stays
in 'client' mode for the remainder of its lifetime.

QEMU v2.7.0+ is required when using OVS in vHost client mode and QEMU in
vHost server mode.

Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Daniele Di Proietto <diproiettod@vmware.com>
This commit is contained in:
Ciara Loftus
2016-08-15 16:11:26 +01:00
committed by Daniele Di Proietto
parent 53f50d2400
commit c1ff66ac80
4 changed files with 165 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
lib/netdev-dpdk.c
View File

@@ -356,10 +356,9 @@ struct netdev_dpdk {
/* True if vHost device is 'up' and has been reconfigured at least once */
bool vhost_reconfigured;
/* Identifier used to distinguish vhost devices from each other. It does
* not change during the lifetime of a struct netdev_dpdk. It can be read
* without holding any mutex. */
const char vhost_id[PATH_MAX];
/* Identifiers used to distinguish vhost devices from each other. */
char vhost_server_id[PATH_MAX];
char vhost_client_id[PATH_MAX];
/* In dpdk_list. */
struct ovs_list list_node OVS_GUARDED_BY(dpdk_mutex);
@@ -378,6 +377,9 @@ struct netdev_dpdk {
/* Socket ID detected when vHost device is brought up */
int requested_socket_id;
/* Denotes whether vHost port is client/server mode */
uint64_t vhost_driver_flags;
/* Ingress Policer */
OVSRCU_TYPE(struct ingress_policer *) ingress_policer;
uint32_t policer_rate;
@@ -812,6 +814,8 @@ netdev_dpdk_init(struct netdev *netdev, unsigned int port_no,
dev->max_packet_len = MTU_TO_FRAME_LEN(dev->mtu);
ovsrcu_index_init(&dev->vid, -1);
dev->vhost_reconfigured = false;
/* initialise vHost port in server mode */
dev->vhost_driver_flags &= ~RTE_VHOST_USER_CLIENT;
err = netdev_dpdk_mempool_configure(dev);
if (err) {
@@ -874,13 +878,22 @@ dpdk_dev_parse_name(const char dev_name[], const char prefix[],
}
}
/* Returns a pointer to the relevant vHost socket ID depending on the mode in
* use */
static char *
get_vhost_id(struct netdev_dpdk *dev)
OVS_REQUIRES(dev->mutex)
{
return dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT ?
dev->vhost_client_id : dev->vhost_server_id;
}
static int
netdev_dpdk_vhost_construct(struct netdev *netdev)
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
const char *name = netdev->name;
int err;
uint64_t flags = 0;
/* 'name' is appended to 'vhost_sock_dir' and used to create a socket in
* the file system. '/' or '\' would traverse directories, so they're not
@@ -898,19 +911,24 @@ netdev_dpdk_vhost_construct(struct netdev *netdev)
ovs_mutex_lock(&dpdk_mutex);
/* Take the name of the vhost-user port and append it to the location where
* the socket is to be created, then register the socket.
* the socket is to be created, then register the socket. Sockets are
* registered initially in 'server' mode.
*/
snprintf(CONST_CAST(char *, dev->vhost_id), sizeof dev->vhost_id, "%s/%s",
snprintf(dev->vhost_server_id, sizeof dev->vhost_server_id, "%s/%s",
vhost_sock_dir, name);
err = rte_vhost_driver_register(dev->vhost_id, flags);
err = rte_vhost_driver_register(dev->vhost_server_id,
dev->vhost_driver_flags);
if (err) {
VLOG_ERR("vhost-user socket device setup failure for socket %s\n",
dev->vhost_id);
dev->vhost_server_id);
} else {
fatal_signal_add_file_to_unlink(dev->vhost_id);
VLOG_INFO("Socket %s created for vhost-user port %s\n",
dev->vhost_id, name);
if (!(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)) {
/* OVS server mode - add this socket to list for deletion */
fatal_signal_add_file_to_unlink(dev->vhost_server_id);
VLOG_INFO("Socket %s created for vhost-user port %s\n",
dev->vhost_server_id, name);
}
err = netdev_dpdk_init(netdev, -1, DPDK_DEV_VHOST);
}
@@ -964,28 +982,31 @@ netdev_dpdk_destruct(struct netdev *netdev)
* try to acquire 'dpdk_mutex' and possibly 'dev->mutex'. To avoid a
* deadlock, none of the mutexes must be held while calling this function. */
static int
dpdk_vhost_driver_unregister(struct netdev_dpdk *dev)
dpdk_vhost_driver_unregister(struct netdev_dpdk *dev OVS_UNUSED,
char *vhost_id)
OVS_EXCLUDED(dpdk_mutex)
OVS_EXCLUDED(dev->mutex)
{
return rte_vhost_driver_unregister(dev->vhost_id);
return rte_vhost_driver_unregister(vhost_id);
}
static void
netdev_dpdk_vhost_destruct(struct netdev *netdev)
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
char *vhost_id;
ovs_mutex_lock(&dpdk_mutex);
ovs_mutex_lock(&dev->mutex);
/* Guest becomes an orphan if still attached. */
if (netdev_dpdk_get_vid(dev) >= 0) {
if (netdev_dpdk_get_vid(dev) >= 0
&& !(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)) {
VLOG_ERR("Removing port '%s' while vhost device still attached.",
netdev->name);
VLOG_ERR("To restore connectivity after re-adding of port, VM on socket"
" '%s' must be restarted.",
dev->vhost_id);
get_vhost_id(dev));
}
free(ovsrcu_get_protected(struct ingress_policer *,
@@ -995,14 +1016,18 @@ netdev_dpdk_vhost_destruct(struct netdev *netdev)
ovs_list_remove(&dev->list_node);
dpdk_mp_put(dev->dpdk_mp);
vhost_id = xstrdup(get_vhost_id(dev));
ovs_mutex_unlock(&dev->mutex);
ovs_mutex_unlock(&dpdk_mutex);
if (dpdk_vhost_driver_unregister(dev)) {
VLOG_ERR("Unable to remove vhost-user socket %s", dev->vhost_id);
} else {
fatal_signal_remove_file_to_unlink(dev->vhost_id);
if (dpdk_vhost_driver_unregister(dev, vhost_id)) {
VLOG_ERR("Unable to remove vhost-user socket %s", vhost_id);
} else if (!(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)) {
/* OVS server mode - remove this socket from list for deletion */
fatal_signal_remove_file_to_unlink(vhost_id);
}
free(vhost_id);
}
static void
@@ -1082,6 +1107,23 @@ netdev_dpdk_ring_set_config(struct netdev *netdev, const struct smap *args)
return 0;
}
static int
netdev_dpdk_vhost_set_config(struct netdev *netdev, const struct smap *args)
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
const char *path;
if (!(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)) {
path = smap_get(args, "vhost-server-path");
if (path && strcmp(path, dev->vhost_client_id)) {
strcpy(dev->vhost_client_id, path);
netdev_request_reconfigure(netdev);
}
}
return 0;
}
static int
netdev_dpdk_get_numa_id(const struct netdev *netdev)
{
@@ -2284,7 +2326,7 @@ netdev_dpdk_remap_txqs(struct netdev_dpdk *dev)
}
}
VLOG_DBG("TX queue mapping for %s\n", dev->vhost_id);
VLOG_DBG("TX queue mapping for %s\n", get_vhost_id(dev));
for (i = 0; i < total_txqs; i++) {
VLOG_DBG("%2d --> %2d", i, dev->tx_q[i].map);
}
@@ -2308,10 +2350,10 @@ new_device(int vid)
ovs_mutex_lock(&dpdk_mutex);
/* Add device to the vhost port with the same name as that passed down. */
LIST_FOR_EACH(dev, list_node, &dpdk_list) {
if (strncmp(ifname, dev->vhost_id, IF_NAME_SZ) == 0) {
ovs_mutex_lock(&dev->mutex);
if (strncmp(ifname, get_vhost_id(dev), IF_NAME_SZ) == 0) {
uint32_t qp_num = rte_vhost_get_queue_num(vid);
ovs_mutex_lock(&dev->mutex);
/* Get NUMA information */
newnode = rte_vhost_get_numa_node(vid);
if (newnode == -1) {
@@ -2341,6 +2383,7 @@ new_device(int vid)
ovs_mutex_unlock(&dev->mutex);
break;
}
ovs_mutex_unlock(&dev->mutex);
}
ovs_mutex_unlock(&dpdk_mutex);
@@ -2434,8 +2477,8 @@ vring_state_changed(int vid, uint16_t queue_id, int enable)
ovs_mutex_lock(&dpdk_mutex);
LIST_FOR_EACH (dev, list_node, &dpdk_list) {
if (strncmp(ifname, dev->vhost_id, IF_NAME_SZ) == 0) {
ovs_mutex_lock(&dev->mutex);
ovs_mutex_lock(&dev->mutex);
if (strncmp(ifname, get_vhost_id(dev), IF_NAME_SZ) == 0) {
if (enable) {
dev->tx_q[qid].map = qid;
} else {
@@ -2446,6 +2489,7 @@ vring_state_changed(int vid, uint16_t queue_id, int enable)
ovs_mutex_unlock(&dev->mutex);
break;
}
ovs_mutex_unlock(&dev->mutex);
}
ovs_mutex_unlock(&dpdk_mutex);
@@ -2931,6 +2975,7 @@ static int
netdev_dpdk_vhost_reconfigure(struct netdev *netdev)
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
int err = 0;
ovs_mutex_lock(&dpdk_mutex);
ovs_mutex_lock(&dev->mutex);
@@ -2956,6 +3001,45 @@ netdev_dpdk_vhost_reconfigure(struct netdev *netdev)
dev->vhost_reconfigured = true;
}
/* Configure vHost client mode if requested and if the following criteria
* are met:
* 1. Device is currently in 'server' mode.
* 2. Device is currently not active.
* 3. A path has been specified.
*/
if (!(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)
&& !(netdev_dpdk_get_vid(dev) >= 0)
&& strlen(dev->vhost_client_id)) {
/* Unregister server-mode device */
char *vhost_id = xstrdup(get_vhost_id(dev));
ovs_mutex_unlock(&dev->mutex);
ovs_mutex_unlock(&dpdk_mutex);
err = dpdk_vhost_driver_unregister(dev, vhost_id);
free(vhost_id);
ovs_mutex_lock(&dpdk_mutex);
ovs_mutex_lock(&dev->mutex);
if (err) {
VLOG_ERR("Unable to remove vhost-user socket %s",
get_vhost_id(dev));
} else {
fatal_signal_remove_file_to_unlink(get_vhost_id(dev));
/* Register client-mode device */
err = rte_vhost_driver_register(dev->vhost_client_id,
RTE_VHOST_USER_CLIENT);
if (err) {
VLOG_ERR("vhost-user device setup failure for device %s\n",
dev->vhost_client_id);
} else {
/* Configuration successful */
dev->vhost_driver_flags |= RTE_VHOST_USER_CLIENT;
VLOG_INFO("vHost User device '%s' changed to 'client' mode, "
"using client socket '%s'",
dev->up.name, get_vhost_id(dev));
}
}
}
ovs_mutex_unlock(&dev->mutex);
ovs_mutex_unlock(&dpdk_mutex);
@@ -3459,7 +3543,7 @@ static const struct netdev_class dpdk_vhost_class =
"dpdkvhostuser",
netdev_dpdk_vhost_construct,
netdev_dpdk_vhost_destruct,
NULL,
netdev_dpdk_vhost_set_config,
NULL,
netdev_dpdk_vhost_send,
netdev_dpdk_vhost_get_carrier,