netdev-dpdk: Properly support non pmd threads.

We used to reserve DPDK lcore 0 for non pmd operations, making it
difficult to use core 0 for packet processing.
DPDK 2.0 properly support non EAL threads with lcore LCORE_ID_ANY.

Using non EAL threads for non pmd threads, we do not need to reserve
any core for non pmd operations

Signed-off-by: Daniele Di Proietto <diproiettod@vmware.com>
Signed-off-by: Ethan Jackson <ethan@nicira.com>
Acked-by: Ethan Jackson <ethan@nicira.com>

INSTALL.DPDK.md

@@ -260,9 +260,6 @@ Using the DPDK with ovs-vswitchd:
    Note, the pmd threads on a numa node are only created if there is at least
    one DPDK interface from the numa node that has been added to OVS.
 
-   Note, core 0 is always reserved from non-pmd threads and should never be set
-   in the cpu mask.
-
    To understand where most of the time is spent and whether the caches are
    effective, these commands can be used:
 

lib/dpctl.c

@@ -783,6 +783,12 @@ dpctl_dump_flows(int argc, const char *argv[], struct dpctl_params *dpctl_p)
         }
     }
 
+    /* Make sure that these values are different. PMD_ID_NULL means that the
+     * pmd is unspecified (e.g. because the datapath doesn't have different
+     * pmd threads), while NON_PMD_CORE_ID refers to every non pmd threads
+     * in the userspace datapath */
+    BUILD_ASSERT(PMD_ID_NULL != NON_PMD_CORE_ID);
+
     ds_init(&ds);
     flow_dump = dpif_flow_dump_create(dpif, false);
     flow_dump_thread = dpif_flow_dump_thread_create(flow_dump);

lib/dpif-netdev.c

@@ -580,7 +580,7 @@ pmd_info_show_stats(struct ds *reply,
     if (pmd->numa_id != OVS_NUMA_UNSPEC) {
         ds_put_format(reply, " numa_id %d", pmd->numa_id);
     }
-    if (pmd->core_id != OVS_CORE_UNSPEC) {
+    if (pmd->core_id != OVS_CORE_UNSPEC && pmd->core_id != NON_PMD_CORE_ID) {
         ds_put_format(reply, " core_id %u", pmd->core_id);
     }
     ds_put_cstr(reply, ":\n");
@@ -829,8 +829,6 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
     ovs_mutex_init_recursive(&dp->non_pmd_mutex);
     ovsthread_key_create(&dp->per_pmd_key, NULL);
 
-    /* Reserves the core NON_PMD_CORE_ID for all non-pmd threads. */
-    ovs_numa_try_pin_core_specific(NON_PMD_CORE_ID);
     dp_netdev_set_nonpmd(dp);
     dp->n_dpdk_rxqs = NR_QUEUE;
 

lib/netdev-dpdk.c

@@ -1927,7 +1927,7 @@ dpdk_init(int argc, char **argv)
     }
 
     /* We are called from the main thread here */
-    thread_set_nonpmd();
+    RTE_PER_LCORE(_lcore_id) = NON_PMD_CORE_ID;
 
     return result + 1 + base;
 }
@@ -2012,14 +2012,6 @@ pmd_thread_setaffinity_cpu(unsigned cpu)
     return 0;
 }
 
-void
-thread_set_nonpmd(void)
-{
-    /* We have to use NON_PMD_CORE_ID to allow non-pmd threads to perform
-     * certain DPDK operations, like rte_eth_dev_configure(). */
-    RTE_PER_LCORE(_lcore_id) = NON_PMD_CORE_ID;
-}
-
 static bool
 thread_is_pmd(void)
 {

lib/netdev-dpdk.h

@@ -5,11 +5,6 @@
 
 struct dp_packet;
 
-/* Reserves cpu core 0 for all non-pmd threads.  Changing the value of this
- * macro will allow pmd thread to be pinned on cpu core 0.  This may not be
- * ideal since the core may be non-isolated. */
-#define NON_PMD_CORE_ID 0
-
 #ifdef DPDK_NETDEV
 
 #include <rte_config.h>
@@ -25,14 +20,17 @@ struct dp_packet;
 #include <rte_launch.h>
 #include <rte_malloc.h>
 
+#define NON_PMD_CORE_ID LCORE_ID_ANY
+
 int dpdk_init(int argc, char **argv);
 void netdev_dpdk_register(void);
 void free_dpdk_buf(struct dp_packet *);
 int pmd_thread_setaffinity_cpu(unsigned cpu);
-void thread_set_nonpmd(void);
 
 #else
 
+#define NON_PMD_CORE_ID UINT32_MAX
+
 #include "util.h"
 
 static inline int
@@ -62,11 +60,5 @@ pmd_thread_setaffinity_cpu(unsigned cpu OVS_UNUSED)
     return 0;
 }
 
-static inline void
-thread_set_nonpmd(void)
-{
-    /* Nothing */
-}
-
 #endif /* DPDK_NETDEV */
 #endif

lib/ovs-thread.c

@@ -334,8 +334,6 @@ ovsthread_wrapper(void *aux_)
     set_subprogram_name("%s%u", aux.name, id);
     ovsrcu_quiesce_end();
 
-    thread_set_nonpmd();
-
     return aux.start(aux.arg);
 }