mirror of
https://github.com/openvswitch/ovs
synced 2025-10-11 13:57:52 +00:00
dpif-netdev: Add percentage of pmd/core used by each rxq.
It is based on the length of history that is stored about an rxq (currently 1 min). $ ovs-appctl dpif-netdev/pmd-rxq-show pmd thread numa_id 0 core_id 4: isolated : false port: dpdkphy1 queue-id: 0 pmd usage: 70 % port: dpdkvhost0 queue-id: 0 pmd usage: 0 % pmd thread numa_id 0 core_id 6: isolated : false port: dpdkphy0 queue-id: 0 pmd usage: 64 % port: dpdkvhost1 queue-id: 0 pmd usage: 0 % These values are what would be used as part of rxq to pmd assignment due to a reconfiguration event e.g. adding pmds, adding rxqs or with the command: ovs-appctl dpif-netdev/pmd-rxq-rebalance Signed-off-by: Jan Scheurich <jan.scheurich@ericsson.com> Co-authored-by: Jan Scheurich <jan.scheurich@ericsson.com> Signed-off-by: Kevin Traynor <ktraynor@redhat.com> Signed-off-by: Ian Stokes <ian.stokes@intel.com>
This commit is contained in:
committed by
Ian Stokes
parent
4f5d13e241
commit
2a2c67b435
@@ -139,6 +139,17 @@ Core 3: Q1 (80%) |
|
|||||||
Core 7: Q4 (70%) | Q5 (10%)
|
Core 7: Q4 (70%) | Q5 (10%)
|
||||||
core 8: Q3 (60%) | Q0 (30%)
|
core 8: Q3 (60%) | Q0 (30%)
|
||||||
|
|
||||||
|
To see the current measured usage history of pmd core cycles for each rxq::
|
||||||
|
|
||||||
|
$ ovs-appctl dpif-netdev/pmd-rxq-show
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
A history of one minute is recorded and shown for each rxq to allow for
|
||||||
|
traffic pattern spikes. An rxq's pmd core cycles usage changes due to traffic
|
||||||
|
pattern or reconfig changes will take one minute before they are fully
|
||||||
|
reflected in the stats.
|
||||||
|
|
||||||
Rxq to pmds assignment takes place whenever there are configuration changes
|
Rxq to pmds assignment takes place whenever there are configuration changes
|
||||||
or can be triggered by using::
|
or can be triggered by using::
|
||||||
|
|
||||||
|
1
NEWS
1
NEWS
@@ -42,6 +42,7 @@ Post-v2.8.0
|
|||||||
"management" statistics.
|
"management" statistics.
|
||||||
- ovs-ofctl dump-ports command now prints new of set custom statistics
|
- ovs-ofctl dump-ports command now prints new of set custom statistics
|
||||||
if available (for OpenFlow 1.4+).
|
if available (for OpenFlow 1.4+).
|
||||||
|
* Add rxq utilization of pmd to appctl 'dpif-netdev/pmd-rxq-show'.
|
||||||
- Userspace datapath:
|
- Userspace datapath:
|
||||||
* Output packet batching support.
|
* Output packet batching support.
|
||||||
- vswitchd:
|
- vswitchd:
|
||||||
|
@@ -567,6 +567,11 @@ struct dp_netdev_pmd_thread {
|
|||||||
are stored for each polled rxq. */
|
are stored for each polled rxq. */
|
||||||
long long int rxq_next_cycle_store;
|
long long int rxq_next_cycle_store;
|
||||||
|
|
||||||
|
/* Last interval timestamp. */
|
||||||
|
uint64_t intrvl_tsc_prev;
|
||||||
|
/* Last interval cycles. */
|
||||||
|
atomic_ullong intrvl_cycles;
|
||||||
|
|
||||||
/* Current context of the PMD thread. */
|
/* Current context of the PMD thread. */
|
||||||
struct dp_netdev_pmd_thread_ctx ctx;
|
struct dp_netdev_pmd_thread_ctx ctx;
|
||||||
|
|
||||||
@@ -932,9 +937,9 @@ static void
|
|||||||
pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd)
|
pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd)
|
||||||
{
|
{
|
||||||
if (pmd->core_id != NON_PMD_CORE_ID) {
|
if (pmd->core_id != NON_PMD_CORE_ID) {
|
||||||
const char *prev_name = NULL;
|
|
||||||
struct rxq_poll *list;
|
struct rxq_poll *list;
|
||||||
size_t i, n;
|
size_t n_rxq;
|
||||||
|
uint64_t total_cycles = 0;
|
||||||
|
|
||||||
ds_put_format(reply,
|
ds_put_format(reply,
|
||||||
"pmd thread numa_id %d core_id %u:\n\tisolated : %s\n",
|
"pmd thread numa_id %d core_id %u:\n\tisolated : %s\n",
|
||||||
@@ -942,22 +947,34 @@ pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd)
|
|||||||
? "true" : "false");
|
? "true" : "false");
|
||||||
|
|
||||||
ovs_mutex_lock(&pmd->port_mutex);
|
ovs_mutex_lock(&pmd->port_mutex);
|
||||||
sorted_poll_list(pmd, &list, &n);
|
sorted_poll_list(pmd, &list, &n_rxq);
|
||||||
for (i = 0; i < n; i++) {
|
|
||||||
const char *name = netdev_rxq_get_name(list[i].rxq->rx);
|
|
||||||
|
|
||||||
if (!prev_name || strcmp(name, prev_name)) {
|
/* Get the total pmd cycles for an interval. */
|
||||||
if (prev_name) {
|
atomic_read_relaxed(&pmd->intrvl_cycles, &total_cycles);
|
||||||
ds_put_cstr(reply, "\n");
|
/* Estimate the cycles to cover all intervals. */
|
||||||
}
|
total_cycles *= PMD_RXQ_INTERVAL_MAX;
|
||||||
ds_put_format(reply, "\tport: %s\tqueue-id:", name);
|
|
||||||
|
for (int i = 0; i < n_rxq; i++) {
|
||||||
|
struct dp_netdev_rxq *rxq = list[i].rxq;
|
||||||
|
const char *name = netdev_rxq_get_name(rxq->rx);
|
||||||
|
uint64_t proc_cycles = 0;
|
||||||
|
|
||||||
|
for (int j = 0; j < PMD_RXQ_INTERVAL_MAX; j++) {
|
||||||
|
proc_cycles += dp_netdev_rxq_get_intrvl_cycles(rxq, j);
|
||||||
}
|
}
|
||||||
ds_put_format(reply, " %d",
|
ds_put_format(reply, "\tport: %-16s\tqueue-id: %2d", name,
|
||||||
netdev_rxq_get_queue_id(list[i].rxq->rx));
|
netdev_rxq_get_queue_id(list[i].rxq->rx));
|
||||||
prev_name = name;
|
ds_put_format(reply, "\tpmd usage: ");
|
||||||
|
if (total_cycles) {
|
||||||
|
ds_put_format(reply, "%2"PRIu64"",
|
||||||
|
proc_cycles * 100 / total_cycles);
|
||||||
|
ds_put_cstr(reply, " %");
|
||||||
|
} else {
|
||||||
|
ds_put_format(reply, "%s", "NOT AVAIL");
|
||||||
|
}
|
||||||
|
ds_put_cstr(reply, "\n");
|
||||||
}
|
}
|
||||||
ovs_mutex_unlock(&pmd->port_mutex);
|
ovs_mutex_unlock(&pmd->port_mutex);
|
||||||
ds_put_cstr(reply, "\n");
|
|
||||||
free(list);
|
free(list);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4117,6 +4134,8 @@ reload:
|
|||||||
lc = UINT_MAX;
|
lc = UINT_MAX;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pmd->intrvl_tsc_prev = 0;
|
||||||
|
atomic_store_relaxed(&pmd->intrvl_cycles, 0);
|
||||||
cycles_counter_update(s);
|
cycles_counter_update(s);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
uint64_t iter_packets = 0;
|
uint64_t iter_packets = 0;
|
||||||
@@ -6116,6 +6135,7 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd,
|
|||||||
struct dpcls *cls;
|
struct dpcls *cls;
|
||||||
|
|
||||||
if (pmd->ctx.now > pmd->rxq_next_cycle_store) {
|
if (pmd->ctx.now > pmd->rxq_next_cycle_store) {
|
||||||
|
uint64_t curr_tsc;
|
||||||
/* Get the cycles that were used to process each queue and store. */
|
/* Get the cycles that were used to process each queue and store. */
|
||||||
for (unsigned i = 0; i < poll_cnt; i++) {
|
for (unsigned i = 0; i < poll_cnt; i++) {
|
||||||
uint64_t rxq_cyc_curr = dp_netdev_rxq_get_cycles(poll_list[i].rxq,
|
uint64_t rxq_cyc_curr = dp_netdev_rxq_get_cycles(poll_list[i].rxq,
|
||||||
@@ -6124,6 +6144,13 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd,
|
|||||||
dp_netdev_rxq_set_cycles(poll_list[i].rxq, RXQ_CYCLES_PROC_CURR,
|
dp_netdev_rxq_set_cycles(poll_list[i].rxq, RXQ_CYCLES_PROC_CURR,
|
||||||
0);
|
0);
|
||||||
}
|
}
|
||||||
|
curr_tsc = cycles_counter_update(&pmd->perf_stats);
|
||||||
|
if (pmd->intrvl_tsc_prev) {
|
||||||
|
/* There is a prev timestamp, store a new intrvl cycle count. */
|
||||||
|
atomic_store_relaxed(&pmd->intrvl_cycles,
|
||||||
|
curr_tsc - pmd->intrvl_tsc_prev);
|
||||||
|
}
|
||||||
|
pmd->intrvl_tsc_prev = curr_tsc;
|
||||||
/* Start new measuring interval */
|
/* Start new measuring interval */
|
||||||
pmd->rxq_next_cycle_store = pmd->ctx.now + PMD_RXQ_INTERVAL_LEN;
|
pmd->rxq_next_cycle_store = pmd->ctx.now + PMD_RXQ_INTERVAL_LEN;
|
||||||
}
|
}
|
||||||
|
51
tests/pmd.at
51
tests/pmd.at
@@ -6,7 +6,15 @@ m4_divert_push([PREPARE_TESTS])
|
|||||||
# of every rxq (one per line) in the form:
|
# of every rxq (one per line) in the form:
|
||||||
# port_name rxq_id numa_id core_id
|
# port_name rxq_id numa_id core_id
|
||||||
parse_pmd_rxq_show () {
|
parse_pmd_rxq_show () {
|
||||||
awk '/pmd/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\t/{for (i=4; i<=NF; i++) print $2, $i, numa, core}' | sort
|
awk '/pmd thread/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\tport:/ {print $2, $4, numa, core}' | sort
|
||||||
|
}
|
||||||
|
|
||||||
|
# Given the output of `ovs-appctl dpif-netdev/pmd-rxq-show`,
|
||||||
|
# and with queues for each core on one line, prints the rxqs
|
||||||
|
# of the core on one line
|
||||||
|
# 'port:' port_name 'queue_id:' rxq_id rxq_id rxq_id rxq_id
|
||||||
|
parse_pmd_rxq_show_group () {
|
||||||
|
awk '/port:/ {print $1, $2, $3, $4, $12, $20, $28}'
|
||||||
}
|
}
|
||||||
|
|
||||||
# Given the output of `ovs-appctl dpctl/dump-flows`, prints a list of flows
|
# Given the output of `ovs-appctl dpctl/dump-flows`, prints a list of flows
|
||||||
@@ -53,7 +61,7 @@ m4_define([CHECK_PMD_THREADS_CREATED], [
|
|||||||
])
|
])
|
||||||
|
|
||||||
m4_define([SED_NUMA_CORE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1<cleared>\2<cleared>:/"])
|
m4_define([SED_NUMA_CORE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1<cleared>\2<cleared>:/"])
|
||||||
m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1<cleared>\2<cleared>:/;s/\(queue-id: \)1 2 5 6/\1<cleared>/;s/\(queue-id: \)0 3 4 7/\1<cleared>/"])
|
m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/1 2 5 6/<group>/;s/0 3 4 7/<group>/"])
|
||||||
m4_define([DUMMY_NUMA], [--dummy-numa="0,0,0,0"])
|
m4_define([DUMMY_NUMA], [--dummy-numa="0,0,0,0"])
|
||||||
|
|
||||||
AT_SETUP([PMD - creating a thread/add-port])
|
AT_SETUP([PMD - creating a thread/add-port])
|
||||||
@@ -65,7 +73,7 @@ CHECK_PMD_THREADS_CREATED()
|
|||||||
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
pmd thread numa_id <cleared> core_id <cleared>:
|
||||||
isolated : false
|
isolated : false
|
||||||
port: p0 queue-id: 0
|
port: p0 queue-id: 0 pmd usage: NOT AVAIL
|
||||||
])
|
])
|
||||||
|
|
||||||
AT_CHECK([ovs-appctl dpif/show | sed 's/\(tx_queues=\)[[0-9]]*/\1<cleared>/g'], [0], [dnl
|
AT_CHECK([ovs-appctl dpif/show | sed 's/\(tx_queues=\)[[0-9]]*/\1<cleared>/g'], [0], [dnl
|
||||||
@@ -96,7 +104,14 @@ dummy@ovs-dummy: hit:0 missed:0
|
|||||||
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
pmd thread numa_id <cleared> core_id <cleared>:
|
||||||
isolated : false
|
isolated : false
|
||||||
port: p0 queue-id: 0 1 2 3 4 5 6 7
|
port: p0 queue-id: 0 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 1 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 2 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 3 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 4 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 5 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 6 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 7 pmd usage: NOT AVAIL
|
||||||
])
|
])
|
||||||
|
|
||||||
OVS_VSWITCHD_STOP
|
OVS_VSWITCHD_STOP
|
||||||
@@ -120,20 +135,23 @@ dummy@ovs-dummy: hit:0 missed:0
|
|||||||
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
pmd thread numa_id <cleared> core_id <cleared>:
|
||||||
isolated : false
|
isolated : false
|
||||||
port: p0 queue-id: 0 1 2 3 4 5 6 7
|
port: p0 queue-id: 0 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 1 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 2 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 3 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 4 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 5 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 6 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 7 pmd usage: NOT AVAIL
|
||||||
])
|
])
|
||||||
|
|
||||||
TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
|
TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
|
||||||
AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x3])
|
AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x3])
|
||||||
CHECK_PMD_THREADS_CREATED([2], [], [+$TMP])
|
CHECK_PMD_THREADS_CREATED([2], [], [+$TMP])
|
||||||
|
|
||||||
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl
|
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed ':a;/AVAIL$/{N;s/\n//;ba}' | parse_pmd_rxq_show_group | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
port: p0 queue-id: <group>
|
||||||
isolated : false
|
port: p0 queue-id: <group>
|
||||||
port: p0 queue-id: <cleared>
|
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
|
||||||
isolated : false
|
|
||||||
port: p0 queue-id: <cleared>
|
|
||||||
])
|
])
|
||||||
|
|
||||||
TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
|
TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]])
|
||||||
@@ -143,7 +161,14 @@ CHECK_PMD_THREADS_CREATED([1], [], [+$TMP])
|
|||||||
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl
|
||||||
pmd thread numa_id <cleared> core_id <cleared>:
|
pmd thread numa_id <cleared> core_id <cleared>:
|
||||||
isolated : false
|
isolated : false
|
||||||
port: p0 queue-id: 0 1 2 3 4 5 6 7
|
port: p0 queue-id: 0 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 1 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 2 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 3 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 4 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 5 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 6 pmd usage: NOT AVAIL
|
||||||
|
port: p0 queue-id: 7 pmd usage: NOT AVAIL
|
||||||
])
|
])
|
||||||
|
|
||||||
OVS_VSWITCHD_STOP
|
OVS_VSWITCHD_STOP
|
||||||
|
Reference in New Issue
Block a user