diff --git a/Documentation/howto/dpdk.rst b/Documentation/howto/dpdk.rst index 587aaed5b..40f9d9649 100644 --- a/Documentation/howto/dpdk.rst +++ b/Documentation/howto/dpdk.rst @@ -139,6 +139,17 @@ Core 3: Q1 (80%) | Core 7: Q4 (70%) | Q5 (10%) core 8: Q3 (60%) | Q0 (30%) +To see the current measured usage history of pmd core cycles for each rxq:: + + $ ovs-appctl dpif-netdev/pmd-rxq-show + +.. note:: + + A history of one minute is recorded and shown for each rxq to allow for + traffic pattern spikes. An rxq's pmd core cycles usage changes due to traffic + pattern or reconfig changes will take one minute before they are fully + reflected in the stats. + Rxq to pmds assignment takes place whenever there are configuration changes or can be triggered by using:: diff --git a/NEWS b/NEWS index 2c28456b5..96c7c8da4 100644 --- a/NEWS +++ b/NEWS @@ -42,6 +42,7 @@ Post-v2.8.0 "management" statistics. - ovs-ofctl dump-ports command now prints new of set custom statistics if available (for OpenFlow 1.4+). + * Add rxq utilization of pmd to appctl 'dpif-netdev/pmd-rxq-show'. - Userspace datapath: * Output packet batching support. - vswitchd: diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 48a8ebb90..f94b189e0 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -567,6 +567,11 @@ struct dp_netdev_pmd_thread { are stored for each polled rxq. */ long long int rxq_next_cycle_store; + /* Last interval timestamp. */ + uint64_t intrvl_tsc_prev; + /* Last interval cycles. */ + atomic_ullong intrvl_cycles; + /* Current context of the PMD thread. */ struct dp_netdev_pmd_thread_ctx ctx; @@ -932,9 +937,9 @@ static void pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd) { if (pmd->core_id != NON_PMD_CORE_ID) { - const char *prev_name = NULL; struct rxq_poll *list; - size_t i, n; + size_t n_rxq; + uint64_t total_cycles = 0; ds_put_format(reply, "pmd thread numa_id %d core_id %u:\n\tisolated : %s\n", @@ -942,22 +947,34 @@ pmd_info_show_rxq(struct ds *reply, struct dp_netdev_pmd_thread *pmd) ? "true" : "false"); ovs_mutex_lock(&pmd->port_mutex); - sorted_poll_list(pmd, &list, &n); - for (i = 0; i < n; i++) { - const char *name = netdev_rxq_get_name(list[i].rxq->rx); + sorted_poll_list(pmd, &list, &n_rxq); - if (!prev_name || strcmp(name, prev_name)) { - if (prev_name) { - ds_put_cstr(reply, "\n"); - } - ds_put_format(reply, "\tport: %s\tqueue-id:", name); + /* Get the total pmd cycles for an interval. */ + atomic_read_relaxed(&pmd->intrvl_cycles, &total_cycles); + /* Estimate the cycles to cover all intervals. */ + total_cycles *= PMD_RXQ_INTERVAL_MAX; + + for (int i = 0; i < n_rxq; i++) { + struct dp_netdev_rxq *rxq = list[i].rxq; + const char *name = netdev_rxq_get_name(rxq->rx); + uint64_t proc_cycles = 0; + + for (int j = 0; j < PMD_RXQ_INTERVAL_MAX; j++) { + proc_cycles += dp_netdev_rxq_get_intrvl_cycles(rxq, j); } - ds_put_format(reply, " %d", + ds_put_format(reply, "\tport: %-16s\tqueue-id: %2d", name, netdev_rxq_get_queue_id(list[i].rxq->rx)); - prev_name = name; + ds_put_format(reply, "\tpmd usage: "); + if (total_cycles) { + ds_put_format(reply, "%2"PRIu64"", + proc_cycles * 100 / total_cycles); + ds_put_cstr(reply, " %"); + } else { + ds_put_format(reply, "%s", "NOT AVAIL"); + } + ds_put_cstr(reply, "\n"); } ovs_mutex_unlock(&pmd->port_mutex); - ds_put_cstr(reply, "\n"); free(list); } } @@ -4117,6 +4134,8 @@ reload: lc = UINT_MAX; } + pmd->intrvl_tsc_prev = 0; + atomic_store_relaxed(&pmd->intrvl_cycles, 0); cycles_counter_update(s); for (;;) { uint64_t iter_packets = 0; @@ -6116,6 +6135,7 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd, struct dpcls *cls; if (pmd->ctx.now > pmd->rxq_next_cycle_store) { + uint64_t curr_tsc; /* Get the cycles that were used to process each queue and store. */ for (unsigned i = 0; i < poll_cnt; i++) { uint64_t rxq_cyc_curr = dp_netdev_rxq_get_cycles(poll_list[i].rxq, @@ -6124,6 +6144,13 @@ dp_netdev_pmd_try_optimize(struct dp_netdev_pmd_thread *pmd, dp_netdev_rxq_set_cycles(poll_list[i].rxq, RXQ_CYCLES_PROC_CURR, 0); } + curr_tsc = cycles_counter_update(&pmd->perf_stats); + if (pmd->intrvl_tsc_prev) { + /* There is a prev timestamp, store a new intrvl cycle count. */ + atomic_store_relaxed(&pmd->intrvl_cycles, + curr_tsc - pmd->intrvl_tsc_prev); + } + pmd->intrvl_tsc_prev = curr_tsc; /* Start new measuring interval */ pmd->rxq_next_cycle_store = pmd->ctx.now + PMD_RXQ_INTERVAL_LEN; } diff --git a/tests/pmd.at b/tests/pmd.at index 83d60f8d3..5e9d01e8c 100644 --- a/tests/pmd.at +++ b/tests/pmd.at @@ -6,7 +6,15 @@ m4_divert_push([PREPARE_TESTS]) # of every rxq (one per line) in the form: # port_name rxq_id numa_id core_id parse_pmd_rxq_show () { - awk '/pmd/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\t/{for (i=4; i<=NF; i++) print $2, $i, numa, core}' | sort + awk '/pmd thread/ {numa=$4; core=substr($6, 1, length($6) - 1)} /\tport:/ {print $2, $4, numa, core}' | sort +} + +# Given the output of `ovs-appctl dpif-netdev/pmd-rxq-show`, +# and with queues for each core on one line, prints the rxqs +# of the core on one line +# 'port:' port_name 'queue_id:' rxq_id rxq_id rxq_id rxq_id +parse_pmd_rxq_show_group () { + awk '/port:/ {print $1, $2, $3, $4, $12, $20, $28}' } # Given the output of `ovs-appctl dpctl/dump-flows`, prints a list of flows @@ -53,7 +61,7 @@ m4_define([CHECK_PMD_THREADS_CREATED], [ ]) m4_define([SED_NUMA_CORE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1\2:/"]) -m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/\(numa_id \)[[0-9]]*\( core_id \)[[0-9]]*:/\1\2:/;s/\(queue-id: \)1 2 5 6/\1/;s/\(queue-id: \)0 3 4 7/\1/"]) +m4_define([SED_NUMA_CORE_QUEUE_PATTERN], ["s/1 2 5 6//;s/0 3 4 7//"]) m4_define([DUMMY_NUMA], [--dummy-numa="0,0,0,0"]) AT_SETUP([PMD - creating a thread/add-port]) @@ -65,7 +73,7 @@ CHECK_PMD_THREADS_CREATED() AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl pmd thread numa_id core_id : isolated : false - port: p0 queue-id: 0 + port: p0 queue-id: 0 pmd usage: NOT AVAIL ]) AT_CHECK([ovs-appctl dpif/show | sed 's/\(tx_queues=\)[[0-9]]*/\1/g'], [0], [dnl @@ -96,7 +104,14 @@ dummy@ovs-dummy: hit:0 missed:0 AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl pmd thread numa_id core_id : isolated : false - port: p0 queue-id: 0 1 2 3 4 5 6 7 + port: p0 queue-id: 0 pmd usage: NOT AVAIL + port: p0 queue-id: 1 pmd usage: NOT AVAIL + port: p0 queue-id: 2 pmd usage: NOT AVAIL + port: p0 queue-id: 3 pmd usage: NOT AVAIL + port: p0 queue-id: 4 pmd usage: NOT AVAIL + port: p0 queue-id: 5 pmd usage: NOT AVAIL + port: p0 queue-id: 6 pmd usage: NOT AVAIL + port: p0 queue-id: 7 pmd usage: NOT AVAIL ]) OVS_VSWITCHD_STOP @@ -120,20 +135,23 @@ dummy@ovs-dummy: hit:0 missed:0 AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl pmd thread numa_id core_id : isolated : false - port: p0 queue-id: 0 1 2 3 4 5 6 7 + port: p0 queue-id: 0 pmd usage: NOT AVAIL + port: p0 queue-id: 1 pmd usage: NOT AVAIL + port: p0 queue-id: 2 pmd usage: NOT AVAIL + port: p0 queue-id: 3 pmd usage: NOT AVAIL + port: p0 queue-id: 4 pmd usage: NOT AVAIL + port: p0 queue-id: 5 pmd usage: NOT AVAIL + port: p0 queue-id: 6 pmd usage: NOT AVAIL + port: p0 queue-id: 7 pmd usage: NOT AVAIL ]) TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]]) AT_CHECK([ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x3]) CHECK_PMD_THREADS_CREATED([2], [], [+$TMP]) -AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl -pmd thread numa_id core_id : - isolated : false - port: p0 queue-id: -pmd thread numa_id core_id : - isolated : false - port: p0 queue-id: +AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed ':a;/AVAIL$/{N;s/\n//;ba}' | parse_pmd_rxq_show_group | sed SED_NUMA_CORE_QUEUE_PATTERN], [0], [dnl +port: p0 queue-id: +port: p0 queue-id: ]) TMP=$(cat ovs-vswitchd.log | wc -l | tr -d [[:blank:]]) @@ -143,7 +161,14 @@ CHECK_PMD_THREADS_CREATED([1], [], [+$TMP]) AT_CHECK([ovs-appctl dpif-netdev/pmd-rxq-show | sed SED_NUMA_CORE_PATTERN], [0], [dnl pmd thread numa_id core_id : isolated : false - port: p0 queue-id: 0 1 2 3 4 5 6 7 + port: p0 queue-id: 0 pmd usage: NOT AVAIL + port: p0 queue-id: 1 pmd usage: NOT AVAIL + port: p0 queue-id: 2 pmd usage: NOT AVAIL + port: p0 queue-id: 3 pmd usage: NOT AVAIL + port: p0 queue-id: 4 pmd usage: NOT AVAIL + port: p0 queue-id: 5 pmd usage: NOT AVAIL + port: p0 queue-id: 6 pmd usage: NOT AVAIL + port: p0 queue-id: 7 pmd usage: NOT AVAIL ]) OVS_VSWITCHD_STOP