mir/ovs
2
0
Fork 0
mirror of https://github.com/openvswitch/ovs synced 2025-08-22 18:07:40 +00:00
Code Issues Releases Wiki Activity
ovs/utilities/usdt-scripts/reval_monitor.py

895 lines
29 KiB
Python
Raw Normal View History

utilities: Add revalidator measurement script and needed USDT probes. This patch adds a Python script that can be used to analyze the revalidator runs by providing statistics (including some real time graphs). The USDT events can also be captured to a file and used for later offline analysis. The following blog explains the Open vSwitch revalidator implementation and how this tool can help you understand what is happening in your system. https://developers.redhat.com/articles/2022/10/19/open-vswitch-revalidator-process-explained Signed-off-by: Eelco Chaudron <echaudro@redhat.com> Acked-by: Adrian Moreno <amorenoz@redhat.com> Acked-by: Simon Horman <simon.horman@corigine.com> Signed-off-by: Ilya Maximets <i.maximets@ovn.org>
2023-01-23 12:03:29 +01:00
#!/usr/bin/env python3
#
# Copyright (c) 2022 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Script information:
# -------------------
# reval_monitor.py uses various user-defined tracepoints to get all the
# revalidator-process related variables and will display them in a (dynamic)
# graph. In addition, it will also dump the data to the console
# in a CSV format. Note that all the graphical output can be disabled.
#
# All the USDT events can be saved to a file and than can be used to
# replay the trace offline and still get the plots.
#
# The script can simple be invoked without any options, and it will try
# to find the running ovs-vswitchd instance:
#
# # ./reval_monitor.py
# # Starting trace @2022-09-20T04:07:43.588749 (08:07:43 UTC)
# ts_start, ts_complete, n_flows, n_reval_flows, avg_n_flows, max_n_flows,
# flow_limit, dump_duration, poll_wait, actual_wait
# 1741367714251645, 1741367714532545, 0, 0, 0, 10000, 69000, 1, 500, 500.52
# 1741368215056961, 1741368215318223, 0, 0, 0, 10000, 69000, 1, 500, 500.55
# 1741368715865871, 1741368716107089, 0, 0, 0, 10000, 69000, 1, 500, 499.48
# ^C# Stopping trace @2022-09-20T04:07:49.893827 (08:07:49 UTC)
#
# IMPORTANT NOTE: This script only works when a single datapath is configured!
# 2nd IMPORTANT NOTE: ovs-vswitchd either needs to be built with debug info
# or the debug info package needs to be installed!
#
# The following are the available options:
#
# usage: reval_monitor.py [-h] [-c] [--buffer-page-count NUMBER]
# [-D [DEBUG]] [-g] [--no-ukey-count]
# [-p VSWITCHD_PID] [-P PAHOLE] [-r FILE] [-R]
# [-u SECONDS] [-w FILE] [-W FILE]
#
# options:
# -h, --help show this help message and exit
# -c, --compress-output
# Compress output, i.e. only dump changes in
# the dataset
# --buffer-page-count NUMBER
# Number of BPF ring buffer pages, default
# 1024
# -D [DEBUG], --debug [DEBUG]
# Enable eBPF debugging
# -g, --no-gui Do not use the gui to display plots
# --no-ukey-count No revalidate_ukey() counting
# -p VSWITCHD_PID, --pid VSWITCHD_PID
# ovs-vswitch's PID
# -P PAHOLE, --pahole PAHOLE
# Pahole executable to use, default pahole
# -r FILE, --read-events FILE
# Read events from <FILE> instead of
# installing tracepoints
# -R, --no-realtime-plots
# Do not show realtime plot while tracing
# -u SECONDS, --update-interval SECONDS
# Seconds to wait between real time update,
# default 1
# -w FILE, --write-events FILE
# Write events to <FILE>
# -W FILE, --write-charts FILE
# Write overall charts to <FILE>.png
# [-D [DEBUG]] [-g] [--no-ukey-count]
# [-p VSWITCHD_PID] [-r FILE] [-R]
# [-u SECONDS] [-w FILE] [-W FILE]
#
# The -g option disabled all GUI output of matplotlib, -R only disables the
# real-time plots. As real-time plots are rather slow, the -u option can be
# used to only update the graph every x seconds, which might speed up the
# processing.
#
# The --no-ukey-count option disables counting of the number of flows actually
# being revalidated against the current OpenFlow ruleset. This will not install
# the specific tracepoint which would be called for each flow being
# revalidated.
#
# What is plotted in the graphs (and dumped in the CSV output)?
# - n_flows: Number of flows active in the system.
# - n_reval_flows: Number of flows that where revalidated against the OpenFlow
# ruleset.
# - dump_duration: Time it took to dump and process all flows.
# - avg_n_flows: Average number of flows in the system.
# - max_n_flows: Maximum number of flows in the system.
# - flow_limit: Dynamic flow limit.
# - poll_wait: Time requested for the poll wait.
# - actual_wait: Time it took to be woken up.
#
# Dependencies:
# This script needs the 'readelf' binary to be available. In addition, it also
# needs pahole to be installed, and it needs a version that is equal or newer
# than the following commit on the next branch:
#
# https://git.kernel.org/pub/scm/devel/pahole/pahole.git/?h=next
# c55b13b9d785 ("WIP: Remove DW_TAG_atomic_type when encoding BTF")
#
# To use a locally compiled pahole the --pahole option can be used.
# For example:
# # ./reval_monitor.py --pahole ~/pahole/build/pahole -g
# Starting trace @2022-12-20T14:57:26.077815 (13:57:26 UTC)
# ts_start, ts_complete, n_flows, n_reval_flows, avg_n_flows, max_n_flows, \
# flow_limit, dump_duration, poll_wait, actual_wait
# 4202771850983494, 4202771851472838, 0, 0, 0, 0, 10000, 1, 500, 15.06
# 4202771866531996, 4202771867713366, 0, 0, 0, 0, 10000, 1, 500, 4.23
# 4202771871941979, 4202771872749915, 0, 0, 0, 0, 10000, 1, 500, 500.02
# 4202772372770361, 4202772373531820, 0, 0, 0, 0, 10000, 1, 500, 499.96
# 4202772873487942, 4202772874514753, 0, 0, 0, 0, 10000, 1, 500, 500.01
# 4202773374528435, 4202773375695054, 0, 0, 0, 0, 10000, 1, 500, 500.01
# 4202773875701559, 4202773876880763, 0, 0, 0, 0, 10000, 1, 500, 500.04
# 4202774376925058, 4202774377905799, 0, 0, 0, 0, 10000, 1, 500, 500.03
# ^C# Stopping trace @2022-12-20T14:57:40.391730 (13:57:40 UTC)
#
try:
from bcc import BPF, USDT, USDTException
except ModuleNotFoundError:
print("WARNING: Can't find the BPF Compiler Collection (BCC) tools!")
print(" This is NOT problem if you analyzing previously collected"
" data.\n")
from collections import namedtuple
from enum import IntEnum
from pathlib import Path
import argparse
import ast
import datetime
import re
import subprocess
import sys
import pytz
import psutil
import matplotlib.pyplot as plt
#
# Actual eBPF source code
#
EBPF_SOURCE = """
#include <linux/sched.h>
<OVS_INCLUDE_DEFINITIONS>
enum {
<EVENT_ENUM>
};
struct event_t {
u64 ts;
u32 pid;
u32 id;
u64 n_flows;
u32 avg_n_flows;
u32 max_n_flows;
u32 flow_limit;
u32 dump_duration;
u32 poll_wait;
};
BPF_RINGBUF_OUTPUT(events, <BUFFER_PAGE_CNT>);
BPF_TABLE("percpu_array", uint32_t, uint64_t, dropcnt, 1);
static struct event_t *get_event(uint32_t id) {
struct event_t *event = events.ringbuf_reserve(sizeof(struct event_t));
if (!event) {
dropcnt.increment(0);
return NULL;
}
event->id = id;
event->ts = bpf_ktime_get_ns();
event->pid = bpf_get_current_pid_tgid();
return event;
}
int probe__start_dump(struct pt_regs *ctx) {
struct event_t *event = get_event(EVENT_START_DUMP);
if (!event)
return 1;
events.ringbuf_submit(event, 0);
return 0;
};
int probe__sweep_done(struct pt_regs *ctx) {
struct udpif udpif;
bpf_usdt_readarg_p(1, ctx, &udpif, sizeof(udpif));
struct event_t *event = get_event(EVENT_SWEEP_DONE);
if (!event)
return 1;
event->avg_n_flows = udpif.avg_n_flows;
event->max_n_flows = udpif.max_n_flows;
event->flow_limit = udpif.flow_limit;
event->dump_duration = udpif.dump_duration;
bpf_usdt_readarg(2, ctx, &event->n_flows);
bpf_usdt_readarg(3, ctx, &event->poll_wait);
events.ringbuf_submit(event, 0);
return 0;
};
int probe__reval_entry(struct pt_regs *ctx) {
struct event_t *event = get_event(EVENT_REVAL_ENTRY);
if (!event)
return 1;
events.ringbuf_submit(event, 0);
return 0;
};
"""
#
# event_to_dict()
#
def event_to_dict(event):
return dict([(field, getattr(event, field))
for (field, _) in event._fields_
if isinstance(getattr(event, field), (int, bytes))])
#
# print_csv_header()
#
def print_csv_header():
print("ts_start, ts_complete, n_flows, n_reval_flows, avg_n_flows, "
"max_n_flows, flow_limit, dump_duration, poll_wait, actual_wait")
#
# Event enum
#
Event = IntEnum("Event", ["START_DUMP",
"SWEEP_DONE",
"REVAL_ENTRY"], start=0)
#
# process_event()
#
def process_event(ctx, data, size):
event = b['events'].event(data)
_process_event(event)
def _process_event(event):
global graph
if export_file is not None:
export_file.write("event = {}\n".format(event_to_dict(event)))
if event.id == Event.START_DUMP and not state['running']:
start = state["last_start"]
done = state["last_done"]
if done and start:
actual_wait = (event.ts - done.ts) / 1000000
csv = "{}, {}, {}, {}, {}, {}, {}, {}, {}, {:.2f}".format(
start.ts, done.ts, done.n_flows, graph.ukey_count,
done.avg_n_flows, done.max_n_flows, done.flow_limit,
done.dump_duration, done.poll_wait, actual_wait)
if graph.base_time == 0:
graph = graph._replace(base_time=done.ts)
graph.time.append((done.ts - graph.base_time) / 1000000000)
graph.n_flows.append(done.n_flows)
graph.n_reval_flows.append(graph.ukey_count)
graph.avg_n_flows.append(done.avg_n_flows)
graph.max_n_flows.append(done.max_n_flows)
graph.flow_limit.append(done.flow_limit)
graph.dump_duration.append(done.dump_duration)
graph.poll_wait.append(done.poll_wait)
graph.actual_wait.append(actual_wait)
if not options.no_gui and not options.no_realtime_plots:
updated_graph = dynamic_plot_update(
graph, refresh=options.update_interval)
if updated_graph is None:
raise KeyboardInterrupt
graph = updated_graph
if options.compress_output:
last_csv = state["last_csv"]
if not last_csv or \
csv.split(",")[2:-1] != last_csv.split(",")[2:-1] or \
abs((event.ts - done.ts) / 1000000 - done.poll_wait) > 100:
print(csv)
else:
state["last_not_printed_csv"] = csv
state["last_csv"] = csv
else:
print(csv)
state["last_start"] = event
state['running'] = True
graph = graph._replace(ukey_count=0)
elif event.id == Event.SWEEP_DONE and state['running']:
state["last_done"] = event
state['running'] = False
elif event.id == Event.REVAL_ENTRY and state['running']:
graph = graph._replace(ukey_count=graph.ukey_count + 1)
#
# run_program()
#
def run_program(command):
try:
process = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding='utf8',
check=True)
except subprocess.CalledProcessError as perror:
return perror.returncode, perror.stdout
return 0, process.stdout
#
# get_ovs_definitions()
#
def get_ovs_definitions(objects, pahole="pahole", pid=None):
if pid is None:
raise ValueError("A valid pid value should be supplied!")
if not isinstance(objects, list):
objects = [objects]
if len(objects) == 0:
raise ValueError("Must supply at least one object!")
vswitchd = Path("/proc/{}/exe".format(str(pid))).resolve()
object_str = ','.join(objects)
def run_pahole(debug_file):
error, result = run_program([pahole, "-C", object_str, "--compile",
debug_file])
if error:
if "pahole: {}: Invalid argument".format(debug_file) not in result:
print("ERROR: Pahole failed to get ovs-vswitchd data "
"structures!\n{}".format(re.sub('^', ' ' * 7,
result.rstrip(),
flags=re.MULTILINE)))
sys.exit(-1)
return None
if bool(re.search("pahole: type .* not found", result)):
return None
return result
def run_readelf(bin_file):
error, result = run_program(['readelf', "-n",
"--debug-dump=links", bin_file])
if error:
print("ERROR: Failed 'readelf' on \"{}\"!\n{}".
format(bin_file, re.sub('^', ' ' * 7, result,
flags=re.MULTILINE)))
sys.exit(-1)
return result
def get_debug_file(bin_file):
elf_result = run_readelf(bin_file)
match = re.search("Build ID: ([0-9a-fA-F]+)", elf_result)
if not match:
print("ERROR: Can't find build ID to read debug symbols!")
sys.exit(-1)
dbg_file = "/usr/lib/debug/.build-id/{}/{}.debug".format(
match.group(1)[:2], match.group(1)[2:])
return dbg_file
def get_from_shared_library(debug_file):
ovs_libs = ['libofproto', 'libopenvswitch', 'libovsdb', 'libsflow',
'libvtep']
error, ldd_result = run_program(['ldd', debug_file])
if error:
print("ERROR: Failed 'ldd' on \"{}\"!\n{}".
format(debug_file, re.sub('^', ' ' * 7, ldd_result,
flags=re.MULTILINE)))
sys.exit(-1)
for lib in ovs_libs:
match = re.search(r"^\s*{}.* => (.*) \(.*\)$".format(lib),
ldd_result, flags=re.MULTILINE)
if match is None:
continue
result = run_pahole(match.group(1))
if result is None:
result = run_pahole(get_debug_file(match.group(1)))
if result:
return result
return None
#
# First try to find the debug data as part of the executable.
#
result = run_pahole(vswitchd)
if result is None:
print("INFO: Failed to find debug info in \"{}\"!".format(vswitchd))
#
# Get additional .debug information if available.
#
dbg_file = get_debug_file(vswitchd)
result = run_pahole(dbg_file)
if result is None:
print("INFO: Failed to find debug info in \"{}\"!".format(
dbg_file))
#
# Try to get information from shared libraries if used.
#
result = get_from_shared_library(vswitchd)
if result is None:
print("ERROR: Failed to find needed data structures through pahole!")
sys.exit(-1)
#
# We need an empty _Atomic definition to avoid compiler complaints.
#
result = "#define _Atomic\n" + result
#
# Remove the uint64_t definition as it conflicts with the kernel one.
#
result = re.sub("^typedef.*uint64_t;$", "", result, flags=re.MULTILINE)
return result
#
# next_power_of_two()
#
def next_power_of_two(val):
np = 1
while np < val:
np *= 2
return np
#
# dynamic_plot_init()
#
def dynamic_plot_init(real_time=True):
if real_time:
lines = []
fig, axs = plt.subplots(4, figsize=(19, 10))
fig.suptitle('Revalidator Handling')
for ax in axs:
ax.grid()
axs[0].set_ylabel("Numer of flows", weight='bold')
axs[1].set_ylabel("Time spend (ms)", weight='bold')
axs[2].set_ylabel("Numer of flows", weight='bold')
axs[3].set_ylabel("Time spend (ms)", weight='bold')
axs[3].set_xlabel("Time (seconds since start)", weight='bold')
lines.append(axs[0].plot([], [], label="n_flows", marker='o')[0])
lines.append(axs[0].plot([], [], label="n_reval_flows")[0])
axs[0].legend(bbox_to_anchor=(1, 1), loc='upper left',
borderaxespad=0.5)
axs[0].set_xlim(0, 30)
axs[0].set_ylim(-4, 104)
lines.append(axs[1].plot([], [], color="orange",
label="dump_duration")[0])
axs[1].legend(bbox_to_anchor=(1, 1),
loc='upper left', borderaxespad=0.5)
axs[1].set_xlim(0, 30)
axs[1].set_ylim(-0.4, 10.4)
lines.append(axs[2].plot([], [], label="avg_n_flows")[0])
lines.append(axs[2].plot([], [], label="max_n_flows")[0])
lines.append(axs[2].plot([], [], label="flow_limit")[0])
axs[2].legend(bbox_to_anchor=(1, 1), loc='upper left',
borderaxespad=0.5)
axs[2].set_xlim(0, 30)
axs[2].set_ylim(-600, 15600)
lines.append(axs[3].plot([], [], label="poll_wait")[0])
lines.append(axs[3].plot([], [], label="actual_wait")[0])
axs[3].legend(bbox_to_anchor=(1, 1), loc='upper left',
borderaxespad=0.5)
axs[3].set_xlim(0, 30)
axs[3].set_ylim(-20, 520)
fig.tight_layout()
plt.ion()
plt.show()
else:
fig = None
axs = None
lines = None
graph_data = {"base_time": 0,
"l_index": 0,
"fig": fig,
"axs": axs,
"lines": lines,
"last_update": 0,
"ukey_count": 0,
"time": [],
"n_flows": [],
"n_reval_flows": [],
"avg_n_flows": [],
"max_n_flows": [],
"flow_limit": [],
"dump_duration": [],
"poll_wait": [],
"actual_wait": []}
return namedtuple("GraphData", graph_data.keys())(*graph_data.values())
#
# dynamic_plot_update()
#
def dynamic_plot_update(graph_data, refresh=1):
if graph_data.last_update != 0 and \
(graph_data.time[-1] - graph_data.last_update) < refresh:
return graph_data
graph_data = graph_data._replace(last_update=graph_data.time[-1])
if (graph_data.time[-1] - graph_data.time[graph_data.l_index]) > 30:
for i in range(graph_data.l_index + 1, len(graph_data.time)):
if (graph_data.time[-1] - graph_data.time[i]) <= 30:
graph_data = graph_data._replace(l_index=i)
break
for line in graph_data.lines:
line.set_xdata(graph_data.time[graph_data.l_index:])
graph_data.lines[0].set_ydata(graph_data.n_flows[graph_data.l_index:])
graph_data.lines[1].set_ydata(
graph_data.n_reval_flows[graph_data.l_index:])
graph_data.lines[2].set_ydata(
graph_data.dump_duration[graph_data.l_index:])
graph_data.lines[3].set_ydata(graph_data.avg_n_flows[graph_data.l_index:])
graph_data.lines[4].set_ydata(graph_data.max_n_flows[graph_data.l_index:])
graph_data.lines[5].set_ydata(graph_data.flow_limit[graph_data.l_index:])
graph_data.lines[6].set_ydata(graph_data.poll_wait[graph_data.l_index:])
graph_data.lines[7].set_ydata(graph_data.actual_wait[graph_data.l_index:])
for ax in graph_data.axs:
if graph_data.l_index == 0:
ax.autoscale(enable=True, axis='y')
else:
ax.autoscale(enable=True)
ax.relim(visible_only=True)
ax.autoscale_view(tight=True, scalex=True, scaley=True)
try:
graph_data.fig.canvas.draw()
graph_data.fig.canvas.flush_events()
except KeyboardInterrupt:
return None
return graph_data
#
# show_graph()
#
def show_graph(graph_data, gui=False, file_name=None):
if len(graph_data.time) == 0 or (not gui and file_name is None):
return
plt.ioff()
fig, (nf_ax, dd_ax, f_ax, t_ax) = plt.subplots(4, figsize=(19, 10))
fig.suptitle('Revalidator Handling')
nf_ax.grid()
f_ax.grid()
dd_ax.grid()
t_ax.grid()
nf_ax.set_ylabel("Numer of flows", weight='bold')
f_ax.set_ylabel("Numer of flows", weight='bold')
dd_ax.set_ylabel("Time spend (ms)", weight='bold')
t_ax.set_ylabel("Time spend (ms)", weight='bold')
t_ax.set_xlabel("Time (seconds since start)", weight='bold')
nf_ax.plot(graph_data.time, graph_data.n_flows, label="n_flows")
nf_ax.plot(graph_data.time, graph_data.n_reval_flows,
label="n_reval_flows")
nf_ax.legend(bbox_to_anchor=(1, 1), loc='upper left', borderaxespad=0.5)
dd_ax.plot(graph_data.time, graph_data.dump_duration, color="orange",
label="dump_duration")
dd_ax.legend(bbox_to_anchor=(1, 1), loc='upper left', borderaxespad=0.5)
f_ax.plot(graph_data.time, graph_data.avg_n_flows, label="avg_n_flows")
f_ax.plot(graph_data.time, graph_data.max_n_flows, label="max_n_flows")
f_ax.plot(graph_data.time, graph_data.flow_limit, label="flow_limit")
f_ax.legend(bbox_to_anchor=(1, 1), loc='upper left', borderaxespad=0.5)
t_ax.plot(graph_data.time, graph_data.poll_wait, label="poll_wait")
t_ax.plot(graph_data.time, graph_data.actual_wait, label="actual_wait")
t_ax.legend(bbox_to_anchor=(1, 1), loc='upper left', borderaxespad=0.5)
fig.tight_layout()
if file_name is not None and file_name != "":
fig.savefig(file_name + '.png')
if gui:
try:
plt.show()
except KeyboardInterrupt:
pass
plt.close(fig)
#
# process_events_from_file()
#
def process_events_from_file(file_name):
try:
with open(file_name, 'r') as fd:
print("- Reading events from \"{}\"...".format(file_name))
print_csv_header()
for entry in fd:
entry.rstrip()
if entry.startswith('event = {'):
event = ast.literal_eval(entry[8:])
event = namedtuple("EventObject",
event.keys())(*event.values())
try:
_process_event(event)
except KeyboardInterrupt:
break
except (FileNotFoundError, PermissionError):
print("ERROR: Can't open file \"{}\" for reading!".format(file_name))
sys.exit(-1)
show_graph(graph, gui=not options.no_gui, file_name=options.write_charts)
#
# main()
#
def main():
#
# Don't like these globals, but ctx passing does not seem to work with the
# existing open_ring_buffer() API :(
#
global b
global export_file
global options
global state
global graph
#
# Argument parsing
#
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--compress-output", action="store_true",
help="Compress output, i.e. only dump changes in "
"the dataset")
parser.add_argument("--buffer-page-count",
help="Number of BPF ring buffer pages, default 1024",
type=int, default=1024, metavar="NUMBER")
parser.add_argument("-D", "--debug",
help="Enable eBPF debugging",
type=int, const=0x3f, default=0, nargs='?')
parser.add_argument("-g", "--no-gui", action="store_true",
help="Do not use the gui to display plots")
parser.add_argument("--no-ukey-count", action="store_true",
help="No revalidate_ukey() counting")
parser.add_argument("-p", "--pid", metavar="VSWITCHD_PID",
help="ovs-vswitch's PID",
type=int, default=None)
parser.add_argument("-P", "--pahole", metavar="PAHOLE",
help="Pahole executable to use, default pahole",
type=str, default="pahole")
parser.add_argument("-r", "--read-events",
help="Read events from <FILE> instead of installing "
"tracepoints", type=str, default=None, metavar="FILE")
parser.add_argument("-R", "--no-realtime-plots", action="store_true",
help="Do not show realtime plot while tracing")
parser.add_argument("-u", "--update-interval",
help="Seconds to wait between real time update, "
"default 1", type=float, default=1, metavar="SECONDS")
parser.add_argument("-w", "--write-events",
help="Write events to <FILE>",
type=str, default=None, metavar="FILE")
parser.add_argument("-W", "--write-charts",
help="Write overall charts to <FILE>.png",
type=str, default=None, metavar="FILE")
options = parser.parse_args()
#
# Find the PID of the ovs-vswitchd daemon if not specified.
#
if options.pid is None and options.read_events is None:
for proc in psutil.process_iter():
if 'ovs-vswitchd' in proc.name():
if options.pid is not None:
print("ERROR: Multiple ovs-vswitchd daemons running, "
"use the -p option!")
sys.exit(-1)
options.pid = proc.pid
#
# Error checking on input parameters.
#
if options.pid is None and options.read_events is None:
print("ERROR: Failed to find ovs-vswitchd's PID!")
sys.exit(-1)
if options.read_events is not None and options.write_events is not None:
print("ERROR: Either supply the read or write events option, "
"not both!")
sys.exit(-1)
options.buffer_page_count = next_power_of_two(options.buffer_page_count)
#
# Define the state and graph.
#
state = {"last_start": None,
"last_done": None,
"running": False,
"last_csv": None,
"last_not_printed_csv": None}
export_file = None
graph = dynamic_plot_init(real_time=(not options.no_gui
and not options.no_realtime_plots))
#
# Process events from file if required.
#
if options.read_events is not None:
process_events_from_file(options.read_events)
sys.exit(0)
#
# Open write handle if needed.
#
if options.write_events is not None:
try:
export_file = open(options.write_events, "w")
except (FileNotFoundError, IOError, PermissionError) as e:
print("ERROR: Can't create export file \"{}\": {}".format(
options.write_events, e.strerror))
sys.exit(-1)
#
# Attach the usdt probe.
#
u = USDT(pid=int(options.pid))
try:
u.enable_probe(probe="start_dump", fn_name="probe__start_dump")
u.enable_probe(probe="sweep_done", fn_name="probe__sweep_done")
if not options.no_ukey_count:
u.enable_probe(probe="revalidate_ukey__:entry",
fn_name="probe__reval_entry")
except USDTException as e:
print("ERROR: {}".format(
(re.sub('^', ' ' * 7, str(e), flags=re.MULTILINE)).strip().
replace("--with-dtrace or --enable-dtrace",
"--enable-usdt-probes")))
sys.exit(-1)
#
# Attach probe to running process.
#
source = EBPF_SOURCE.replace("<EVENT_ENUM>", "\n".join(
[" EVENT_{} = {},".format(
event.name, event.value) for event in Event]))
source = source.replace("<BUFFER_PAGE_CNT>",
str(options.buffer_page_count))
source = source.replace("<OVS_INCLUDE_DEFINITIONS>",
get_ovs_definitions("udpif", pid=options.pid,
pahole=options.pahole))
b = BPF(text=source, usdt_contexts=[u], debug=options.debug)
#
# Print header.
#
ltz = datetime.datetime.now()
utc = ltz.astimezone(pytz.utc)
time_string = "# Starting trace @{} ({} UTC)".format(
ltz.isoformat(), utc.strftime("%H:%M:%S"))
if export_file is not None:
export_file.write(time_string + "\n")
print(time_string)
print_csv_header()
#
# Process all events.
b['events'].open_ring_buffer(process_event)
while 1:
try:
b.ring_buffer_poll()
except KeyboardInterrupt:
break
dropcnt = b.get_table("dropcnt")
for k in dropcnt.keys():
count = dropcnt.sum(k).value
if k.value == 0 and count > 0:
print("\n# WARNING: Not all upcalls were captured, {} were "
"dropped!\n# Increase the BPF ring buffer size "
"with the --buffer-page-count option.".format(count))
#
# Display footer.
#
if state["last_not_printed_csv"] is not None:
print(state["last_not_printed_csv"])
ltz = datetime.datetime.now()
utc = ltz.astimezone(pytz.utc)
time_string = "# Stopping trace @{} ({} UTC)".format(
ltz.isoformat(), utc.strftime("%H:%M:%S"))
if export_file is not None:
export_file.write(time_string + "\n")
print(time_string)
#
# Close event file is used.
#
if options.write_events is not None:
export_file.close()
#
# Do final graph if requested.
#
show_graph(graph, gui=not options.no_gui, file_name=options.write_charts)
#
# Start main() as the default entry point...
#
if __name__ == '__main__':
main()
Reference in New Issue Copy Permalink