mirror of
https://github.com/openvswitch/ovs
synced 2025-08-28 21:07:47 +00:00
6f24c2bc76
The only way to configure settings on a remote (e.g. inactivity_probe) is via --remote=db:DB,table,row. There is no way to do this via the existing CLI options. For a clustered DB with multiple servers listening on unique addresses there is no way to store these entries in the DB as the DB is shared. For example, three servers listening on 1.1.1.1, 1.1.1.2, and 1.1.1.3 respectively would require a Manager/Connection row each, but then all three servers would try to listen on all three addresses. It is possible for ovsdb-server to serve multiple databases. This means that we can have a local "config" database in addition to the main database we are serving (Open_vSwitch, OVN_Southbound, etc.) and this patch adds a Local_Config schema that currently just mirrors the Connection table and a Config table with a 'connections' row that stores each Connection. Signed-off-by: Terry Wilson <twilson@redhat.com> Acked-by: Dumitru Ceara <dceara@redhat.com> Signed-off-by: Ilya Maximets <i.maximets@ovn.org>
940 lines
36 KiB
Plaintext
940 lines
36 KiB
Plaintext
OVS_START_SHELL_HELPERS
|
||
# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT USE_LOCAL_CONFIG TRANSACTION...
|
||
ovsdb_check_cluster () {
|
||
set -x
|
||
local n=$1 schema_func=$2 output=$3 local_config=$4
|
||
shift; shift; shift; shift
|
||
|
||
$schema_func > schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
if test X$local_config == X"yes"; then
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-tool create c$i.db $top_srcdir/ovsdb/local-config.ovsschema], [0], [], [stderr])
|
||
local ctxn="[[\"Local_Config\",
|
||
{\"op\": \"insert\", \"table\": \"Config\",
|
||
\"row\": {\"connections\": [\"named-uuid\",\"conn$n\"]}},
|
||
{\"op\": \"insert\", \"table\": \"Connection\", \"uuid-name\": \"conn$n\",
|
||
\"row\": {\"target\": \"punix:s$i.ovsdb\"}}]]"
|
||
|
||
AT_CHECK([ovsdb-tool transact c$i.db "$ctxn"], [0], [ignore], [stderr])
|
||
done
|
||
fi
|
||
AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
local remote=punix:s$i.ovsdb
|
||
local config_db=
|
||
if test X$local_config == X"yes"; then
|
||
remote=db:Local_Config,Config,connections
|
||
config_db=c$i.db
|
||
fi
|
||
AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=$remote s$i.db $config_db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
|
||
done
|
||
|
||
for txn
|
||
do
|
||
AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
|
||
cat stdout >> output
|
||
done
|
||
AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-tool check-cluster s*.db])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
# Test a 1-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (1 server)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
|
||
ovsdb_check_cluster 1 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 3-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (3 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
|
||
ovsdb_check_cluster 3 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 5-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (5 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 5])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
|
||
ovsdb_check_cluster 5 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 3-server cluster using a Local_Config db.
|
||
AT_BANNER([OVSDB - clustered transactions Local_Config (3 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3 Local_Config $5])
|
||
ovsdb_check_cluster 3 "$2" '$4' yes m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
|
||
AT_BANNER([OVSDB - disconnect from cluster])
|
||
|
||
OVS_START_SHELL_HELPERS
|
||
# ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING]
|
||
# Test server disconnected from the cluster.
|
||
# N_SERVERS: Number of servers in the cluster.
|
||
# LEADER_OR_FOLLOWER: The role of the server that is disconnected from the
|
||
# cluster: "leader" or "follower".
|
||
# CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no".
|
||
ovsdb_test_cluster_disconnect () {
|
||
n=$1
|
||
leader_or_follower=$2
|
||
check_flapping=$3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "simple",
|
||
"row": {"i": 1}}]]'], [0], [ignore], [ignore])
|
||
|
||
# When a node is disconnected from the cluster, the IDL should disconnect
|
||
# and retry even if it uses a single remote, because the remote IP can be
|
||
# a VIP on a load-balance. So we use single remote to test here.
|
||
if test $leader_or_follower == "leader"; then
|
||
target=1
|
||
shutdown=`seq $(($n/2 + 1)) $n`
|
||
cleanup=`seq $(($n/2))`
|
||
else
|
||
target=$n
|
||
|
||
# shutdown followers before the leader (s1) so that there is no chance for
|
||
# s$n to become leader during the process.
|
||
shutdown="`seq 2 $(($n/2 + 1))` 1"
|
||
cleanup=`seq $(($n/2 + 2)) $n`
|
||
fi
|
||
echo shutdown=$shutdown
|
||
echo cleanup=$cleanup
|
||
|
||
# Connect to $target. Use "wait" to trigger a non-op transaction so
|
||
# that test-ovsdb will not quit.
|
||
|
||
txn='[["idltest",
|
||
{"op": "wait",
|
||
"table": "simple",
|
||
"where": [["i", "==", 1]],
|
||
"columns": ["i"],
|
||
"until": "==",
|
||
"rows": [{"i": 1}]}]]'
|
||
|
||
test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl \
|
||
unix:s$target.ovsdb "$txn" > test-ovsdb.log 2>&1 &
|
||
echo $! > test-ovsdb.pid
|
||
|
||
OVS_WAIT_UNTIL([grep "000: table simple: i=1" test-ovsdb.log])
|
||
|
||
$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $abs_srcdir/idltest.ovsschema \
|
||
unix:s$target.ovsdb "$txn" > test-ovsdb-py.log 2>&1 &
|
||
echo $! > test-ovsdb-py.pid
|
||
|
||
OVS_WAIT_UNTIL([grep "000: table simple: i=1" test-ovsdb-py.log])
|
||
|
||
# Start collecting raft_is_connected logs for $target before shutting down
|
||
# any servers.
|
||
tail -f s$target.log > raft_is_connected.log &
|
||
echo $! > tail.pid
|
||
|
||
# Shutdown the other servers so that $target is disconnected from the cluster.
|
||
for i in $shutdown; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
# The test-ovsdb should detect the disconnect and retry.
|
||
OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log])
|
||
OVS_WAIT_UNTIL([grep disconnect test-ovsdb-py.log])
|
||
|
||
# The $target debug log should show raft_is_connected: false.
|
||
OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log])
|
||
|
||
# Save the current count of "raft_is_connected: true"
|
||
count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
|
||
echo count_old $count_old
|
||
|
||
if test X$check_flapping == X"yes"; then
|
||
sleep 10
|
||
fi
|
||
# Make sure raft_is_connected didn't flap from false to true.
|
||
count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
|
||
echo count_new $count_new
|
||
AT_CHECK([test $count_new == $count_old])
|
||
|
||
for i in $cleanup; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 3 follower
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 3 leader
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 5 leader yes
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - initial status should be disconnected])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
|
||
n=3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# Stop all servers, and start the s1 only, to test initial connection status
|
||
# when there is no leader yet.
|
||
for i in `seq 1 $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
i=1
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
|
||
# The initial status should be disconnected. So wait should fail.
|
||
AT_CHECK([ovsdb_client_wait --timeout=1 unix:s$i.ovsdb $schema_name connected], [142], [ignore], [ignore])
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
|
||
AT_BANNER([OVSDB cluster election timer change])
|
||
|
||
AT_SETUP([OVSDB cluster - election timer change])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster timer])
|
||
|
||
n=3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# Change not allowed through follower.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore])
|
||
|
||
# Timer cannot be changed to bigger than 2x the original value.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl
|
||
change of election timer initiated.
|
||
], [])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl
|
||
change of election timer initiated.
|
||
], [])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
|
||
# Latest timer should be used after restart
|
||
for i in `seq $n`; do
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
|
||
# Wait until cluster is ready
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
|
||
done
|
||
|
||
# Latest timer should be restored after DB compact and restart.
|
||
# This is to test the install_snapshot RPC.
|
||
|
||
# Compact online
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$i ovsdb-server/compact])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Election timer: 4000"])
|
||
done
|
||
|
||
# Wait until cluster is ready
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
|
||
done
|
||
|
||
# Newly joined member should use latest timer value
|
||
AT_CHECK([ovsdb-tool join-cluster s4.db $schema_name unix:s4.raft unix:s1.raft])
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s4.log --pidfile=s4.pid --unixctl=s4 --remote=punix:s4.ovsdb s4.db])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
# for i in `seq 10`; do
|
||
# ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name
|
||
# sleep 1
|
||
# done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_BANNER([OVSDB cluster install snapshot RPC])
|
||
|
||
AT_SETUP([OVSDB cluster - install snapshot RPC])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster snapshot])
|
||
|
||
n=3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 0}}]]'], [0], [ignore], [ignore])
|
||
|
||
# Kill one follower (s2) and write some data to cluster, so that the follower is falling behind
|
||
printf "\ns2: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s2], [s2.pid])
|
||
|
||
# Delete "i":0 and readd it to get a different UUID for it.
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "delete",
|
||
"table": "indexed",
|
||
"where": [["i", "==", 0]]}]]'], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 0}}]]'], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 1}}]]'], [0], [ignore], [ignore])
|
||
|
||
# Compact leader online to generate snapshot
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 ovsdb-server/compact])
|
||
|
||
# Start the follower s2 again.
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s2.log --pidfile=s2.pid --unixctl=s2 --remote=punix:s2.ovsdb s2.db])
|
||
AT_CHECK([ovsdb_client_wait unix:s2.ovsdb $schema_name connected])
|
||
|
||
# A client transaction through s2. During this transaction, there will be a
|
||
# install_snapshot RPC because s2 detects it is behind and s1 doesn't have the
|
||
# pre_log_index requested by s2 because it is already compacted.
|
||
# After the install_snapshot RPC process, the transaction through s2 should
|
||
# succeed.
|
||
AT_CHECK([ovsdb-client transact unix:s2.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 2}}]]'], [0], [ignore], [ignore])
|
||
|
||
# The snapshot should overwrite the in-memory contents of the DB on S2
|
||
# without generating any constraint violations. All tree records (0, 1, 2)
|
||
# should be in the DB at this point.
|
||
AT_CHECK([ovsdb-client --no-headings dump unix:s2.ovsdb idltest indexed | uuidfilt | sort -k 2], [0], [dnl
|
||
<0> 0
|
||
<1> 1
|
||
<2> 2
|
||
indexed table
|
||
])
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
AT_BANNER([OVSDB - cluster failure while joining])
|
||
AT_SETUP([OVSDB cluster - follower crash while joining])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster join])
|
||
|
||
n=3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl
|
||
$abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
|
||
dnl Starting followers first, so we can configure them to crash on join.
|
||
for j in `seq $n`; do
|
||
i=$(($n + 1 - $j))
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off dnl
|
||
--detach --no-chdir --log-file=s$i.log dnl
|
||
--pidfile=s$i.pid --unixctl=s$i dnl
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
if test $i != 1; then
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i dnl
|
||
cluster/failure-test crash-before-sending-install-snapshot-reply dnl
|
||
| grep -q "engaged"])
|
||
fi
|
||
done
|
||
|
||
dnl Make sure that followers really crashed.
|
||
for i in `seq 2 $n`; do
|
||
OVS_WAIT_WHILE([test -s s$i.pid])
|
||
done
|
||
|
||
dnl Bring them back.
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off dnl
|
||
--detach --no-chdir --log-file=s$i.log dnl
|
||
--pidfile=s$i.pid --unixctl=s$i dnl
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
|
||
dnl Make sure that all servers joined the cluster.
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
|
||
OVS_START_SHELL_HELPERS
|
||
# ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
|
||
ovsdb_cluster_failure_test () {
|
||
# Initial state: s1 is leader, s2 and s3 are followers
|
||
remote_1=$1
|
||
remote_2=$2
|
||
crash_node=$3
|
||
crash_command=$4
|
||
if test "$crash_node" == "1"; then
|
||
new_leader=$5
|
||
fi
|
||
log_grep=$6
|
||
|
||
cp $top_srcdir/vswitchd/vswitch.ovsschema schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
AT_CHECK([sed < stderr "/ovsdb|WARN|schema: changed .* columns in 'Open_vSwitch' database from ephemeral to persistent/d"])
|
||
|
||
n=3
|
||
join_cluster() {
|
||
local i=$1
|
||
others=
|
||
for j in `seq 1 $n`; do
|
||
if test $i != $j; then
|
||
others="$others unix:s$j.raft"
|
||
fi
|
||
done
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
|
||
}
|
||
start_server() {
|
||
local i=$1
|
||
printf "\ns$i: starting\n"
|
||
AT_CHECK([ovsdb-server -vjsonrpc -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
}
|
||
connect_server() {
|
||
local i=$1
|
||
printf "\ns$i: waiting to connect to storage\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
|
||
}
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do join_cluster $i; done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do start_server $i; done
|
||
for i in `seq $n`; do connect_server $i; done
|
||
|
||
db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
|
||
|
||
# To ensure $new_leader node the new leader, we delay election timer for
|
||
# the other follower.
|
||
if test -n "$new_leader"; then
|
||
if test "$new_leader" == "2"; then
|
||
delay_election_node=3
|
||
else
|
||
delay_election_node=2
|
||
fi
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
|
||
fi
|
||
|
||
# Initializing the database separately to avoid extra 'wait' operation
|
||
# in later transactions.
|
||
AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait init], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
|
||
AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore])
|
||
|
||
# Make sure that the node really crashed or has specific log message.
|
||
if test -z "$log_grep"; then
|
||
AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
|
||
# XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
|
||
if test "$remote_1" = "$crash_node"; then
|
||
db=unix:s$remote_2.ovsdb
|
||
fi
|
||
else
|
||
OVS_WAIT_UNTIL([grep -q "$log_grep" s${crash_node}.log])
|
||
fi
|
||
AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x
|
||
])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
AT_BANNER([OVSDB - cluster failure with pending transaction])
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader transfers leadership after sending appendReq])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn transfer])
|
||
ovsdb_cluster_failure_test 1 2 1 transfer-leadership-after-sending-append-request -1 "Transferring leadership"
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_SETUP([OVSDB cluster - competing candidates])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster competing-candidates])
|
||
|
||
n=3
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
ordinal_schema > schema
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# We need to simulate the situation when 2 candidates starts election with same
|
||
# term.
|
||
#
|
||
# Before triggering leader election, tell follower s2 don't send vote request (simulating
|
||
# vote-request lost or not handled in time), and tell follower s3 to delay
|
||
# election timer to make sure s3 doesn't send vote-request before s2 enters
|
||
# term 2.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/failure-test dont-send-vote-request], [0], [ignore])
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s3 cluster/failure-test delay-election], [0], [ignore])
|
||
|
||
# Restart leader, which will become follower, and both old followers will start
|
||
# election as candidate. The new follower (old leader) will vote one of them,
|
||
# and the other candidate should step back as follower as again.
|
||
kill -9 `cat s1.pid`
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s1.log --pidfile=s1.pid --unixctl=s1 --remote=punix:s1.ovsdb s1.db])
|
||
|
||
# Tell s1 to delay election timer so that it won't start election before s3
|
||
# becomes candidate.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/failure-test delay-election], [0], [ignore])
|
||
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Term: 2"])
|
||
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "candidate"])
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_BANNER([OVSDB - cluster tests])
|
||
|
||
# Torture test.
|
||
OVS_START_SHELL_HELPERS
|
||
ovsdb_torture_test () {
|
||
local n=$1 # Number of cluster members
|
||
local victim=$2 # Cluster member to kill or remove
|
||
local variant=$3 # 'kill' and restart or 'remove' and add
|
||
cp $top_srcdir/vswitchd/vswitch.ovsschema schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
AT_CHECK([sed < stderr "/ovsdb|WARN|schema: changed .* columns in 'Open_vSwitch' database from ephemeral to persistent/d"])
|
||
|
||
join_cluster() {
|
||
local i=$1
|
||
others=
|
||
for j in `seq 1 $n`; do
|
||
if test $i != $j; then
|
||
others="$others unix:s$j.raft"
|
||
fi
|
||
done
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
|
||
}
|
||
|
||
start_server() {
|
||
local i=$1
|
||
printf "\ns$i: starting\n"
|
||
AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
}
|
||
stop_server() {
|
||
local i=$1
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
}
|
||
connect_server() {
|
||
local i=$1
|
||
printf "\ns$i: waiting to connect to storage\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
|
||
}
|
||
remove_server() {
|
||
local i=$1
|
||
printf "\ns$i: removing from cluster\n"
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch])
|
||
printf "\ns$i: waiting for removal to complete\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
|
||
stop_server $i
|
||
}
|
||
add_server() {
|
||
local i=$1
|
||
rm s$i.db
|
||
join_cluster $i
|
||
start_server $i
|
||
connect_server $i
|
||
}
|
||
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do join_cluster $i; done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do start_server $i; done
|
||
for i in `seq $n`; do connect_server $i; done
|
||
|
||
db=unix:s1.ovsdb
|
||
for i in `seq 2 $n`; do
|
||
db=$db,unix:s$i.ovsdb
|
||
done
|
||
|
||
n1=10 n2=5 n3=50
|
||
echo "starting $n1*$n2 ovs-vsctl processes..."
|
||
for i in $(seq 0 $(expr $n1 - 1) ); do
|
||
(for j in $(seq $n2); do
|
||
: > $i-$j.running
|
||
txn="add Open_vSwitch . external_ids $i-$j=$i-$j"
|
||
for k in $(seq $n3); do
|
||
txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k"
|
||
done
|
||
run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn
|
||
status=$?
|
||
if test $status != 0; then
|
||
echo "$i-$j exited with status $status" > $i-$j:$status
|
||
fi
|
||
rm $i-$j.running
|
||
done
|
||
: > $i.done)&
|
||
done
|
||
echo "...done"
|
||
|
||
echo "waiting for ovs-vsctl processes to exit..."
|
||
# Use file instead of var because code inside "while" runs in a subshell.
|
||
echo 0 > phase
|
||
i=0
|
||
(while :; do echo || exit 0; sleep 0.1; done) | while read REPLY; do
|
||
printf "t=%2d s:" $i
|
||
done=0
|
||
for j in $(seq 0 $(expr $n1 - 1)); do
|
||
if test -f $j.done; then
|
||
printf " $j"
|
||
done=$(expr $done + 1)
|
||
fi
|
||
done
|
||
printf '\n'
|
||
if test $done = $n1; then
|
||
break
|
||
fi
|
||
|
||
case $(cat phase) in # (
|
||
0)
|
||
if test $done -ge $(expr $n1 / 10); then
|
||
if test $variant = kill; then
|
||
stop_server $victim
|
||
else
|
||
remove_server $victim
|
||
fi
|
||
echo 1 > phase
|
||
next=$(expr $i + 2)
|
||
fi
|
||
;; # (
|
||
1)
|
||
if test $i -ge $next; then
|
||
if test $variant = kill; then
|
||
start_server $victim
|
||
connect_server $victim
|
||
else
|
||
add_server $victim
|
||
fi
|
||
echo 2 > phase
|
||
fi
|
||
;;
|
||
esac
|
||
|
||
i=$(expr $i + 1)
|
||
done
|
||
echo "...done"
|
||
AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
|
||
|
||
for i in $(seq 0 $(expr $n1 - 1) ); do
|
||
for j in `seq $n2`; do
|
||
echo "$i-$j=$i-$j"
|
||
for k in `seq $n3`; do
|
||
echo "$i-$j-$k=$i-$j-$k"
|
||
done
|
||
done
|
||
done | sort > expout
|
||
AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])
|
||
|
||
for i in `seq $n`; do
|
||
if test $i != $victim || test $(cat phase) != 1; then
|
||
stop_server $i
|
||
fi
|
||
done
|
||
|
||
# We ignore stdout because non-fatal warnings get printed there.
|
||
AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 1 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 2 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 3 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 1 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 2 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 3 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 4 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 5 kill
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 1 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 2 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 3 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 1 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 2 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 3 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 4 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 5 remove
|
||
AT_CLEANUP
|