mirror of
https://github.com/openvswitch/ovs
synced 2025-08-31 06:15:47 +00:00
5b9feacfc8
Consider the following sequence of events: 1. Cluster with 2 nodes - A and B. A is a leader. 2. C connects to A and sends a join request. 3. A sends an append request to C. C is in CATCHUP phase for A. 4. A looses leadership to B. Sends join failure notification to C. 5. C sends append reply to A. 6. A discards append reply (not leader). 7. B looses leadership back to A. 8. C sends a new join request to A. 9. A replies with failure (already in progress). 10. GoTo step 8. At this point A is waiting for an append reply that it already discarded at step 6 and fails all the new attempts of C to join with 'already in progress' verdict. C stays forever in a joining state and in a CATCHUP phase from A's perspective. This is a similar case to a sudden disconnect from a leader fixed in commit999ba294fb("ovsdb: raft: Fix inability to join the cluster after interrupted attempt."), but since we're not disconnecting, the servers are not getting destroyed. Fix that by destroying all the servers that are not yet part of the configuration after leadership is lost. This way, server C will be able to simply re-start the joining process from scratch. New failure test command is added in order to simulate leadership change before we receive the append reply, so it gets discarded. New cluster test is added to exercise this scenario. Fixes:1b1d2e6daa("ovsdb: Introduce experimental support for clustered databases.") Reported-at: https://github.com/ovn-org/ovn/issues/235 Acked-by: Han Zhou <hzhou@ovn.org> Signed-off-by: Ilya Maximets <i.maximets@ovn.org>
1077 lines
41 KiB
Plaintext
1077 lines
41 KiB
Plaintext
OVS_START_SHELL_HELPERS
|
||
# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT USE_LOCAL_CONFIG TRANSACTION...
|
||
ovsdb_check_cluster () {
|
||
set -x
|
||
local n=$1 schema_func=$2 output=$3 local_config=$4
|
||
shift; shift; shift; shift
|
||
|
||
$schema_func > schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
if test X$local_config = X"yes"; then
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-tool create c$i.db $top_srcdir/ovsdb/local-config.ovsschema], [0], [], [stderr])
|
||
local ctxn="[[\"Local_Config\",
|
||
{\"op\": \"insert\", \"table\": \"Config\",
|
||
\"row\": {\"connections\": [\"named-uuid\",\"conn$n\"]}},
|
||
{\"op\": \"insert\", \"table\": \"Connection\", \"uuid-name\": \"conn$n\",
|
||
\"row\": {\"target\": \"punix:s$i.ovsdb\"}}]]"
|
||
|
||
AT_CHECK([ovsdb-tool transact c$i.db "$ctxn"], [0], [ignore], [stderr])
|
||
done
|
||
fi
|
||
AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
local remote=punix:s$i.ovsdb
|
||
local config_db=
|
||
if test X$local_config = X"yes"; then
|
||
remote=db:Local_Config,Config,connections
|
||
config_db=c$i.db
|
||
fi
|
||
AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=$remote s$i.db $config_db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
|
||
done
|
||
|
||
for txn
|
||
do
|
||
AT_CHECK([ovsdb-client -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
|
||
cat stdout >> output
|
||
done
|
||
AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-tool check-cluster s*.db])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
# Test a 1-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (1 server)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
|
||
ovsdb_check_cluster 1 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 3-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (3 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
|
||
ovsdb_check_cluster 3 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 5-server cluster.
|
||
AT_BANNER([OVSDB - clustered transactions (5 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 5])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
|
||
ovsdb_check_cluster 5 "$2" '$4' no m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
# Test a 3-server cluster using a Local_Config db.
|
||
AT_BANNER([OVSDB - clustered transactions Local_Config (3 servers)])
|
||
m4_define([OVSDB_CHECK_EXECUTION],
|
||
[AT_SETUP([$1 - cluster of 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3 Local_Config $5])
|
||
ovsdb_check_cluster 3 "$2" '$4' yes m4_foreach([txn], [$3], ['txn' ])
|
||
AT_CLEANUP])
|
||
EXECUTION_EXAMPLES
|
||
|
||
|
||
AT_BANNER([OVSDB - disconnect from cluster])
|
||
|
||
OVS_START_SHELL_HELPERS
|
||
# ovsdb_test_cluster_disconnect N_SERVERS LEADER_OR_FOLLOWER [CHECK_FLAPPING]
|
||
# Test server disconnected from the cluster.
|
||
# N_SERVERS: Number of servers in the cluster.
|
||
# LEADER_OR_FOLLOWER: The role of the server that is disconnected from the
|
||
# cluster: "leader" or "follower".
|
||
# CHECK_FLAPPING: Whether to check if is_disconnected flapped. "yes", "no".
|
||
ovsdb_test_cluster_disconnect () {
|
||
n=$1
|
||
leader_or_follower=$2
|
||
check_flapping=$3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "simple",
|
||
"row": {"i": 1}}]]'], [0], [ignore], [ignore])
|
||
|
||
# When a node is disconnected from the cluster, the IDL should disconnect
|
||
# and retry even if it uses a single remote, because the remote IP can be
|
||
# a VIP on a load-balance. So we use single remote to test here.
|
||
if test $leader_or_follower = "leader"; then
|
||
target=1
|
||
shutdown=`seq $(($n/2 + 1)) $n`
|
||
cleanup=`seq $(($n/2))`
|
||
else
|
||
target=$n
|
||
|
||
# shutdown followers before the leader (s1) so that there is no chance for
|
||
# s$n to become leader during the process.
|
||
shutdown="`seq 2 $(($n/2 + 1))` 1"
|
||
cleanup=`seq $(($n/2 + 2)) $n`
|
||
fi
|
||
echo shutdown=$shutdown
|
||
echo cleanup=$cleanup
|
||
|
||
# Connect to $target. Use "wait" to trigger a non-op transaction so
|
||
# that test-ovsdb will not quit.
|
||
|
||
txn='[["idltest",
|
||
{"op": "wait",
|
||
"table": "simple",
|
||
"where": [["i", "==", 1]],
|
||
"columns": ["i"],
|
||
"until": "==",
|
||
"rows": [{"i": 1}]}]]'
|
||
|
||
test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -v -t10 idl \
|
||
unix:s$target.ovsdb "$txn" > test-ovsdb.log 2>&1 &
|
||
echo $! > test-ovsdb.pid
|
||
|
||
OVS_WAIT_UNTIL([grep "000: table simple: i=1" test-ovsdb.log])
|
||
|
||
$PYTHON3 $srcdir/test-ovsdb.py -t10 idl $abs_srcdir/idltest.ovsschema \
|
||
unix:s$target.ovsdb "$txn" > test-ovsdb-py.log 2>&1 &
|
||
echo $! > test-ovsdb-py.pid
|
||
|
||
OVS_WAIT_UNTIL([grep "000: table simple: i=1" test-ovsdb-py.log])
|
||
|
||
# Start collecting raft_is_connected logs for $target before shutting down
|
||
# any servers.
|
||
tail -f s$target.log > raft_is_connected.log &
|
||
echo $! > tail.pid
|
||
|
||
# Shutdown the other servers so that $target is disconnected from the cluster.
|
||
for i in $shutdown; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
# The test-ovsdb should detect the disconnect and retry.
|
||
OVS_WAIT_UNTIL([grep disconnect test-ovsdb.log])
|
||
OVS_WAIT_UNTIL([grep disconnect test-ovsdb-py.log])
|
||
|
||
# The $target debug log should show raft_is_connected: false.
|
||
OVS_WAIT_UNTIL([grep "raft_is_connected: false" raft_is_connected.log])
|
||
|
||
# Save the current count of "raft_is_connected: true"
|
||
count_old=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
|
||
echo count_old $count_old
|
||
|
||
if test X$check_flapping = X"yes"; then
|
||
sleep 10
|
||
fi
|
||
# Make sure raft_is_connected didn't flap from false to true.
|
||
count_new=`grep "raft_is_connected: true" raft_is_connected.log | wc -l`
|
||
echo count_new $count_new
|
||
AT_CHECK([test $count_new = $count_old])
|
||
|
||
for i in $cleanup; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
AT_SETUP([OVSDB cluster - follower disconnect from cluster, single remote])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 3 follower
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leader disconnect from cluster, single remote])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 3 leader
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leader disconnect from cluster, check flapping])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
ovsdb_test_cluster_disconnect 5 leader yes
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - initial status should be disconnected])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disconnect])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# Stop all servers, and start the s1 only, to test initial connection status
|
||
# when there is no leader yet.
|
||
for i in `seq 1 $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
i=1
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
|
||
# The initial status should be disconnected. So wait should fail.
|
||
AT_CHECK([ovsdb_client_wait --timeout=1 unix:s$i.ovsdb $schema_name connected], [142], [ignore], [ignore])
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
|
||
AT_BANNER([OVSDB cluster election timer change])
|
||
|
||
AT_SETUP([OVSDB cluster - election timer change])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster timer])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# Change not allowed through follower.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/change-election-timer $schema_name 2000], [2], [], [ignore])
|
||
|
||
# Timer cannot be changed to bigger than 2x the original value.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [2], [], [ignore])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 2000], [0], [dnl
|
||
change of election timer initiated.
|
||
], [])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 2000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 2000"])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/change-election-timer $schema_name 4000], [0], [dnl
|
||
change of election timer initiated.
|
||
], [])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
|
||
# Latest timer should be used after restart
|
||
for i in `seq $n`; do
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s2 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
|
||
# Wait until cluster is ready
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
|
||
done
|
||
|
||
# Latest timer should be restored after DB compact and restart.
|
||
# This is to test the install_snapshot RPC.
|
||
|
||
# Compact online
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$i ovsdb-server/compact])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Election timer: 4000"])
|
||
done
|
||
|
||
# Wait until cluster is ready
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "Leader: unknown"])
|
||
done
|
||
|
||
# Newly joined member should use latest timer value
|
||
AT_CHECK([ovsdb-tool join-cluster s4.db $schema_name unix:s4.raft unix:s1.raft])
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s4.log --pidfile=s4.pid --unixctl=s4 --remote=punix:s4.ovsdb s4.db])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name | grep "Election timer: 4000"])
|
||
# for i in `seq 10`; do
|
||
# ovs-appctl -t "`pwd`"/s4 cluster/status $schema_name
|
||
# sleep 1
|
||
# done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_BANNER([OVSDB cluster install snapshot RPC])
|
||
|
||
AT_SETUP([OVSDB cluster - install snapshot RPC])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster snapshot])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 0}}]]'], [0], [ignore], [ignore])
|
||
|
||
# Kill one follower (s2) and write some data to cluster, so that the follower is falling behind
|
||
printf "\ns2: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s2], [s2.pid])
|
||
|
||
# Delete "i":0 and readd it to get a different UUID for it.
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "delete",
|
||
"table": "indexed",
|
||
"where": [["i", "==", 0]]}]]'], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 0}}]]'], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovsdb-client transact unix:s1.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 1}}]]'], [0], [ignore], [ignore])
|
||
|
||
# Compact leader online to generate snapshot
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 ovsdb-server/compact])
|
||
|
||
# Start the follower s2 again.
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s2.log --pidfile=s2.pid --unixctl=s2 --remote=punix:s2.ovsdb s2.db])
|
||
AT_CHECK([ovsdb_client_wait unix:s2.ovsdb $schema_name connected])
|
||
|
||
# A client transaction through s2. During this transaction, there will be a
|
||
# install_snapshot RPC because s2 detects it is behind and s1 doesn't have the
|
||
# pre_log_index requested by s2 because it is already compacted.
|
||
# After the install_snapshot RPC process, the transaction through s2 should
|
||
# succeed.
|
||
AT_CHECK([ovsdb-client transact unix:s2.ovsdb '[["idltest",
|
||
{"op": "insert",
|
||
"table": "indexed",
|
||
"row": {"i": 2}}]]'], [0], [ignore], [ignore])
|
||
|
||
# The snapshot should overwrite the in-memory contents of the DB on S2
|
||
# without generating any constraint violations. All tree records (0, 1, 2)
|
||
# should be in the DB at this point.
|
||
AT_CHECK([ovsdb-client --no-headings dump unix:s2.ovsdb idltest indexed | uuidfilt | sort -k 2], [0], [dnl
|
||
<0> 0
|
||
<1> 1
|
||
<2> 2
|
||
indexed table
|
||
])
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
AT_BANNER([OVSDB - cluster failure while joining])
|
||
AT_SETUP([OVSDB cluster - follower crash while joining])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster join])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl
|
||
$abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
|
||
dnl Starting followers first, so we can configure them to crash on join.
|
||
for j in `seq $n`; do
|
||
i=$(($n + 1 - $j))
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off dnl
|
||
--detach --no-chdir --log-file=s$i.log dnl
|
||
--pidfile=s$i.pid --unixctl=s$i dnl
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
if test $i != 1; then
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s$i dnl
|
||
cluster/failure-test crash-before-sending-install-snapshot-reply dnl
|
||
| grep -q "engaged"])
|
||
fi
|
||
done
|
||
|
||
dnl Make sure that followers really crashed.
|
||
for i in `seq 2 $n`; do
|
||
OVS_WAIT_WHILE([test -s s$i.pid])
|
||
done
|
||
|
||
dnl Bring them back.
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off dnl
|
||
--detach --no-chdir --log-file=s$i.log dnl
|
||
--pidfile=s$i.pid --unixctl=s$i dnl
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
|
||
dnl Make sure that all servers joined the cluster.
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leadership change after replication while joining])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster join])
|
||
|
||
n=5
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl
|
||
$abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=$(ovsdb-tool db-cid s1.db)
|
||
schema_name=$(ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema)
|
||
for i in $(seq 2 $n); do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill $(cat *.pid)'
|
||
on_exit "
|
||
for i in \$(ls $(pwd)/s[[0-$n]]); do
|
||
ovs-appctl --timeout 1 -t \$i cluster/status $schema_name;
|
||
done
|
||
"
|
||
|
||
dnl Starting servers one by one asking all exisitng servers to transfer
|
||
dnl leadership after append reply forcing the joining server to try another
|
||
dnl one that will also transfer leadership. Since transfer is happening
|
||
dnl after the servers update is replicated to other servers, one of the
|
||
dnl other servers will actually commit it. It may be a new leader from
|
||
dnl one of the old members or the new joining server itself.
|
||
for i in $(seq $n); do
|
||
dnl Make sure that all already started servers joined the cluster.
|
||
for j in $(seq $((i - 1)) ); do
|
||
AT_CHECK([ovsdb_client_wait unix:s$j.ovsdb $schema_name connected])
|
||
done
|
||
for j in $(seq $((i - 1)) ); do
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "$(pwd)"/s$j \
|
||
cluster/failure-test \
|
||
transfer-leadership-after-sending-append-request \
|
||
| grep -q "engaged"])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off \
|
||
--detach --no-chdir --log-file=s$i.log \
|
||
--pidfile=s$i.pid --unixctl=s$i \
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
|
||
dnl Make sure that all servers joined the cluster.
|
||
for i in $(seq $n); do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in $(seq $n); do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([$(pwd)/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - leadership change before replication while joining])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster join])
|
||
|
||
n=5
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db dnl
|
||
$abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=$(ovsdb-tool db-cid s1.db)
|
||
schema_name=$(ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema)
|
||
for i in $(seq 2 $n); do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill $(cat *.pid)'
|
||
on_exit "
|
||
for i in \$(ls $(pwd)/s[[0-$n]]); do
|
||
ovs-appctl --timeout 1 -t \$i cluster/status $schema_name;
|
||
done
|
||
"
|
||
|
||
dnl Starting servers one by one asking all exisitng servers to transfer
|
||
dnl leadership right after starting to add a server. Joining server will
|
||
dnl need to find a new leader that will also transfer leadership.
|
||
dnl This will continue until the same server will not become a leader
|
||
dnl for the second time and will be able to add a new server.
|
||
for i in $(seq $n); do
|
||
dnl Make sure that all already started servers joined the cluster.
|
||
for j in $(seq $((i - 1)) ); do
|
||
AT_CHECK([ovsdb_client_wait unix:s$j.ovsdb $schema_name connected])
|
||
done
|
||
for j in $(seq $((i - 1)) ); do
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "$(pwd)"/s$j \
|
||
cluster/failure-test \
|
||
transfer-leadership-after-starting-to-add \
|
||
| grep -q "engaged"])
|
||
done
|
||
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off \
|
||
--detach --no-chdir --log-file=s$i.log \
|
||
--pidfile=s$i.pid --unixctl=s$i \
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
|
||
dnl Make sure that all servers joined the cluster.
|
||
for i in $(seq $n); do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in $(seq $n); do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([$(pwd)/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
OVS_START_SHELL_HELPERS
|
||
# ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
|
||
ovsdb_cluster_failure_test () {
|
||
# Initial state: s1 is leader, s2 and s3 are followers
|
||
remote_1=$1
|
||
remote_2=$2
|
||
crash_node=$3
|
||
crash_command=$4
|
||
if test "$crash_node" = "1"; then
|
||
new_leader=$5
|
||
fi
|
||
log_grep=$6
|
||
|
||
cp $top_srcdir/vswitchd/vswitch.ovsschema schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
AT_CHECK([sed < stderr "/ovsdb|WARN|schema: changed .* columns in 'Open_vSwitch' database from ephemeral to persistent/d"])
|
||
|
||
n=3
|
||
join_cluster() {
|
||
local i=$1
|
||
others=
|
||
for j in `seq 1 $n`; do
|
||
if test $i != $j; then
|
||
others="$others unix:s$j.raft"
|
||
fi
|
||
done
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
|
||
}
|
||
start_server() {
|
||
local i=$1
|
||
printf "\ns$i: starting\n"
|
||
AT_CHECK([ovsdb-server -vjsonrpc -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
}
|
||
connect_server() {
|
||
local i=$1
|
||
printf "\ns$i: waiting to connect to storage\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
|
||
}
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do join_cluster $i; done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do start_server $i; done
|
||
for i in `seq $n`; do connect_server $i; done
|
||
|
||
db=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
|
||
|
||
# To ensure $new_leader node the new leader, we delay election timer for
|
||
# the other follower.
|
||
if test -n "$new_leader"; then
|
||
if test "$new_leader" = "2"; then
|
||
delay_election_node=3
|
||
else
|
||
delay_election_node=2
|
||
fi
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
|
||
fi
|
||
|
||
# Initializing the database separately to avoid extra 'wait' operation
|
||
# in later transactions.
|
||
AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait init], [0], [ignore], [ignore])
|
||
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
|
||
AT_CHECK([ovs-vsctl -v --db="$db" --no-leader-only --no-shuffle-remotes --no-wait create QoS type=x], [0], [ignore], [ignore])
|
||
|
||
# Make sure that the node really crashed or has specific log message.
|
||
if test -z "$log_grep"; then
|
||
AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
|
||
# XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
|
||
if test "$remote_1" = "$crash_node"; then
|
||
db=unix:s$remote_2.ovsdb
|
||
fi
|
||
else
|
||
OVS_WAIT_UNTIL([grep -q "$log_grep" s${crash_node}.log])
|
||
fi
|
||
AT_CHECK([ovs-vsctl --db="$db" --no-leader-only --no-wait --columns=type --bare list QoS], [0], [x
|
||
])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
AT_BANNER([OVSDB - cluster failure with pending transaction])
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
# XXX: Detect and skip repeated transaction before enabling this test
|
||
AT_CHECK([exit 77])
|
||
ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
|
||
ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB cluster - txn on leader, leader transfers leadership after sending appendReq])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn transfer])
|
||
ovsdb_cluster_failure_test 1 2 1 transfer-leadership-after-sending-append-request -1 "Transferring leadership"
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_SETUP([OVSDB cluster - competing candidates])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster competing-candidates])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
schema_name=`ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema`
|
||
for i in `seq 2 $n`; do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in `seq $n`; do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# We need to simulate the situation when 2 candidates starts election with same
|
||
# term.
|
||
#
|
||
# Before triggering leader election, tell follower s2 don't send vote request (simulating
|
||
# vote-request lost or not handled in time), and tell follower s3 to delay
|
||
# election timer to make sure s3 doesn't send vote-request before s2 enters
|
||
# term 2.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s2 cluster/failure-test dont-send-vote-request], [0], [ignore])
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s3 cluster/failure-test delay-election], [0], [ignore])
|
||
|
||
# Restart leader, which will become follower, and both old followers will start
|
||
# election as candidate. The new follower (old leader) will vote one of them,
|
||
# and the other candidate should step back as follower as again.
|
||
kill -9 `cat s1.pid`
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s1.log --pidfile=s1.pid --unixctl=s1 --remote=punix:s1.ovsdb s1.db])
|
||
|
||
# Tell s1 to delay election timer so that it won't start election before s3
|
||
# becomes candidate.
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s1 cluster/failure-test delay-election], [0], [ignore])
|
||
|
||
OVS_WAIT_UNTIL([ovs-appctl -t "`pwd`"/s1 cluster/status $schema_name | grep "Term: 2"])
|
||
|
||
for i in `seq $n`; do
|
||
OVS_WAIT_WHILE([ovs-appctl -t "`pwd`"/s$i cluster/status $schema_name | grep "candidate"])
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
for i in `seq $n`; do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_SETUP([OVSDB cluster - disruptive server])
|
||
AT_KEYWORDS([ovsdb server negative unix cluster disruptive])
|
||
|
||
n=3
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster \
|
||
s1.db $abs_srcdir/idltest.ovsschema unix:s1.raft], [0], [], [stderr])
|
||
cid=$(ovsdb-tool db-cid s1.db)
|
||
schema_name=$(ovsdb-tool schema-name $abs_srcdir/idltest.ovsschema)
|
||
for i in $(seq 2 $n); do
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema_name unix:s$i.raft unix:s1.raft])
|
||
done
|
||
|
||
on_exit 'kill $(cat *.pid)'
|
||
for i in $(seq $n); do
|
||
AT_CHECK([ovsdb-server -v -vconsole:off -vsyslog:off --detach --no-chdir \
|
||
--log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i \
|
||
--remote=punix:s$i.ovsdb s$i.db])
|
||
done
|
||
for i in $(seq $n); do
|
||
AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema_name connected])
|
||
done
|
||
|
||
# An unstable follower shouldn't disrupt the healthy cluster - shouldn't
|
||
# trigger term change.
|
||
AT_CHECK([ovs-appctl -t $(pwd)/s2 cluster/failure-test stop-raft-rpc], [0], [ignore])
|
||
OVS_WAIT_UNTIL([ovs-appctl -t $(pwd)/s2 cluster/status $schema_name | grep "Role: candidate"])
|
||
AT_CHECK([ovs-appctl -t $(pwd)/s2 cluster/failure-test clear], [0], [ignore])
|
||
|
||
# Should step back to follower.
|
||
OVS_WAIT_UNTIL([ovs-appctl -t $(pwd)/s2 cluster/status $schema_name | grep "Role: follower"])
|
||
|
||
# No term change.
|
||
for i in $(seq $n); do
|
||
AT_CHECK([ovs-appctl -t $(pwd)/s$i cluster/status $schema_name | grep "Term: 1"], [0], [ignore])
|
||
done
|
||
|
||
for i in $(seq $n); do
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([$(pwd)/s$i], [s$i.pid])
|
||
done
|
||
|
||
AT_CLEANUP
|
||
|
||
|
||
AT_BANNER([OVSDB - cluster tests])
|
||
|
||
# Torture test.
|
||
OVS_START_SHELL_HELPERS
|
||
ovsdb_torture_test () {
|
||
local n=$1 # Number of cluster members
|
||
local victim=$2 # Cluster member to kill or remove
|
||
local variant=$3 # 'kill' and restart or 'remove' and add
|
||
cp $top_srcdir/vswitchd/vswitch.ovsschema schema
|
||
schema=`ovsdb-tool schema-name schema`
|
||
AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
|
||
AT_CHECK([sed < stderr "/ovsdb|WARN|schema: changed .* columns in 'Open_vSwitch' database from ephemeral to persistent/d"])
|
||
|
||
join_cluster() {
|
||
local i=$1
|
||
others=
|
||
for j in `seq 1 $n`; do
|
||
if test $i != $j; then
|
||
others="$others unix:s$j.raft"
|
||
fi
|
||
done
|
||
AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
|
||
}
|
||
|
||
start_server() {
|
||
local i=$1
|
||
printf "\ns$i: starting\n"
|
||
AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
|
||
}
|
||
stop_server() {
|
||
local i=$1
|
||
printf "\ns$i: stopping\n"
|
||
OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
|
||
}
|
||
connect_server() {
|
||
local i=$1
|
||
printf "\ns$i: waiting to connect to storage\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
|
||
}
|
||
remove_server() {
|
||
local i=$1
|
||
printf "\ns$i: removing from cluster\n"
|
||
AT_CHECK([ovs-appctl -t "`pwd`"/s$i cluster/leave Open_vSwitch])
|
||
printf "\ns$i: waiting for removal to complete\n"
|
||
AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
|
||
stop_server $i
|
||
}
|
||
add_server() {
|
||
local i=$1
|
||
rm s$i.db
|
||
join_cluster $i
|
||
start_server $i
|
||
connect_server $i
|
||
}
|
||
|
||
cid=`ovsdb-tool db-cid s1.db`
|
||
for i in `seq 2 $n`; do join_cluster $i; done
|
||
|
||
on_exit 'kill `cat *.pid`'
|
||
for i in `seq $n`; do start_server $i; done
|
||
for i in `seq $n`; do connect_server $i; done
|
||
|
||
db=unix:s1.ovsdb
|
||
for i in `seq 2 $n`; do
|
||
db=$db,unix:s$i.ovsdb
|
||
done
|
||
|
||
n1=10 n2=5 n3=50
|
||
echo "starting $n1*$n2 ovs-vsctl processes..."
|
||
for i in $(seq 0 $(expr $n1 - 1) ); do
|
||
(for j in $(seq $n2); do
|
||
: > $i-$j.running
|
||
txn="add Open_vSwitch . external_ids $i-$j=$i-$j"
|
||
for k in $(seq $n3); do
|
||
txn="$txn -- add Open_vSwitch . external_ids $i-$j-$k=$i-$j-$k"
|
||
done
|
||
run_as "ovs-vsctl($i-$j)" ovs-vsctl "-vPATTERN:console:ovs-vsctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --db="$db" --no-leader-only --no-wait $txn
|
||
status=$?
|
||
if test $status != 0; then
|
||
echo "$i-$j exited with status $status" > $i-$j:$status
|
||
fi
|
||
rm $i-$j.running
|
||
done
|
||
: > $i.done)&
|
||
done
|
||
echo "...done"
|
||
|
||
echo "waiting for ovs-vsctl processes to exit..."
|
||
# Use file instead of var because code inside "while" runs in a subshell.
|
||
echo 0 > phase
|
||
i=0
|
||
(while :; do echo || exit 0; sleep 0.1; done) | while read REPLY; do
|
||
printf "t=%2d s:" $i
|
||
done=0
|
||
for j in $(seq 0 $(expr $n1 - 1)); do
|
||
if test -f $j.done; then
|
||
printf " $j"
|
||
done=$(expr $done + 1)
|
||
fi
|
||
done
|
||
printf '\n'
|
||
if test $done = $n1; then
|
||
break
|
||
fi
|
||
|
||
case $(cat phase) in # (
|
||
0)
|
||
if test $done -ge $(expr $n1 / 10); then
|
||
if test $variant = kill; then
|
||
stop_server $victim
|
||
else
|
||
remove_server $victim
|
||
fi
|
||
echo 1 > phase
|
||
next=$(expr $i + 2)
|
||
fi
|
||
;; # (
|
||
1)
|
||
if test $i -ge $next; then
|
||
if test $variant = kill; then
|
||
start_server $victim
|
||
connect_server $victim
|
||
else
|
||
add_server $victim
|
||
fi
|
||
echo 2 > phase
|
||
fi
|
||
;;
|
||
esac
|
||
|
||
i=$(expr $i + 1)
|
||
done
|
||
echo "...done"
|
||
AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
|
||
|
||
for i in $(seq 0 $(expr $n1 - 1) ); do
|
||
for j in `seq $n2`; do
|
||
echo "$i-$j=$i-$j"
|
||
for k in `seq $n3`; do
|
||
echo "$i-$j-$k=$i-$j-$k"
|
||
done
|
||
done
|
||
done | sort > expout
|
||
AT_CHECK([ovs-vsctl --db="$db" --no-wait --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get Open_vSwitch . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])
|
||
|
||
for i in `seq $n`; do
|
||
if test $i != $victim || test $(cat phase) != 1; then
|
||
stop_server $i
|
||
fi
|
||
done
|
||
|
||
# We ignore stdout because non-fatal warnings get printed there.
|
||
AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
|
||
}
|
||
OVS_END_SHELL_HELPERS
|
||
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 1 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 2 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 3 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 1 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 2 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 3 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 4 kill
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 5 kill
|
||
AT_CLEANUP
|
||
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 1 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 2 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
|
||
ovsdb_torture_test 3 3 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 1 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 2 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 3 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 4 remove
|
||
AT_CLEANUP
|
||
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
|
||
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
|
||
ovsdb_torture_test 5 5 remove
|
||
AT_CLEANUP
|