[#1402] Watch for invalid partner state

If the local server detects that its partner is in an unexpected state
it will transition to the waiting state. Insufficient checks were indicated
in the review comments.

src/hooks/dhcp/high_availability/ha_messages.cc

@@ -1,4 +1,4 @@
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Tue Dec 22 2020 09:50
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes
 
 #include <cstddef>
 #include <log/message_types.h>
@@ -43,6 +43,9 @@ extern const isc::log::MessageID HA_HEARTBEAT_HANDLER_FAILED = "HA_HEARTBEAT_HAN
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW = "HA_HIGH_CLOCK_SKEW";
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION = "HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION";
 extern const isc::log::MessageID HA_INIT_OK = "HA_INIT_OK";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY = "HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY = "HA_INVALID_PARTNER_STATE_HOT_STANDBY";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING = "HA_INVALID_PARTNER_STATE_LOAD_BALANCING";
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_FAILED = "HA_LEASES4_COMMITTED_FAILED";
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE = "HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE";
 extern const isc::log::MessageID HA_LEASES6_COMMITTED_FAILED = "HA_LEASES6_COMMITTED_FAILED";
@@ -134,6 +137,9 @@ const char* values[] = {
     "HA_HIGH_CLOCK_SKEW", "%1, please synchronize clocks!",
     "HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION", "%1, causing HA service to terminate",
     "HA_INIT_OK", "loading High Availability hooks library successful",
+    "HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY", "partner is in the communication-recovery state unexpectedly",
+    "HA_INVALID_PARTNER_STATE_HOT_STANDBY", "partner is in the hot-standby state unexpectedly",
+    "HA_INVALID_PARTNER_STATE_LOAD_BALANCING", "partner is in the load-balancing state unexpectedly",
     "HA_LEASES4_COMMITTED_FAILED", "leases4_committed callout failed: %1",
     "HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE", "%1: leases4_committed callout was invoked without any leases",
     "HA_LEASES6_COMMITTED_FAILED", "leases6_committed callout failed: %1",

src/hooks/dhcp/high_availability/ha_messages.h

@@ -1,4 +1,4 @@
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Tue Dec 22 2020 09:50
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes
 
 #ifndef HA_MESSAGES_H
 #define HA_MESSAGES_H
@@ -44,6 +44,9 @@ extern const isc::log::MessageID HA_HEARTBEAT_HANDLER_FAILED;
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW;
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION;
 extern const isc::log::MessageID HA_INIT_OK;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING;
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_FAILED;
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE;
 extern const isc::log::MessageID HA_LEASES6_COMMITTED_FAILED;

src/hooks/dhcp/high_availability/ha_messages.mes

@@ -228,6 +228,24 @@ servers to resume the HA service.
 This informational message indicates that the High Availability hooks library
 has been loaded successfully.
 
+% HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY partner is in the communication-recovery state unexpectedly
+This warning message is issued when a partner is in the communication-recovery
+state, and this server is not running in the load balancing mode. The server
+may only transition to the communication-recovery state when it runs in the
+load balancing mode. The HA mode of both servers must be the same.
+
+% HA_INVALID_PARTNER_STATE_HOT_STANDBY partner is in the hot-standby state unexpectedly
+This warning message is issued when a partner is in the hot-standby state,
+and this server is not running in the hot standby mode. The server may only
+transition to the hot-standby state when it runs in the hot standby mode.
+The HA mode of both servers must be the same.
+
+% HA_INVALID_PARTNER_STATE_LOAD_BALANCING partner is in the load-balancing state unexpectedly
+This warning message is issued when a partner is in the load-balancing state,
+and this server is not running in the load balancing mode. The server may only
+transition to the load-balancing state when it runs in the load balancing mode.
+The HA mode of both servers must be the same.
+
 % HA_LEASES4_COMMITTED_FAILED leases4_committed callout failed: %1
 This error message is issued when the callout for the leases4_committed hook
 point failed. This includes unexpected errors like wrong arguments provided to

src/hooks/dhcp/high_availability/ha_service.cc

@@ -184,6 +184,9 @@ HAService::communicationRecoveryHandler() {
     } else if (shouldTerminate()) {
         verboseTransition(HA_TERMINATED_ST);
 
+    } else if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+
     } else {
 
         // Transitions based on the partner's state.
@@ -286,6 +289,14 @@ HAService::normalStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_IN_MAINTENANCE_ST:
         verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
@@ -397,6 +408,14 @@ HAService::partnerDownStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_HOT_STANDBY_ST:
     case HA_LOAD_BALANCING_ST:
@@ -504,6 +523,14 @@ HAService::readyStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_HOT_STANDBY_ST:
     case HA_LOAD_BALANCING_ST:
@@ -573,6 +600,14 @@ HAService::syncingStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     // We don't want to perform synchronous attempt to synchronize with
     // a partner until we know that the partner is responding. Therefore,
     // we wait for the heartbeat to complete successfully before we
@@ -695,6 +730,14 @@ HAService::waitingStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's sit in the waiting state until the configuration
+    // is corrected.
+    if (isPartnerStateInvalid()) {
+        postNextEvent(NOP_EVT);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_COMMUNICATION_RECOVERY_ST:
     case HA_HOT_STANDBY_ST:
@@ -968,6 +1011,36 @@ HAService::isMaintenanceCanceled() const {
     return (getLastEvent() == HA_MAINTENANCE_CANCEL_EVT);
 }
 
+bool
+HAService::isPartnerStateInvalid() const {
+    switch (communication_state_->getPartnerState()) {
+        case HA_COMMUNICATION_RECOVERY_ST:
+            if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY);
+                return (true);
+            }
+            break;
+
+        case HA_HOT_STANDBY_ST:
+            if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_HOT_STANDBY);
+                return (true);
+            }
+            break;
+
+        case HA_LOAD_BALANCING_ST:
+            if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_LOAD_BALANCING);
+                return (true);
+            }
+            break;
+
+       default:
+           ;
+    }
+    return (false);
+}
+
 size_t
 HAService::asyncSendLeaseUpdates(const dhcp::Pkt4Ptr& query,
                                  const dhcp::Lease4CollectionPtr& leases,

src/hooks/dhcp/high_availability/ha_service.h

@@ -470,6 +470,23 @@ protected:
     /// @return true if the maintenance was canceled, false otherwise.
     bool isMaintenanceCanceled() const;
 
+    /// @brief Indicates if the partner's state is invalid.
+    ///
+    /// Partner's state is invalid from the local server's perspective when the
+    /// remote server can't transition to this state if the configuration is
+    /// consistent with the local server's configuration.
+    ///
+    /// The following cases are currently checked:
+    /// - partner in communication-recovery state but this server not in the
+    ///   load balancing mode,
+    /// - partner in the hot-standby state but this server not in the hot standby
+    ///   mode,
+    /// - partner in the load-balancing state but this server not in the
+    ///   load balancing mode.
+    ///
+    /// @return true if the partner's state is invalid, false otherwise.
+    bool isPartnerStateInvalid() const;
+
 public:
 
     /// @brief Schedules asynchronous IPv4 leases updates.

src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc

@@ -5376,6 +5376,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5411,6 +5414,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5446,6 +5452,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -5481,6 +5490,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_WAITING_ST));
 
@@ -5516,6 +5528,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -5551,6 +5566,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5586,6 +5604,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -5721,6 +5742,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5756,6 +5780,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5791,6 +5818,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -5826,6 +5856,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_WAITING_ST));
 
@@ -5861,6 +5894,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -5896,6 +5932,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5931,6 +5970,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6373,11 +6415,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("HOT STANDBY state transitions");
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6410,6 +6455,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -6448,6 +6496,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
 
@@ -6480,6 +6531,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6513,11 +6567,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("READY state transitions");
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6548,11 +6605,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("WAITING state transitions");
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_SYNCING_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6593,7 +6653,7 @@ TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyPrimary) {
 
     startService(valid_config);
 
-    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                    FinalState(HA_READY_ST));
 
     testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
@@ -6642,11 +6702,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("HOT STANDBY state transitions");
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6679,6 +6742,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -6717,6 +6783,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
 
@@ -6748,11 +6817,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("READY state transitions");
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6783,11 +6855,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("WAITING state transitions");
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_SYNCING_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6830,7 +6905,7 @@ TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyStandby) {
 
     startService(valid_config);
 
-    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                    FinalState(HA_READY_ST));
 
     testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),