[#1716] Used a timer instead of a sleep call

src/bin/dhcp4/ctrl_dhcp4_srv.cc

@@ -935,6 +935,11 @@ ControlledDhcpv4Srv::processConfig(isc::data::ConstElementPtr config) {
         return (isc::config::createAnswer(1, err.str()));
     }
 
+    // Configure a callback to shut down the server when the bind socket
+    // attempts exceeded.
+    CfgIface::open_sockets_failed_callback_ =
+        std::bind(&ControlledDhcpv4Srv::openSocketsFailedCallback, srv, ph::_1);
+
     // Configuration may change active interfaces. Therefore, we have to reopen
     // sockets according to new configuration. It is possible that this
     // operation will fail for some interfaces but the openSockets function
@@ -1305,6 +1310,23 @@ ControlledDhcpv4Srv::dbFailedCallback(ReconnectCtlPtr db_reconnect_ctl) {
     return (true);
 }
 
+void
+ControlledDhcpv4Srv::openSocketsFailedCallback(
+        util::ReconnectCtlPtr db_reconnect_ctl) {
+    if (!db_reconnect_ctl) {
+        // This should never happen
+        LOG_ERROR(dhcp4_logger, DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL);
+        return;
+    }
+
+    LOG_INFO(dhcp4_logger, DHCP4_OPEN_SOCKETS_FAILED)
+            .arg(db_reconnect_ctl->maxRetries());
+
+    if (db_reconnect_ctl->exitOnFailure()) {
+        shutdownServer(EXIT_FAILURE);
+    }
+}
+
 void
 ControlledDhcpv4Srv::cbFetchUpdates(const SrvConfigPtr& srv_cfg,
                                     boost::shared_ptr<unsigned> failure_count) {

src/bin/dhcp4/ctrl_dhcp4_srv.h

@@ -11,7 +11,7 @@
 #include <asiolink/asiolink.h>
 #include <cc/data.h>
 #include <cc/command_interpreter.h>
-#include <database/database_connection.h>
+#include <util/reconnect_ctl.h>
 #include <dhcpsrv/timer_mgr.h>
 #include <dhcp4/dhcp4_srv.h>
 
@@ -403,7 +403,7 @@ private:
     /// configured reconnect parameters
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbLostCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbLostCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback DB backends should invoke upon restoration of
     /// connectivity
@@ -413,7 +413,7 @@ private:
     /// recovered.
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbRecoveredCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbRecoveredCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback DB backends should invoke upon failing to restore
     /// connectivity
@@ -422,7 +422,13 @@ private:
     /// connectivity. It stops the server.
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbFailedCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbFailedCallback(util::ReconnectCtlPtr db_reconnect_ctl);
+
+    /// @brief This callback should be invoked upon failing to bind sockets.
+    ///
+    /// This function is invoked during the configuration of the interfaces
+    /// when they fail to bind the service sockets. It may stop the server.
+    void openSocketsFailedCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback invoked periodically to fetch configuration updates
     /// from the Config Backends.

src/bin/dhcp4/dhcp4_messages.cc

@@ -95,6 +95,8 @@ extern const isc::log::MessageID DHCP4_NO_LEASE_INIT_REBOOT = "DHCP4_NO_LEASE_IN
 extern const isc::log::MessageID DHCP4_NO_SOCKETS_OPEN = "DHCP4_NO_SOCKETS_OPEN";
 extern const isc::log::MessageID DHCP4_OPEN_CONFIG_DB = "DHCP4_OPEN_CONFIG_DB";
 extern const isc::log::MessageID DHCP4_OPEN_SOCKET = "DHCP4_OPEN_SOCKET";
+extern const isc::log::MessageID DHCP4_OPEN_SOCKETS_FAILED = "DHCP4_OPEN_SOCKETS_FAILED";
+extern const isc::log::MessageID DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL = "DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL";
 extern const isc::log::MessageID DHCP4_OPEN_SOCKET_FAIL = "DHCP4_OPEN_SOCKET_FAIL";
 extern const isc::log::MessageID DHCP4_PACKET_DROP_0001 = "DHCP4_PACKET_DROP_0001";
 extern const isc::log::MessageID DHCP4_PACKET_DROP_0002 = "DHCP4_PACKET_DROP_0002";
@@ -254,6 +256,8 @@ const char* values[] = {
     "DHCP4_NO_SOCKETS_OPEN", "no interface configured to listen to DHCP traffic",
     "DHCP4_OPEN_CONFIG_DB", "Opening configuration database: %1",
     "DHCP4_OPEN_SOCKET", "opening service sockets on port %1",
+    "DHCP4_OPEN_SOCKETS_FAILED", "maximum number of open service sockets attempts: %1, has been exhausted without success",
+    "DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL", "unexpected error in bind service sockets.",
     "DHCP4_OPEN_SOCKET_FAIL", "failed to open socket: %1",
     "DHCP4_PACKET_DROP_0001", "failed to parse packet from %1 to %2, received over interface %3, reason: %4",
     "DHCP4_PACKET_DROP_0002", "%1, from interface %2: no suitable subnet configured for a direct client",

src/bin/dhcp4/dhcp4_messages.h

@@ -96,6 +96,8 @@ extern const isc::log::MessageID DHCP4_NO_LEASE_INIT_REBOOT;
 extern const isc::log::MessageID DHCP4_NO_SOCKETS_OPEN;
 extern const isc::log::MessageID DHCP4_OPEN_CONFIG_DB;
 extern const isc::log::MessageID DHCP4_OPEN_SOCKET;
+extern const isc::log::MessageID DHCP4_OPEN_SOCKETS_FAILED;
+extern const isc::log::MessageID DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL;
 extern const isc::log::MessageID DHCP4_OPEN_SOCKET_FAIL;
 extern const isc::log::MessageID DHCP4_PACKET_DROP_0001;
 extern const isc::log::MessageID DHCP4_PACKET_DROP_0002;

src/bin/dhcp4/dhcp4_messages.mes

@@ -226,6 +226,17 @@ should be reported.
 This info message indicates that the connection has been recovered and the dhcp
 service has been restored.
 
+% DHCP4_OPEN_SOCKETS_NO_RECONNECT_CTL unexpected error in bind service sockets.
+This is an error message indicating a programmatic error that should not
+occur. It prohibits the server from attempting to bind to its
+service sockets if they are unavailable, and the server exits. This error
+should be reported.
+
+% DHCP4_OPEN_SOCKETS_FAILED maximum number of open service sockets attempts: %1, has been exhausted without success
+This error indicates that the server failed to bind service sockets after making
+the maximum configured number of reconnect attempts. This might cause the server
+to shut down as specified in the configuration.
+
 % DHCP4_DDNS_REQUEST_SEND_FAILED failed sending a request to kea-dhcp-ddns, error: %1,  ncr: %2
 This error message indicates that DHCP4 server attempted to send a DDNS
 update request to the DHCP-DDNS server.  This is most likely a configuration or

src/bin/dhcp6/ctrl_dhcp6_srv.cc

@@ -958,6 +958,11 @@ ControlledDhcpv6Srv::processConfig(isc::data::ConstElementPtr config) {
         return (isc::config::createAnswer(1, err.str()));
     }
 
+    // Configure a callback to shut down the server when the bind socket
+    // attempts exceeded.
+    CfgIface::open_sockets_failed_callback_ =
+        std::bind(&ControlledDhcpv6Srv::openSocketsFailedCallback, srv, ph::_1);
+
     // Configuration may change active interfaces. Therefore, we have to reopen
     // sockets according to new configuration. It is possible that this
     // operation will fail for some interfaces but the openSockets function
@@ -1325,6 +1330,23 @@ ControlledDhcpv6Srv::dbFailedCallback(ReconnectCtlPtr db_reconnect_ctl) {
     return (true);
 }
 
+void
+ControlledDhcpv6Srv::openSocketsFailedCallback(
+        util::ReconnectCtlPtr db_reconnect_ctl) {
+    if (!db_reconnect_ctl) {
+        // This should never happen
+        LOG_ERROR(dhcp6_logger, DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL);
+        return;
+    }
+
+    LOG_INFO(dhcp6_logger, DHCP6_OPEN_SOCKETS_FAILED)
+            .arg(db_reconnect_ctl->maxRetries());
+
+    if (db_reconnect_ctl->exitOnFailure()) {
+        shutdownServer(EXIT_FAILURE);
+    }
+}
+
 void
 ControlledDhcpv6Srv::cbFetchUpdates(const SrvConfigPtr& srv_cfg,
                                     boost::shared_ptr<unsigned> failure_count) {

src/bin/dhcp6/ctrl_dhcp6_srv.h

@@ -11,7 +11,7 @@
 #include <asiolink/asiolink.h>
 #include <cc/data.h>
 #include <cc/command_interpreter.h>
-#include <database/database_connection.h>
+#include <util/reconnect_ctl.h>
 #include <dhcpsrv/timer_mgr.h>
 #include <dhcp6/dhcp6_srv.h>
 
@@ -403,7 +403,7 @@ private:
     /// configured reconnect parameters
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbLostCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbLostCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback DB backends should invoke upon restoration of
     /// connectivity
@@ -413,7 +413,7 @@ private:
     /// recovered.
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbRecoveredCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbRecoveredCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback DB backends should invoke upon failing to restore
     /// connectivity
@@ -422,7 +422,13 @@ private:
     /// connectivity. It stops the server.
     ///
     /// @return false if reconnect is not configured, true otherwise
-    bool dbFailedCallback(db::ReconnectCtlPtr db_reconnect_ctl);
+    bool dbFailedCallback(util::ReconnectCtlPtr db_reconnect_ctl);
+
+    /// @brief This callback should be invoked upon failing to bind sockets.
+    ///
+    /// This function is invoked during the configuration of the interfaces
+    /// when they fail to bind the service sockets. It may stop the server.
+    void openSocketsFailedCallback(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Callback invoked periodically to fetch configuration updates
     /// from the Config Backends.

src/bin/dhcp6/dhcp6_messages.cc

@@ -95,6 +95,8 @@ extern const isc::log::MessageID DHCP6_NOT_RUNNING = "DHCP6_NOT_RUNNING";
 extern const isc::log::MessageID DHCP6_NO_INTERFACES = "DHCP6_NO_INTERFACES";
 extern const isc::log::MessageID DHCP6_NO_SOCKETS_OPEN = "DHCP6_NO_SOCKETS_OPEN";
 extern const isc::log::MessageID DHCP6_OPEN_SOCKET = "DHCP6_OPEN_SOCKET";
+extern const isc::log::MessageID DHCP6_OPEN_SOCKETS_FAILED = "DHCP6_OPEN_SOCKETS_FAILED";
+extern const isc::log::MessageID DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL = "DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL";
 extern const isc::log::MessageID DHCP6_OPEN_SOCKET_FAIL = "DHCP6_OPEN_SOCKET_FAIL";
 extern const isc::log::MessageID DHCP6_PACKET_DROP_DHCP_DISABLED = "DHCP6_PACKET_DROP_DHCP_DISABLED";
 extern const isc::log::MessageID DHCP6_PACKET_DROP_DROP_CLASS = "DHCP6_PACKET_DROP_DROP_CLASS";
@@ -255,6 +257,8 @@ const char* values[] = {
     "DHCP6_NO_INTERFACES", "failed to detect any network interfaces",
     "DHCP6_NO_SOCKETS_OPEN", "no interface configured to listen to DHCP traffic",
     "DHCP6_OPEN_SOCKET", "opening service sockets on port %1",
+    "DHCP6_OPEN_SOCKETS_FAILED", "maximum number of open service sockets attempts: %1, has been exhausted without success",
+    "DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL", "unexpected error in bind service sockets.",
     "DHCP6_OPEN_SOCKET_FAIL", "failed to open socket: %1",
     "DHCP6_PACKET_DROP_DHCP_DISABLED", "%1: DHCP service is globally disabled",
     "DHCP6_PACKET_DROP_DROP_CLASS", "dropped as member of the special class 'DROP': %1",

src/bin/dhcp6/dhcp6_messages.h

@@ -96,6 +96,8 @@ extern const isc::log::MessageID DHCP6_NOT_RUNNING;
 extern const isc::log::MessageID DHCP6_NO_INTERFACES;
 extern const isc::log::MessageID DHCP6_NO_SOCKETS_OPEN;
 extern const isc::log::MessageID DHCP6_OPEN_SOCKET;
+extern const isc::log::MessageID DHCP6_OPEN_SOCKETS_FAILED;
+extern const isc::log::MessageID DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL;
 extern const isc::log::MessageID DHCP6_OPEN_SOCKET_FAIL;
 extern const isc::log::MessageID DHCP6_PACKET_DROP_DHCP_DISABLED;
 extern const isc::log::MessageID DHCP6_PACKET_DROP_DROP_CLASS;

src/bin/dhcp6/dhcp6_messages.mes

@@ -176,6 +176,17 @@ should be reported.
 This info message indicates that the connection has been recovered and the dhcp
 service has been restored.
 
+% DHCP6_OPEN_SOCKETS_NO_RECONNECT_CTL unexpected error in bind service sockets.
+This is an error message indicating a programmatic error that should not
+occur. It prohibits the server from attempting to bind to its
+service sockets if they are unavailable, and the server exits. This error
+should be reported.
+
+% DHCP6_OPEN_SOCKETS_FAILED maximum number of open service sockets attempts: %1, has been exhausted without success
+This error indicates that the server failed to bind service sockets after making
+the maximum configured number of reconnect attempts. This might cause the server
+to shut down as specified in the configuration.
+
 % DHCP6_DDNS_CREATE_ADD_NAME_CHANGE_REQUEST created name change request: %1
 This debug message is logged when the new NameChangeRequest has been created
 to perform the DNS Update, which adds new RRs.

src/lib/database/database_connection.cc

@@ -175,20 +175,20 @@ DatabaseConnection::makeReconnectCtl(const std::string& timer_name) {
         // Wasn't specified so we'll use default of 0;
     }
 
-    OnFailAction action = OnFailAction::STOP_RETRY_EXIT;
+    util::OnFailAction action = util::OnFailAction::STOP_RETRY_EXIT;
     try {
         parm_str = getParameter("on-fail");
-        action = ReconnectCtl::onFailActionFromText(parm_str);
+        action = util::ReconnectCtl::onFailActionFromText(parm_str);
     } catch (...) {
         // Wasn't specified so we'll use default of "stop-retry-exit";
     }
 
-    reconnect_ctl_ = boost::make_shared<ReconnectCtl>(type, timer_name, retries,
+    reconnect_ctl_ = boost::make_shared<util::ReconnectCtl>(type, timer_name, retries,
-                                                      interval, action);
+                                                            interval, action);
 }
 
 bool
-DatabaseConnection::invokeDbLostCallback(const ReconnectCtlPtr& db_reconnect_ctl) {
+DatabaseConnection::invokeDbLostCallback(const util::ReconnectCtlPtr& db_reconnect_ctl) {
     if (DatabaseConnection::db_lost_callback_) {
         return (DatabaseConnection::db_lost_callback_(db_reconnect_ctl));
     }
@@ -197,7 +197,7 @@ DatabaseConnection::invokeDbLostCallback(const ReconnectCtlPtr& db_reconnect_ctl
 }
 
 bool
-DatabaseConnection::invokeDbRecoveredCallback(const ReconnectCtlPtr& db_reconnect_ctl) {
+DatabaseConnection::invokeDbRecoveredCallback(const util::ReconnectCtlPtr& db_reconnect_ctl) {
     if (DatabaseConnection::db_recovered_callback_) {
         return (DatabaseConnection::db_recovered_callback_(db_reconnect_ctl));
     }
@@ -206,7 +206,7 @@ DatabaseConnection::invokeDbRecoveredCallback(const ReconnectCtlPtr& db_reconnec
 }
 
 bool
-DatabaseConnection::invokeDbFailedCallback(const ReconnectCtlPtr& db_reconnect_ctl) {
+DatabaseConnection::invokeDbFailedCallback(const util::ReconnectCtlPtr& db_reconnect_ctl) {
     if (DatabaseConnection::db_failed_callback_) {
         return (DatabaseConnection::db_failed_callback_(db_reconnect_ctl));
     }
@@ -273,32 +273,6 @@ DatabaseConnection::toElementDbAccessString(const std::string& dbaccess) {
     return (toElement(params));
 }
 
-std::string
-ReconnectCtl::onFailActionToText(OnFailAction action) {
-    switch (action) {
-    case OnFailAction::STOP_RETRY_EXIT:
-        return ("stop-retry-exit");
-    case OnFailAction::SERVE_RETRY_EXIT:
-        return ("serve-retry-exit");
-    case OnFailAction::SERVE_RETRY_CONTINUE:
-        return ("serve-retry-continue");
-    }
-    return ("invalid-action-type");
-}
-
-OnFailAction
-ReconnectCtl::onFailActionFromText(const std::string& text) {
-    if (text == "stop-retry-exit") {
-        return (OnFailAction::STOP_RETRY_EXIT);
-    } else if (text == "serve-retry-exit") {
-        return (OnFailAction::SERVE_RETRY_EXIT);
-    } else if (text == "serve-retry-continue") {
-        return (OnFailAction::SERVE_RETRY_CONTINUE);
-    } else {
-        isc_throw(BadValue, "Invalid action on connection loss: " << text);
-    }
-}
-
 DbCallback DatabaseConnection::db_lost_callback_ = 0;
 DbCallback DatabaseConnection::db_recovered_callback_ = 0;
 DbCallback DatabaseConnection::db_failed_callback_ = 0;

src/lib/database/database_connection.h

@@ -12,6 +12,7 @@
 #include <boost/noncopyable.hpp>
 #include <boost/shared_ptr.hpp>
 #include <exceptions/exceptions.h>
+#include <util/reconnect_ctl.h>
 #include <functional>
 #include <map>
 #include <string>
@@ -76,127 +77,8 @@ public:
         isc::Exception(file, line, what) {}
 };
 
-/// @brief Type of action to take on connection loss.
-enum class OnFailAction {
-    STOP_RETRY_EXIT,
-    SERVE_RETRY_EXIT,
-    SERVE_RETRY_CONTINUE
-};
-
-/// @brief Warehouses DB reconnect control values
-///
-/// When a DatabaseConnection loses connectivity to its backend, it
-/// creates an instance of this class based on its configuration parameters and
-/// passes the instance into connection's DB lost callback.  This allows
-/// the layer(s) above the connection to know how to proceed.
-///
-class ReconnectCtl {
-public:
-    /// @brief Constructor.
-    ///
-    /// @param backend_type type of the caller backend.
-    /// @param timer_name timer associated to this object.
-    /// @param max_retries maximum number of reconnect attempts to make.
-    /// @param retry_interval amount of time to between reconnect attempts.
-    /// @param action which should be taken on connection loss.
-    ReconnectCtl(const std::string& backend_type, const std::string& timer_name,
-                 unsigned int max_retries, unsigned int retry_interval,
-                 OnFailAction action) :
-          backend_type_(backend_type), timer_name_(timer_name),
-          max_retries_(max_retries), retries_left_(max_retries),
-          retry_interval_(retry_interval), action_(action) {}
-
-    /// @brief Returns the type of the caller backend.
-    std::string backendType() const {
-        return (backend_type_);
-    }
-
-    /// @brief Returns the associated timer name.
-    ///
-    /// @return the associated timer.
-    std::string timerName() const {
-        return (timer_name_);
-    }
-
-    /// @brief Decrements the number of retries remaining
-    ///
-    /// Each call decrements the number of retries by one until zero is reached.
-    /// @return true the number of retries remaining is greater than zero.
-    bool checkRetries() {
-        return (retries_left_ ? --retries_left_ : false);
-    }
-
-    /// @brief Returns the maximum number of retries allowed.
-    unsigned int maxRetries() {
-        return (max_retries_);
-    }
-
-    /// @brief Returns the number for retries remaining.
-    unsigned int retriesLeft() {
-        return (retries_left_);
-    }
-
-    /// @brief Returns the amount of time to wait between reconnect attempts.
-    unsigned int retryInterval() {
-        return (retry_interval_);
-    }
-
-    /// @brief Resets the retries count.
-    void resetRetries() {
-        retries_left_ = max_retries_;
-    }
-
-    /// @brief Return true if the connection loss should affect the service,
-    /// false otherwise
-    bool alterServiceState() {
-        return (action_ == OnFailAction::STOP_RETRY_EXIT);
-    }
-
-    /// @brief Return true if the connection recovery mechanism should shut down
-    /// the server on failure, false otherwise.
-    bool exitOnFailure() {
-        return ((action_ == OnFailAction::STOP_RETRY_EXIT) ||
-                (action_ == OnFailAction::SERVE_RETRY_EXIT));
-    }
-
-    /// @brief Convert action to string.
-    ///
-    /// @param action The action type to be converted to text.
-    /// @return The text representation of the action type.
-    static std::string onFailActionToText(OnFailAction action);
-
-    /// @brief Convert string to action.
-    ///
-    /// @param text The text to be converted to action type.
-    /// @return The action type corresponding to the text representation.
-    static OnFailAction onFailActionFromText(const std::string& text);
-
-private:
-
-    /// @brief Caller backend type.
-    const std::string backend_type_;
-
-    /// @brief Timer associated to this object.
-    std::string timer_name_;
-
-    /// @brief Maximum number of retry attempts to make.
-    unsigned int max_retries_;
-
-    /// @brief Number of attempts remaining.
-    unsigned int retries_left_;
-
-    /// @brief The amount of time to wait between reconnect attempts.
-    unsigned int retry_interval_;
-
-    /// @brief Action to take on connection loss.
-    OnFailAction action_;
-};
-
-/// @brief Pointer to an instance of ReconnectCtl
-typedef boost::shared_ptr<ReconnectCtl> ReconnectCtlPtr;
-
 /// @brief Defines a callback prototype for propagating events upward
-typedef std::function<bool (ReconnectCtlPtr db_reconnect_ctl)> DbCallback;
+typedef std::function<bool (util::ReconnectCtlPtr db_reconnect_ctl)> DbCallback;
 
 /// @brief Function which returns the IOService that can be used to recover the
 /// connection.
@@ -254,7 +136,7 @@ public:
     /// @brief The reconnect settings.
     ///
     /// @return The reconnect settings.
-    ReconnectCtlPtr reconnectCtl() {
+    util::ReconnectCtlPtr reconnectCtl() {
         return (reconnect_ctl_);
     }
 
@@ -298,19 +180,19 @@ public:
     ///
     /// @return Returns the result of the callback or false if there is no
     /// callback.
-    static bool invokeDbLostCallback(const ReconnectCtlPtr& db_reconnect_ctl);
+    static bool invokeDbLostCallback(const util::ReconnectCtlPtr& db_reconnect_ctl);
 
     /// @brief Invokes the connection's restored connectivity callback
     ///
     /// @return Returns the result of the callback or false if there is no
     /// callback.
-    static bool invokeDbRecoveredCallback(const ReconnectCtlPtr& db_reconnect_ctl);
+    static bool invokeDbRecoveredCallback(const util::ReconnectCtlPtr& db_reconnect_ctl);
 
     /// @brief Invokes the connection's restore failed connectivity callback
     ///
     /// @return Returns the result of the callback or false if there is no
     /// callback.
-    static bool invokeDbFailedCallback(const ReconnectCtlPtr& db_reconnect_ctl);
+    static bool invokeDbFailedCallback(const util::ReconnectCtlPtr& db_reconnect_ctl);
 
     /// @brief Unparse a parameter map
     ///
@@ -380,7 +262,7 @@ private:
     bool unusable_;
 
     /// @brief Reconnect settings.
-    ReconnectCtlPtr reconnect_ctl_;
+    util::ReconnectCtlPtr reconnect_ctl_;
 };
 
 }  // namespace db

src/lib/database/tests/database_connection_unittest.cc

@@ -40,7 +40,7 @@ public:
     ///
     /// @param db_reconnect_ctl ReconnectCtl containing reconnect
     /// parameters
-    bool dbLostCallback(ReconnectCtlPtr db_reconnect_ctl) {
+    bool dbLostCallback(isc::util::ReconnectCtlPtr db_reconnect_ctl) {
         if (!db_reconnect_ctl) {
             isc_throw(isc::BadValue, "db_reconnect_ctl should not be null");
         }
@@ -53,7 +53,7 @@ public:
     ///
     /// @param db_reconnect_ctl ReconnectCtl containing reconnect
     /// parameters
-    bool dbRecoveredCallback(ReconnectCtlPtr db_reconnect_ctl) {
+    bool dbRecoveredCallback(isc::util::ReconnectCtlPtr db_reconnect_ctl) {
         if (!db_reconnect_ctl) {
             isc_throw(isc::BadValue, "db_reconnect_ctl should not be null");
         }
@@ -67,7 +67,7 @@ public:
     ///
     /// @param db_reconnect_ctl ReconnectCtl containing reconnect
     /// parameters
-    bool dbFailedCallback(ReconnectCtlPtr db_reconnect_ctl) {
+    bool dbFailedCallback(isc::util::ReconnectCtlPtr db_reconnect_ctl) {
         if (!db_reconnect_ctl) {
             isc_throw(isc::BadValue, "db_reconnect_ctl should not be null");
         }
@@ -78,7 +78,7 @@ public:
     }
 
     /// @brief Retainer for the control passed into the callback
-    ReconnectCtlPtr db_reconnect_ctl_;
+    isc::util::ReconnectCtlPtr db_reconnect_ctl_;
 };
 
 /// @brief getParameter test

src/lib/dhcp/iface_mgr.cc

@@ -12,7 +12,6 @@
 #include <dhcp/dhcp6.h>
 #include <dhcp/iface_mgr.h>
 #include <dhcp/iface_mgr_error_handler.h>
-#include <dhcp/iface_mgr_retry_callback.h>
 #include <dhcp/pkt_filter_inet.h>
 #include <dhcp/pkt_filter_inet6.h>
 #include <exceptions/exceptions.h>
@@ -518,7 +517,7 @@ void IfaceMgr::stubDetectIfaces() {
 bool
 IfaceMgr::openSockets4(const uint16_t port, const bool use_bcast,
                        IfaceMgrErrorMsgCallback error_handler,
-                       IfaceMgrRetryCallback retry_callback) {
+                       const bool skip_opened) {
     int count = 0;
     int bcast_num = 0;
 
@@ -528,45 +527,46 @@ IfaceMgr::openSockets4(const uint16_t port, const bool use_bcast,
         if (iface->inactive4_) {
             continue;
 
-        } else {
-            // If the interface has been specified in the configuration that
-            // it should be used to listen the DHCP traffic we have to check
-            // that the interface configuration is valid and that the interface
-            // is not a loopback interface. In both cases, we want to report
-            // that the socket will not be opened.
-            // Relax the check when the loopback interface was explicitly
-            // allowed
-            if (iface->flag_loopback_ && !allow_loopback_) {
-                IFACEMGR_ERROR(SocketConfigError, error_handler,
-                               "must not open socket on the loopback"
-                               " interface " << iface->getName());
-                continue;
-
-            }
-
-            if (!iface->flag_up_) {
-                IFACEMGR_ERROR(SocketConfigError, error_handler,
-                               "the interface " << iface->getName()
-                               << " is down");
-                continue;
-            }
-
-            if (!iface->flag_running_) {
-                IFACEMGR_ERROR(SocketConfigError, error_handler,
-                               "the interface " << iface->getName()
-                               << " is not running");
-                continue;
-            }
-
-            IOAddress out_address("0.0.0.0");
-            if (!iface->getAddress4(out_address)) {
-                IFACEMGR_ERROR(SocketConfigError, error_handler,
-                               "the interface " << iface->getName()
-                               << " has no usable IPv4 addresses configured");
-                continue;
-            }
         }
 
+        // If the interface has been specified in the configuration that
+        // it should be used to listen the DHCP traffic we have to check
+        // that the interface configuration is valid and that the interface
+        // is not a loopback interface. In both cases, we want to report
+        // that the socket will not be opened.
+        // Relax the check when the loopback interface was explicitly
+        // allowed
+        if (iface->flag_loopback_ && !allow_loopback_) {
+            IFACEMGR_ERROR(SocketConfigError, error_handler,
+                            "must not open socket on the loopback"
+                            " interface " << iface->getName());
+            continue;
+
+        }
+
+        if (!iface->flag_up_) {
+            IFACEMGR_ERROR(SocketConfigError, error_handler,
+                            "the interface " << iface->getName()
+                            << " is down");
+            continue;
+        }
+
+        if (!iface->flag_running_) {
+            IFACEMGR_ERROR(SocketConfigError, error_handler,
+                            "the interface " << iface->getName()
+                            << " is not running");
+            continue;
+        }
+
+        IOAddress out_address("0.0.0.0");
+        if (!iface->getAddress4(out_address)) {
+            IFACEMGR_ERROR(SocketConfigError, error_handler,
+                            "the interface " << iface->getName()
+                            << " has no usable IPv4 addresses configured");
+            continue;
+        }
+        
+
         for (Iface::Address addr : iface->getAddresses()) {
             // Skip non-IPv4 addresses and those that weren't selected..
             if (addr.unspecified() || !addr.get().isV4()) {
@@ -600,26 +600,26 @@ IfaceMgr::openSockets4(const uint16_t port, const bool use_bcast,
                                    " on remaining interfaces");
                     continue;
             }
-
+        
-            std::stringstream msg_stream("failed to open socket on interface ");
+            // Skip the address that already has a bound socket. It allows
-            msg_stream << iface->getName();
+            // for preventing bind errors or re-opening sockets.
-
+            if (!skip_opened || !IfaceMgr::hasOpenSocket(addr.get())) {
-            try {
+                try {
-                // We haven't open any broadcast sockets yet, so we can
+                    // We haven't open any broadcast sockets yet, so we can
-                // open at least one more or
+                    // open at least one more or
-                // not broadcast capable, do not set broadcast flags.
+                    // not broadcast capable, do not set broadcast flags.
-                callWithRetry<int>(
+                    IfaceMgr::openSocket(
-                    std::bind(&IfaceMgr::openSocket, this,
                         iface->getName(), addr.get(), port,
                         is_open_as_broadcast, is_open_as_broadcast
-                    ),
+                    );
-                    msg_stream.str(), retry_callback
+                } catch (const Exception& ex) {
-                );
+                    IFACEMGR_ERROR(SocketConfigError, error_handler,
-            } catch (const Exception& ex) {
+                        "failed to open socket on interface "
-                msg_stream << ", reason: "
+                            << iface->getName()
-                            << ex.what();
+                            << ", reason: "
-                IFACEMGR_ERROR(SocketConfigError, error_handler, msg_stream.str());
+                            << ex.what());
-                continue;
+                    continue;
+                }
             }
 
             if (is_open_as_broadcast) {
@@ -648,7 +648,7 @@ IfaceMgr::openSockets4(const uint16_t port, const bool use_bcast,
 bool
 IfaceMgr::openSockets6(const uint16_t port,
                        IfaceMgrErrorMsgCallback error_handler,
-                       IfaceMgrRetryCallback retry_callback) {
+                       const bool skip_open) {
     int count = 0;
 
     for (IfacePtr iface : ifaces_) {
@@ -685,21 +685,20 @@ IfaceMgr::openSockets6(const uint16_t port,
 
         // Open unicast sockets if there are any unicast addresses defined
         for (Iface::Address addr : iface->getUnicasts()) {
-            std::stringstream msg_stream("failed to open unicast socket on  interface ");
+            // Skip the address that already has a bound socket. It allows
-            msg_stream << iface->getName();
+            // for preventing bind errors or re-opening sockets.
-
+            if (!skip_open || !IfaceMgr::hasOpenSocket(addr)) {
-            try {
+                try {
-                callWithRetry<int>(
+                    IfaceMgr::openSocket(
-                    std::bind(&IfaceMgr::openSocket, this,
                         iface->getName(), addr, port, false, false
-                    ),
+                    );
-                    msg_stream.str(), retry_callback
+                } catch (const Exception& ex) {
-                );
+                    IFACEMGR_ERROR(SocketConfigError, error_handler, 
-            } catch (const Exception& ex) {
+                        "failed to open unicast socket on  interface "
-                msg_stream << ", reason: "
+                        << iface->getName()
-                    << ex.what();
+                        << ", reason: " << ex.what());
-                IFACEMGR_ERROR(SocketConfigError, error_handler, msg_stream.str());
+                    continue;
-                continue;
+                }
             }
 
             count++;
@@ -724,23 +723,25 @@ IfaceMgr::openSockets6(const uint16_t port,
             // Run OS-specific function to open a socket capable of receiving
             // packets sent to All_DHCP_Relay_Agents_and_Servers multicast
             // address.
-            std::stringstream msg_stream("failed to open multicast socket on  interface ");
-            msg_stream << iface->getName();
 
-            try {
+            // Skip the address that already has a bound socket. It allows
-                callWithRetry<bool>(
+            // for preventing bind errors or re-opening sockets.
-                    std::bind(&IfaceMgr::openMulticastSocket, this,
+            if (!skip_open || !IfaceMgr::hasOpenSocket(addr)) {
+                try {
+                    IfaceMgr::openMulticastSocket(
                         // Pass a null pointer as an error handler to avoid
                         // suppressing an exception in a system-specific function.
-                        std::ref(*iface), addr, port, nullptr),
+                        *iface, addr, port, nullptr
-                    msg_stream.str(), retry_callback
+                    );
-                );
+                } catch (const Exception& ex) {
-                ++count;
+                    IFACEMGR_ERROR(SocketConfigError, error_handler,
-            } catch (const Exception& ex) {
+                        "failed to open multicast socket on  interface "
-                msg_stream << ", reason: "
+                        << iface->getName() << ", reason: " << ex.what());
-                    << ex.what();
+                    continue;
-                IFACEMGR_ERROR(SocketConfigError, error_handler, msg_stream.str());
+                }
             }
+
+            ++count;
         }
     }
 

src/lib/dhcp/iface_mgr.h

@@ -623,15 +623,6 @@ typedef boost::shared_ptr<IfaceMgr> IfaceMgrPtr;
 typedef
 std::function<void(const std::string& errmsg)> IfaceMgrErrorMsgCallback;
 
-/// @brief This type describes the callback function invoked when an opening of
-/// a socket fails and can be retried.
-///
-/// @param retries A number of an opening retries.
-/// @return true if an opening should be retried, false otherwise, and a wait time
-/// from the last attempt.
-typedef
-std::function<std::pair<bool, uint64_t>(uint32_t retries, const std::string& msg)> IfaceMgrRetryCallback;
-
 /// @brief Handles network interfaces, transmission and reception.
 ///
 /// IfaceMgr is an interface manager class that detects available network
@@ -998,17 +989,13 @@ public:
     /// @param error_handler A pointer to an error handler function which is
     /// called by the openSockets6 when it fails to open a socket. This
     /// parameter can be null to indicate that the callback should not be used.
-    /// @param retry_callback A pointer to a retry callback function which is
+    /// @param skip_opened skip the addresses that already have the opened port
-    /// called by the openSockets4 when it fails to open a socket.
-    /// The responsibility of the callback is to decide if the opening should be
-    /// retried and after which time. This parameter can be null to indicate that
-    /// the callback should not be used.
     ///
     /// @throw SocketOpenFailure if tried and failed to open socket.
     /// @return true if any sockets were open
     bool openSockets6(const uint16_t port = DHCP6_SERVER_PORT,
                       IfaceMgrErrorMsgCallback error_handler = 0,
-                      IfaceMgrRetryCallback retry_callback = 0);
+                      const bool skip_opened = false);
 
     /// @brief Opens IPv4 sockets on detected interfaces.
     ///
@@ -1071,14 +1058,10 @@ public:
     ///
     /// @param port specifies port number (usually DHCP4_SERVER_PORT)
     /// @param use_bcast configure sockets to support broadcast messages.
-    /// @param error_handler A pointer to an error handler function which is
+    /// @param error_handler a pointer to an error handler function which is
     /// called by the openSockets4 when it fails to open a socket. This
     /// parameter can be null to indicate that the callback should not be used.
-    /// @param retry_callback A pointer to a retry callback function which is
+    /// @param skip_opened skip the addresses that already have the opened port
-    /// called by the openSockets4 when it fails to open a socket.
-    /// The responsibility of the callback is to decide if the opening should be
-    /// retried and after which time. This parameter can be null to indicate that
-    /// the callback should not be used.
     ///
     /// @throw SocketOpenFailure if tried and failed to open socket and callback
     /// function hasn't been specified.
@@ -1086,7 +1069,7 @@ public:
     bool openSockets4(const uint16_t port = DHCP4_SERVER_PORT,
                       const bool use_bcast = true,
                       IfaceMgrErrorMsgCallback error_handler = 0,
-                      IfaceMgrRetryCallback retry_callback = 0);
+                      const bool skip_opened = false);
 
     /// @brief Closes all open sockets.
     ///

src/lib/dhcp/iface_mgr_retry_callback.h

@@ -1,74 +0,0 @@
-// Copyright (C) 2011-2022 Internet Systems Consortium, Inc. ("ISC")
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#ifndef IFACE_MGR_RETRY_CALLBACK_H
-#define IFACE_MGR_RETRY_CALLBACK_H
-
-#include <functional>
-#include <dhcp/iface_mgr.h>
-
-namespace isc {
-namespace dhcp {
-
-/// @brief An helper to call a function with retry.
-///
-/// There are certain cases when IfaceMgr may hit an error caused by
-/// temporary extarnal factors. A typical case is the function which opens
-/// sockets on available interfaces for a DHCP server. If this function
-/// fails to open a socket on a specific interface (for example, there is
-/// another socket already open on this interface and bound to the same address
-/// and port), it may be helpful to repeat an opening procedure.
-/// It is allowed that the error handler function is not installed (is NULL).
-/// In these cases it is expected that the function is just called without retrying.
-///
-/// @param f A function to call; template type T is an output type of this function.
-/// @param msg A message intended to log with a failed attempt.
-/// @param retry_callback A retry callback that decides to continue retries and wait
-/// time before next try. It should also log the info/warning message.
-template <typename T>
-T callWithRetry(std::function<T()> f,
-                const std::string& msg,
-                IfaceMgrRetryCallback retry_callback) {
-
-    // If the retry callback is NULL, just call the function and return.
-    if (retry_callback == nullptr) {
-        return f();
-    }
-
-    // Counter of the retries.
-    uint64_t retries = 0;
-
-    // Leave the loop on success (return statement)
-    // or stop retrying (throw statement).
-    while (true) {
-        try {
-            return f();
-        } catch (const Exception& ex) {
-            std::stringstream message(msg);
-            message << ", reason: " << ex.what();
-            auto retry_msg = message.str();
-
-            // Callback produces a log message
-            const std::pair<bool, uint64_t>& result = retry_callback(retries++, retry_msg);
-
-            bool should_retry = result.first;
-            uint64_t wait_time = result.second;
-
-            if (!should_retry) {
-                throw;
-            } else {
-                // Wait before next attempt. The initialization cannot end before
-                // opening a socket so we can wait in the foreground.
-                std::this_thread::sleep_for(std::chrono::milliseconds(wait_time));
-            }
-        }
-    }
-}
-
-}
-}
-
-#endif // IFACE_MGR_RETRY_CALLBACK_H

src/lib/dhcp/tests/iface_mgr_unittest.cc

@@ -1989,9 +1989,9 @@ TEST_F(IfaceMgrTest, openSocket4ErrorHandler) {
     EXPECT_EQ(2, errors_count_);
 }
 
-// Test that the external retry callback is called when trying to bind a new
+// Test that no exception is thown when a port is already bound but skip open
-// socket to the address and port being in use. The opening should be repeated.
+// flag is provided.
-TEST_F(IfaceMgrTest, openSocket4RetryCallback) {
+TEST_F(IfaceMgrTest, openSockets4SkipOpen) {
     NakedIfaceMgr ifacemgr;
 
     // Remove all real interfaces and create a set of dummy interfaces.
@@ -2001,46 +2001,18 @@ TEST_F(IfaceMgrTest, openSocket4RetryCallback) {
     ASSERT_TRUE(custom_packet_filter);
     ASSERT_NO_THROW(ifacemgr.setPacketFilter(custom_packet_filter));
 
-    // Open socket on eth0.
+    // Open socket on eth1. The openSockets4 should detect that this
-    ASSERT_NO_THROW(ifacemgr.openSocket("eth0", IOAddress("10.0.0.1"),
+    // socket has been already open and an attempt to open another socket
+    // and bind to this address and port should fail.
+    ASSERT_NO_THROW(ifacemgr.openSocket("eth1", IOAddress("192.0.2.3"),
                                         DHCP4_SERVER_PORT));
 
-    // Install an retry callback before trying to open sockets. This handler
+    // The function doesn't throw an exception when it tries to open a socket
-    // should be called when the IfaceMgr fails to open socket on eth0.
+    // and bind it to the address in use but the skip open flag is provided.
-    // The callback counts the retry attempts. The retry indices must be sequential.
+    EXPECT_NO_THROW(ifacemgr.openSockets4(DHCP4_SERVER_PORT, true, 0, true));
-    // The caller must wait specific time between calls.
-    uint16_t total_attempts = 0;
-    auto last_call_time = std::chrono::system_clock::time_point::min();
-    isc::dhcp::IfaceMgrRetryCallback retry_callback =
-        [&total_attempts, &last_call_time](uint16_t current_attempt, const std::string& msg){
-            // An attempt index must be sequential.
-            EXPECT_EQ(total_attempts, current_attempt);
-            total_attempts++;
 
-            // A waiting time isn't too short.
+    // Check that the other port is bound.
-            auto now = std::chrono::system_clock::now();
+    EXPECT_TRUE(ifacemgr.hasOpenSocket(IOAddress("10.0.0.1")));
-            if (current_attempt != 0) {
-                auto interval = now - last_call_time;
-                EXPECT_GE(interval, std::chrono::milliseconds(10));
-            }
-            last_call_time = now;
-
-            // Message has content.
-            EXPECT_FALSE(msg.empty());
-
-            // Test for 5 retries with 10 milliseconds waiting time.
-            return std::make_pair(current_attempt < 4, 10);
-        };
-
-    // The openSockets4 should detect that there is another socket already
-    // open and bound to the same address and port. An attempt to open
-    // another socket and bind to this address and port should fail and be repeated
-    // a few times. The exception is thrown because the error handler is NULL.
-    EXPECT_THROW(ifacemgr.openSockets4(DHCP4_SERVER_PORT, true, nullptr, retry_callback),
-        isc::dhcp::SocketConfigError);
-
-    // The callback should notice 5 attempts to open a port - 1 initial and 4 retries.
-    EXPECT_EQ(5, total_attempts);
 }
 
 // This test verifies that the function correctly checks that the v4 socket is
@@ -2473,7 +2445,7 @@ TEST_F(IfaceMgrTest, openSockets6NoIfaces) {
 // Test that the external error handler is called when trying to bind a new
 // socket to the address and port being in use. The sockets on the other
 // interfaces should open just fine.
-TEST_F(IfaceMgrTest, openSocket6ErrorHandler) {
+TEST_F(IfaceMgrTest, openSockets6ErrorHandler) {
     NakedIfaceMgr ifacemgr;
 
     // Remove all real interfaces and create a set of dummy interfaces.
@@ -2510,10 +2482,9 @@ TEST_F(IfaceMgrTest, openSocket6ErrorHandler) {
     EXPECT_EQ(2, errors_count_);
 }
 
-// Test that the external retry callback is called when trying to bind a new
+// Test that no exception is thown when a port is already bound but skip open
-// multicast socket to the address and port being in use. The opening should
+// flag is provided.
-// be repeated.
+TEST_F(IfaceMgrTest, openSockets6SkipOpen) {
-TEST_F(IfaceMgrTest, openMulticastSocket6RetryCallback) {
     NakedIfaceMgr ifacemgr;
 
     // Remove all real interfaces and create a set of dummy interfaces.
@@ -2523,105 +2494,19 @@ TEST_F(IfaceMgrTest, openMulticastSocket6RetryCallback) {
     ASSERT_TRUE(filter);
     ASSERT_NO_THROW(ifacemgr.setPacketFilter(filter));
 
-    // Open multicast socket on eth0.
+    // Open socket on eth0. The openSockets6 should detect that this
+    // socket has been already open and an attempt to open another socket
+    // and bind to this address and port should fail.
     ASSERT_NO_THROW(ifacemgr.openSocket("eth0",
                                         IOAddress("fe80::3a60:77ff:fed5:cdef"),
                                         DHCP6_SERVER_PORT, true));
 
-    // Install an retry callback before trying to open sockets. This handler
+    // The function doesn't throw an exception when it tries to open a socket
-    // should be called when the IfaceMgr fails to open socket on eth0.
+    // and bind it to the address in use but the skip open flag is provided.
-    // The callback counts the retry attempts. The retry indices must be sequential.
+    EXPECT_NO_THROW(ifacemgr.openSockets6(DHCP6_SERVER_PORT, 0, true));
-    // The caller must wait specific time between calls.
-    uint16_t total_attempts = 0;
-    auto last_call_time = std::chrono::system_clock::time_point::min();
-    isc::dhcp::IfaceMgrRetryCallback retry_callback =
-        [&total_attempts, &last_call_time](uint16_t current_attempt, const std::string& msg){
-            // An attempt index must be sequential.
-            EXPECT_EQ(total_attempts, current_attempt);
-            total_attempts++;
 
-            // A waiting time isn't too short.
+    // Check that the other port is bound.
-            auto now = std::chrono::system_clock::now();
+    EXPECT_TRUE(ifacemgr.isBound("eth1", "fe80::3a60:77ff:fed5:abcd"));
-            if (current_attempt != 0) {
-                auto interval = now - last_call_time;
-                EXPECT_GE(interval, std::chrono::milliseconds(10));
-            }
-            last_call_time = now;
-
-            // Message has content.
-            EXPECT_FALSE(msg.empty());
-
-            // Test for 5 retries with 10 milliseconds waiting time.
-            return std::make_pair(current_attempt < 4, 10);
-        };
-
-    // The openSockets6 should detect that there is another socket already
-    // open and bound to the same address and port. An attempt to open
-    // another socket and bind to this address and port should fail and be repeated
-    // a few times. The exception is thrown because the error handler is NULL.
-    EXPECT_THROW(ifacemgr.openSockets6(DHCP6_SERVER_PORT, nullptr, retry_callback),
-        isc::dhcp::SocketConfigError);
-
-    // The callback should notice 5 attempts to open a port - 1 initial and 4 retries.
-    EXPECT_EQ(5, total_attempts);
-}
-
-// Test that the external retry callback is called when trying to bind a new
-// unicast socket to the address and port being in use. The opening should be
-// repeated.
-TEST_F(IfaceMgrTest, openUnicastSocket6RetryCallback) {
-    NakedIfaceMgr ifacemgr;
-
-    // Remove all real interfaces and create a set of dummy interfaces.
-    ifacemgr.createIfaces();
-    // Add an unicast.
-    ifacemgr.getIface("eth1")->addUnicast(IOAddress("2001:db8:1::2"));
-
-    boost::shared_ptr<PktFilter6Stub> filter(new PktFilter6Stub());
-    ASSERT_TRUE(filter);
-    ASSERT_NO_THROW(ifacemgr.setPacketFilter(filter));
-
-    // Open unicast socket on eth1.
-    ASSERT_NO_THROW(ifacemgr.openSocket("eth1",
-                                        IOAddress("2001:db8:1::2"),
-                                        DHCP6_SERVER_PORT, false));
-
-    // Install an retry callback before trying to open sockets. This handler
-    // should be called when the IfaceMgr fails to open socket on eth0.
-    // The callback counts the retry attempts. The retry indices must be sequential.
-    // The caller must wait specific time between calls.
-    uint16_t total_attempts = 0;
-    auto last_call_time = std::chrono::system_clock::time_point::min();
-    isc::dhcp::IfaceMgrRetryCallback retry_callback =
-        [&total_attempts, &last_call_time](uint16_t current_attempt, const std::string& msg){
-            // An attempt index must be sequential.
-            EXPECT_EQ(total_attempts, current_attempt);
-            total_attempts++;
-
-            // A waiting time isn't too short.
-            auto now = std::chrono::system_clock::now();
-            if (current_attempt != 0) {
-                auto interval = now - last_call_time;
-                EXPECT_GE(interval, std::chrono::milliseconds(10));
-            }
-            last_call_time = now;
-
-            // Message has content.
-            EXPECT_FALSE(msg.empty());
-
-            // Test for 5 retries with 10 milliseconds waiting time.
-            return std::make_pair(current_attempt < 4, 10);
-        };
-
-    // The openSockets6 should detect that there is another socket already
-    // open and bound to the same address and port. An attempt to open
-    // another socket and bind to this address and port should fail and be repeated
-    // a few times. The exception is thrown because the error handler is NULL.
-    EXPECT_THROW(ifacemgr.openSockets6(DHCP6_SERVER_PORT, nullptr, retry_callback),
-        isc::dhcp::SocketConfigError);
-
-    // The callback should notice 5 attempts to open a port - 1 initial and 4 retries.
-    EXPECT_EQ(5, total_attempts);
 }
 
 // This test verifies that the function correctly checks that the v6 socket is

src/lib/dhcpsrv/cfg_iface.cc

@@ -8,6 +8,9 @@
 #include <dhcp/iface_mgr.h>
 #include <dhcpsrv/dhcpsrv_log.h>
 #include <dhcpsrv/cfg_iface.h>
+#include <dhcpsrv/timer_mgr.h>
+#include <util/reconnect_ctl.h>
+#include <util/multi_threading_mgr.h>
 #include <util/strutil.h>
 #include <algorithm>
 #include <functional>
@@ -42,7 +45,7 @@ CfgIface::equals(const CfgIface& other) const {
 }
 
 bool
-CfgIface::multipleAddressesPerInterfaceActive() const {
+CfgIface::multipleAddressesPerInterfaceActive() {
     for (IfacePtr iface : IfaceMgr::instance().getIfaces()) {
         if (iface->countActive4() > 1) {
             return (true);
@@ -150,40 +153,24 @@ CfgIface::openSockets(const uint16_t family, const uint16_t port,
         }
     }
 
-    // Set the callbacks which are called when the socket fails to open
+    // Use broadcast only if we're using raw sockets. For the UDP sockets,
-    // for some specific interface.
+    // we only handle the relayed (unicast) traffic.
+    const bool can_use_bcast = use_bcast && (socket_type_ == SOCKET_RAW);
 
-    // If the config requires the binding of all sockets, then the error
+    // Opening multiple raw sockets handling brodcast traffic on the single
-    // callback is null - an exception is thrown on failure.
+    // interface may lead to processing the same message multiple times.
-    IfaceMgrErrorMsgCallback error_callback = nullptr;
+    // We don't prohibit such configuration because raw sockets can as well
-    if (!CfgIface::getServiceSocketsRequireAll()) {
+    // handle the relayed traffic. We have to issue a warning, however, to
-        // This callback will simply log a warning message.
+    // draw administrator's attention.
-        error_callback = std::bind(&CfgIface::socketOpenErrorHandler, ph::_1);
+    if (family == AF_INET && can_use_bcast && multipleAddressesPerInterfaceActive()) {
-    }
-
-    IfaceMgrRetryCallback retry_callback =
-        std::bind(&CfgIface::socketOpenRetryHandler, this, ph::_1, ph::_2);
-
-    bool sopen;
-    if (family == AF_INET) {
-        // Use broadcast only if we're using raw sockets. For the UDP sockets,
-        // we only handle the relayed (unicast) traffic.
-        const bool can_use_bcast = use_bcast && (socket_type_ == SOCKET_RAW);
-        // Opening multiple raw sockets handling brodcast traffic on the single
-        // interface may lead to processing the same message multiple times.
-        // We don't prohibit such configuration because raw sockets can as well
-        // handle the relayed traffic. We have to issue a warning, however, to
-        // draw administrator's attention.
         if (can_use_bcast && multipleAddressesPerInterfaceActive()) {
             LOG_WARN(dhcpsrv_logger, DHCPSRV_MULTIPLE_RAW_SOCKETS_PER_IFACE);
         }
-        sopen = IfaceMgr::instance().openSockets4(port, can_use_bcast, error_callback,
-            retry_callback);
-    } else {
-        // use_bcast is ignored for V6.
-        sopen = IfaceMgr::instance().openSockets6(port, error_callback, retry_callback);
     }
 
+    auto reconnect_ctl = makeReconnectCtl(family);
+    auto sopen = openSocketsWithRetry(reconnect_ctl, family, port, can_use_bcast);
+
     if (!sopen) {
         // If no socket were opened, log a warning because the server will
         // not respond to any queries.
@@ -191,6 +178,103 @@ CfgIface::openSockets(const uint16_t family, const uint16_t port,
     }
 }
 
+std::pair<bool, bool>
+CfgIface::openSocketsForFamily(const uint16_t family, const uint16_t port,
+        const bool can_use_bcast, const bool skip_opened) {
+    bool no_errors = true;
+
+    // Set the callbacks which are called when the socket fails to open
+    // for some specific interface.
+    auto error_callback = [&no_errors](const std::string& errmsg){
+        socketOpenErrorHandler(errmsg);
+        no_errors = false;
+    };
+
+    bool sopen = false;
+    if (family == AF_INET) {
+        sopen = IfaceMgr::instance().openSockets4(
+            port, can_use_bcast, error_callback, skip_opened
+        );
+    } else {
+        // use_bcast is ignored for V6.
+        sopen = IfaceMgr::instance().openSockets6(
+            port, error_callback, skip_opened
+        );
+    }
+
+    return std::make_pair(sopen, no_errors);
+}
+
+util::ReconnectCtlPtr CfgIface::makeReconnectCtl(const uint16_t family) const {
+    // Create unique timer name per instance.
+    std::string timer_name = "ConfigInterface[";
+    timer_name += boost::lexical_cast<std::string>(reinterpret_cast<uint64_t>(this));
+    timer_name += "]SocketReopenFamily";
+    timer_name += boost::lexical_cast<std::string>(family);
+    timer_name += "Timer";
+
+    auto on_fail_action = util::OnFailAction::SERVE_RETRY_CONTINUE;
+    if (CfgIface::getServiceSocketsRequireAll()) {
+        on_fail_action = util::OnFailAction::STOP_RETRY_EXIT;
+    }
+
+    auto reconnect_ctl = boost::make_shared<util::ReconnectCtl>("open-sockets", timer_name,
+        // Add one attempt for an initial call.
+        CfgIface::getServiceSocketsMaxRetries(),
+        CfgIface::getServiceSocketsRetryWaitTime(),
+        on_fail_action);
+
+    return reconnect_ctl;
+}
+
+bool 
+CfgIface::openSocketsWithRetry(util::ReconnectCtlPtr reconnect_ctl,
+        const uint16_t family, const uint16_t port, const bool can_use_bcast) {
+    util::MultiThreadingCriticalSection cs;
+
+    // Skip opened sockets in the retry calls.
+    bool skip_opened = reconnect_ctl->retriesLeft() != reconnect_ctl->maxRetries();
+    auto result_pair = openSocketsForFamily(family, port, can_use_bcast, skip_opened);
+    bool sopen = result_pair.first;
+    bool has_errors = !result_pair.second;
+
+    auto timer_name = reconnect_ctl->timerName();
+
+    // Has errors and can retry
+    if (has_errors && reconnect_ctl->retriesLeft() > 0) {
+        // Initial call is excluded from retries counter.
+        reconnect_ctl->checkRetries();
+        // Start the timer.
+        if (!TimerMgr::instance()->isTimerRegistered(timer_name)) {
+            TimerMgr::instance()->registerTimer(timer_name,
+                std::bind(&CfgIface::openSocketsWithRetry, reconnect_ctl,
+                    family, port, can_use_bcast
+                ),
+                reconnect_ctl->retryInterval(),
+                asiolink::IntervalTimer::ONE_SHOT);
+        }
+        TimerMgr::instance()->setup(timer_name);
+    // Has errors but retries exceed    
+    } else if (has_errors) {
+        // Cancel the timer.
+        if (TimerMgr::instance()->isTimerRegistered(timer_name)) {
+            TimerMgr::instance()->unregisterTimer(timer_name);
+        }
+
+        if (open_sockets_failed_callback_ != nullptr) {
+            open_sockets_failed_callback_(reconnect_ctl);
+        }
+    // Has no errors
+    } else {
+        // Cancel the timer.
+        if (TimerMgr::instance()->isTimerRegistered(timer_name)) {
+            TimerMgr::instance()->unregisterTimer(timer_name);
+        }
+    }
+
+    return sopen;
+}
+
 void
 CfgIface::reset() {
     wildcard_used_ = false;
@@ -231,21 +315,6 @@ CfgIface::socketOpenErrorHandler(const std::string& errmsg) {
     LOG_WARN(dhcpsrv_logger, DHCPSRV_OPEN_SOCKET_FAIL).arg(errmsg);
 }
 
-std::pair<bool, uint64_t>
-CfgIface::socketOpenRetryHandler(uint32_t retries, const std::string& msg) const {
-    bool can_retry = retries < service_sockets_max_retries_;
-    if (can_retry) {
-        std::stringstream msg_stream;
-        msg_stream << msg << "; retries left: " << service_sockets_max_retries_ - retries;
-        LOG_INFO(dhcpsrv_logger, DHCPSRV_OPEN_SOCKET_FAIL).arg(msg_stream.str());
-    }
-
-    return std::make_pair(
-        can_retry,
-        service_sockets_retry_wait_time_
-    );
-}
-
 std::string
 CfgIface::socketTypeToText() const {
     switch (socket_type_) {
@@ -536,5 +605,7 @@ CfgIface::toElement() const {
     return (result);
 }
 
+CfgIface::OpenSocketsFailedCallback CfgIface::open_sockets_failed_callback_ = 0;
+
 } // end of isc::dhcp namespace
 } // end of isc namespace

src/lib/dhcpsrv/cfg_iface.h

@@ -9,6 +9,7 @@
 
 #include <asiolink/io_address.h>
 #include <dhcp/iface_mgr.h>
+#include <util/reconnect_ctl.h>
 #include <cc/cfg_to_element.h>
 #include <cc/user_context.h>
 #include <boost/shared_ptr.hpp>
@@ -319,31 +320,39 @@ public:
     /// @brief Set the socket service binding retry interval between attempts.
     ///
     /// @param interval Milliseconds between attempts.
-    void setServiceSocketsRetryWaitTime(uint64_t interval) {
+    void setServiceSocketsRetryWaitTime(unsigned int interval) {
         service_sockets_retry_wait_time_ = interval;
     }
 
     /// @brief Indicates the socket service binding retry interval between attempts.
     ///
     /// @return Milliseconds between attempts.
-    uint64_t getServiceSocketsRetryWaitTime() {
+    unsigned int getServiceSocketsRetryWaitTime() const {
         return (service_sockets_retry_wait_time_);
     }
 
     /// @brief Set a maximum number of service sockets bind attempts.
     ///
     /// @param max_retries Number of attempts. The value 0 disables retries.
-    void setServiceSocketsMaxRetries(uint32_t max_retries) {
+    void setServiceSocketsMaxRetries(unsigned int max_retries) {
         service_sockets_max_retries_ = max_retries;
     }
 
     /// @brief Indicates the maximum number of service sockets bind attempts.
     ///
     /// @return Number of attempts.
-    uint32_t getServiceSocketsMaxRetries() {
+    unsigned int getServiceSocketsMaxRetries() const {
         return (service_sockets_max_retries_);
     }
 
+    /// @brief Represents a callback invoked if all retries of the
+    /// opening sockets fail.
+    typedef std::function<void(util::ReconnectCtlPtr)> OpenSocketsFailedCallback;
+
+    /// @brief Optional callback function to invoke if all retries of the
+    /// opening sockets fail.
+    static OpenSocketsFailedCallback open_sockets_failed_callback_;
+
 private:
 
     /// @brief Checks if multiple IPv4 addresses has been activated on any
@@ -358,7 +367,7 @@ private:
     ///
     /// @return true if multiple addresses are activated on any interface,
     /// false otherwise.
-    bool multipleAddressesPerInterfaceActive() const;
+    static bool multipleAddressesPerInterfaceActive();
 
     /// @brief Selects or deselects interfaces.
     ///
@@ -397,19 +406,62 @@ private:
     /// @param errmsg Error message being logged by the function.
     static void socketOpenErrorHandler(const std::string& errmsg);
 
+    /// @brief Calls a family-specific function to open sockets.
+    ///
+    /// It is a static function for a safe call from a CfgIface instance or a
+    /// timer handler.
+    ///
+    /// @param family Address family (AF_INET or AF_INET6).
+    /// @param port Port number to be used to bind sockets to.
+    /// @param can_use_bcast A boolean flag which indicates if the broadcast
+    /// traffic should be received through the socket and the raw sockets are
+    /// used. For the UDP sockets, we only handle the relayed (unicast)
+    /// traffic. This parameter is ignored for IPv6.
+    /// @param skip_opened Omits the already opened sockets (doesn't try to
+    /// re-bind).
+    /// @return Pair of boolean flags. The first boolean is true if at least
+    /// one socket is successfully opened, and the second is true if no errors
+    /// occur.
+    static std::pair<bool, bool> openSocketsForFamily(const uint16_t family,
+        const uint16_t port, const bool can_use_bcast, const bool skip_opened);
+
+    /// @brief Creates a ReconnectCtl based on the configuration's
+    /// retry parameters.
+    ///
+    /// @param family IP family
+    /// @return The reconnect control created using the configuration
+    /// parameters.
+    util::ReconnectCtlPtr makeReconnectCtl(const uint16_t family) const;
+
+    /// Calls the @c CfgIface::openSocketsForFamily function and retry it if
+    /// socket opening fails.
+    ///
+    /// @param family Address family (AF_INET or AF_INET6).
+    /// @param port Port number to be used to bind sockets to.
+    /// @param can_use_bcast A boolean flag which indicates if the broadcast
+    /// traffic should be received through the socket and the raw sockets are
+    /// used. For the UDP sockets, we only handle the relayed (unicast)
+    /// traffic. This parameter is ignored for IPv6.
+    /// @return True if at least one socket opened successfully.
+    static bool openSocketsWithRetry(
+        util::ReconnectCtlPtr reconnect_ctl,
+        const uint16_t family, const uint16_t port, const bool can_use_bcast);
+
     /// @brief Retry handler for executed when opening a socket fail.
     ///
     /// A pointer to this function is passed to the @c IfaceMgr::openSockets4
     /// or @c IfaceMgr::openSockets6. These functions call this handler when
-    /// they fail to open a socket. The handler decides if the opening should be
+    /// they fail to open a socket. The handler decides if the opening should
-    /// retried and logs info passed in the parameter. It also returns a time to
+    /// be retried and logs info passed in the parameter. It also returns a
-    /// wait from the last attempt. It allows extending the waiting time dynamically
+    /// time to wait from the last attempt. It allows extending the waiting
-    /// with the next tries.
+    /// time dynamically with the next tries.
     ///
     /// @param retries An index of opening retries
     /// @param msg Message being logged by the function.
-    /// @return true if the opening should be retried and milliseconds to wait from last attempt.
+    /// @return true if the opening should be retried and milliseconds to wait
-    std::pair<bool, uint64_t> socketOpenRetryHandler(uint32_t retries, const std::string& msg) const;
+    /// from last attempt.
+    std::pair<bool, uint64_t> socketOpenRetryHandler(uint32_t retries,
+        const std::string& msg) const;
 
     /// @brief Represents a set of interface names.
     typedef std::set<std::string> IfaceSet;

src/lib/dhcpsrv/mysql_lease_mgr.h

@@ -148,7 +148,7 @@ public:
     /// @param db_reconnect_ctl pointer to the ReconnectCtl containing the
     /// configured reconnect parameters.
     /// @return true if connection has been recovered, false otherwise.
-    static bool dbReconnect(db::ReconnectCtlPtr db_reconnect_ctl);
+    static bool dbReconnect(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Local version of getDBVersion() class method
     static std::string getDBVersion();

src/lib/dhcpsrv/pgsql_lease_mgr.h

@@ -147,7 +147,7 @@ public:
     /// @param db_reconnect_ctl pointer to the ReconnectCtl containing the
     /// configured reconnect parameters.
     /// @return true if connection has been recovered, false otherwise.
-    static bool dbReconnect(db::ReconnectCtlPtr db_reconnect_ctl);
+    static bool dbReconnect(util::ReconnectCtlPtr db_reconnect_ctl);
 
     /// @brief Local version of getDBVersion() class method
     static std::string getDBVersion();

src/lib/dhcpsrv/tests/cfg_iface_unittest.cc

@@ -10,6 +10,10 @@
 #include <dhcp/tests/pkt_filter_test_stub.h>
 #include <dhcp/tests/pkt_filter6_test_stub.h>
 #include <dhcpsrv/cfg_iface.h>
+#include <asiolink/io_service.h>
+#include <asiolink/asio_wrapper.h>
+#include <asiolink/interval_timer.h>
+#include <dhcpsrv/timer_mgr.h>
 #include <testutils/test_to_element.h>
 #include <gtest/gtest.h>
 
@@ -52,11 +56,40 @@ public:
     /// @param iface_name Interface name.
     bool unicastOpen(const std::string& iface_name) const;
 
+    /// @brief Wait for specific timeout.
+    ///
+    /// @param timeout Wait timeout in milliseconds.
+    void doWait(const long timeout);
+
     /// @brief Holds a fake configuration of the interfaces.
     IfaceMgrTestConfig iface_mgr_test_config_;
 
+    /// @brief Pointer to IO service used by the tests.
+    asiolink::IOServicePtr io_service_;
+
+private:
+
+    /// @brief Prepares the class for a test.
+    virtual void SetUp();
+
+    /// @brief Cleans up after the test.
+    virtual void TearDown();
+
 };
 
+void
+CfgIfaceTest::SetUp() {
+    io_service_.reset(new asiolink::IOService());
+    auto timer_mgr_ = TimerMgr::instance();
+    timer_mgr_->setIOService(io_service_);
+}
+
+void
+CfgIfaceTest::TearDown() {
+    // Remove all timers.
+    TimerMgr::instance()->unregisterTimers();
+}
+
 bool
 CfgIfaceTest::socketOpen(const std::string& iface_name,
                          const int family) const {
@@ -74,6 +107,16 @@ CfgIfaceTest::unicastOpen(const std::string& iface_name) const {
     return (iface_mgr_test_config_.unicastOpen(iface_name));
 }
 
+void
+CfgIfaceTest::doWait(const long timeout) {
+    asiolink::IntervalTimer timer(*io_service_);
+    timer.setup([this]() {
+        io_service_->stop();
+    }, timeout, asiolink::IntervalTimer::ONE_SHOT);
+    io_service_->run();
+    io_service_->get_io_service().reset();
+}
+
 // This test checks that the interface names can be explicitly selected
 // by their names and IPv4 sockets are opened on these interfaces.
 TEST_F(CfgIfaceTest, explicitNamesV4) {
@@ -82,7 +125,7 @@ TEST_F(CfgIfaceTest, explicitNamesV4) {
     ASSERT_NO_THROW(cfg.use(AF_INET, "eth0"));
     ASSERT_NO_THROW(cfg.use(AF_INET, "eth1"));
 
-    // Open sockets on specified interfaces.
+    // Open sockets on spsetecified interfaces.
     cfg.openSockets(AF_INET, DHCP4_SERVER_PORT);
 
     // Sockets should be now open on eth0 and eth1, but not on loopback.
@@ -514,6 +557,18 @@ TEST_F(CfgIfaceTest, unparse) {
 TEST_F(CfgIfaceTest, requireOpenAllServiceSockets) {
     CfgIface cfg4;
     CfgIface cfg6;
+
+    // Configure a fail callback
+    uint16_t fail_calls = 0;
+    CfgIface::OpenSocketsFailedCallback on_fail_callback =
+        [&fail_calls](util::ReconnectCtlPtr reconnect_ctl){
+            EXPECT_TRUE(reconnect_ctl != nullptr);
+            EXPECT_TRUE(reconnect_ctl->exitOnFailure());
+            fail_calls++;
+        };
+
+    CfgIface::open_sockets_failed_callback_ = on_fail_callback;
+
     ASSERT_NO_THROW(cfg4.use(AF_INET, "eth0"));
     ASSERT_NO_THROW(cfg4.use(AF_INET, "eth1/192.0.2.3"));
     ASSERT_NO_THROW(cfg6.use(AF_INET6, "eth0/2001:db8:1::1"));
@@ -521,9 +576,11 @@ TEST_F(CfgIfaceTest, requireOpenAllServiceSockets) {
 
     // Require all sockets bind successfully
     cfg4.setServiceSocketsRequireAll(true);
+    cfg4.setServiceSocketsMaxRetries(0);
     cfg6.setServiceSocketsRequireAll(true);
+    cfg6.setServiceSocketsMaxRetries(0);
 
-    // Open an available port
+    // Open the available ports
     ASSERT_NO_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT));
     ASSERT_NO_THROW(cfg6.openSockets(AF_INET6, DHCP6_SERVER_PORT));
     cfg4.closeSockets();
@@ -543,66 +600,213 @@ TEST_F(CfgIfaceTest, requireOpenAllServiceSockets) {
     ASSERT_NO_THROW(IfaceMgr::instance().setPacketFilter(filter6));
 
     // Open an unavailable port
-    EXPECT_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT), isc::dhcp::SocketConfigError);
+    EXPECT_NO_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT));
-    EXPECT_THROW(cfg6.openSockets(AF_INET6, DHCP6_SERVER_PORT), isc::dhcp::SocketConfigError);
+    EXPECT_NO_THROW(cfg6.openSockets(AF_INET6, DHCP6_SERVER_PORT));
+
+    // Both instances should call the fail callback.
+    EXPECT_EQ(fail_calls, 2);
+
+    // Reset global handlers
+    CfgIface::open_sockets_failed_callback_ = nullptr;
 }
 
-// This test verifies that if any socket fails to bind, then the opening will retry.
+// This test verifies that if any IPv4 socket fails to bind,
-TEST_F(CfgIfaceTest, retryOpenServiceSockets) {
+// the opening will retry.
+TEST_F(CfgIfaceTest, retryOpenServiceSockets4) {
     CfgIface cfg4;
-    CfgIface cfg6;
 
     ASSERT_NO_THROW(cfg4.use(AF_INET, "eth0"));
     ASSERT_NO_THROW(cfg4.use(AF_INET, "eth1/192.0.2.3"));
-    ASSERT_NO_THROW(cfg6.use(AF_INET6, "eth0/2001:db8:1::1"));
-    ASSERT_NO_THROW(cfg6.use(AF_INET6, "eth1"));
 
+    // Parameters
+    const uint16_t RETRIES = 5;
+    const uint16_t WAIT_TIME = 10; // miliseconds
+    // The number of sockets opened in a single retry attempt.
+    const uint16_t CALLS_PER_RETRY = 2;
+
+    // Require retry socket binding
+    cfg4.setServiceSocketsMaxRetries(RETRIES);
+    cfg4.setServiceSocketsRetryWaitTime(WAIT_TIME);
+
+    // Set the callback to count calls and check wait time
+    size_t total_calls = 0;
+    auto last_call_time = std::chrono::system_clock::time_point::min();
+    auto open_callback = [&total_calls, &last_call_time, RETRIES, WAIT_TIME,
+                          CALLS_PER_RETRY](){
+        auto now = std::chrono::system_clock::now();
+
+        // Check waiting time only for the first call in a retry attempt.
+        if (total_calls % CALLS_PER_RETRY == 0) {
+            // Don't check the waiting time for initial call.
+            if (total_calls != 0) {
+                auto interval = now - last_call_time;
+                auto interval_ms =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(
+                        interval
+                    ).count();
+                
+                EXPECT_GE(interval_ms, WAIT_TIME);
+            }
+
+            last_call_time = now;
+        }
+
+        total_calls++;
+
+        // Fail to open a socket
+        isc_throw(Unexpected, "CfgIfaceTest: cannot open a port");
+    };
+
+    boost::shared_ptr<isc::dhcp::test::PktFilterTestStub> filter(
+        new isc::dhcp::test::PktFilterTestStub()
+    );
+
+    filter->setOpenSocketCallback(open_callback);
+
+    ASSERT_TRUE(filter);
+    ASSERT_NO_THROW(IfaceMgr::instance().setPacketFilter(filter));
+
+    // Open an unavailable port
+    ASSERT_NO_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT));
+
+    // Wait for a finish sockets binding (with a safe margin). 
+    doWait(RETRIES * WAIT_TIME * 2);
+
+    // For each interface perform 1 init open and a few retries.
+    EXPECT_EQ(CALLS_PER_RETRY * (RETRIES + 1), total_calls);
+}
+
+// This test verifies that if any IPv4 socket fails to bind, the opening will
+// retry, but the opened sockets will not be re-bound.
+TEST_F(CfgIfaceTest, retryOpenServiceSockets4OmitBound) {
+    CfgIface cfg4;
+
+    ASSERT_NO_THROW(cfg4.use(AF_INET, "eth0"));
+    ASSERT_NO_THROW(cfg4.use(AF_INET, "eth1/192.0.2.3"));
+
+    // Parameters
     const uint16_t RETRIES = 5;
     const uint16_t WAIT_TIME = 10; // miliseconds
 
     // Require retry socket binding
     cfg4.setServiceSocketsMaxRetries(RETRIES);
     cfg4.setServiceSocketsRetryWaitTime(WAIT_TIME);
-    cfg6.setServiceSocketsMaxRetries(RETRIES);
-    cfg6.setServiceSocketsRetryWaitTime(WAIT_TIME);
 
     // Set the callback to count calls and check wait time
     size_t total_calls = 0;
     auto last_call_time = std::chrono::system_clock::time_point::min();
     auto open_callback = [&total_calls, &last_call_time, RETRIES, WAIT_TIME](){
         auto now = std::chrono::system_clock::now();
+        bool is_eth1 = total_calls == 1;
 
-        // Don't check the waiting time for initial calls as they
+        // Skip the wait time check for the socket when two sockets are
-        // can be done immediately after the last call for the previous socket.
+        // binding in a single attempt.
-        if (total_calls % (RETRIES + 1) != 0) {
+        if (!is_eth1) {
-            auto interval = now - last_call_time;
+            // Don't check the waiting time for initial call.
-            EXPECT_GE(interval, std::chrono::milliseconds(WAIT_TIME));
+            if (total_calls != 0) {
+                auto interval = now - last_call_time;
+                auto interval_ms =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(
+                        interval
+                    ).count();
+                
+                EXPECT_GE(interval_ms, WAIT_TIME);
+            }
+
+            last_call_time = now;
+        }
+
+        total_calls++;
+
+        // Fail to open a socket on eth0, success for eth1
+        if (!is_eth1) {
+            isc_throw(Unexpected, "CfgIfaceTest: cannot open a port");
+        }
+    };
+
+    boost::shared_ptr<isc::dhcp::test::PktFilterTestStub> filter(
+        new isc::dhcp::test::PktFilterTestStub()
+    );
+
+    filter->setOpenSocketCallback(open_callback);
+
+    ASSERT_TRUE(filter);
+    ASSERT_NO_THROW(IfaceMgr::instance().setPacketFilter(filter));
+
+    // Open an unavailable port
+    ASSERT_NO_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT));
+
+    // Wait for a finish sockets binding (with a safe margin). 
+    doWait(RETRIES * WAIT_TIME * 2);
+
+    // For eth0 interface perform 1 init open and a few retries,
+    // for eth1 interface perform only init open.
+    EXPECT_EQ((RETRIES + 1) + 1, total_calls);
+}
+
+// This test verifies that if any IPv6 socket fails to bind,
+// the opening will retry.
+TEST_F(CfgIfaceTest, retryOpenServiceSockets6) {
+    CfgIface cfg6;
+
+    ASSERT_NO_THROW(cfg6.use(AF_INET6, "eth0/2001:db8:1::1"));
+    ASSERT_NO_THROW(cfg6.use(AF_INET6, "eth1"));
+
+    // Parameters
+    const uint16_t RETRIES = 5;
+    const uint16_t WAIT_TIME = 10; // miliseconds
+    // The number of sockets opened in a single retry attempt.
+    // 2 multicast sockets and 1 unicast.
+    const uint16_t CALLS_PER_RETRY = 3;
+
+    // Require retry socket binding
+    cfg6.setServiceSocketsMaxRetries(RETRIES);
+    cfg6.setServiceSocketsRetryWaitTime(WAIT_TIME);
+
+    // Set the callback to count calls and check wait time
+    size_t total_calls = 0;
+    auto last_call_time = std::chrono::system_clock::time_point::min();
+    auto open_callback = [&total_calls, &last_call_time, RETRIES, WAIT_TIME,
+                          CALLS_PER_RETRY](){
+        auto now = std::chrono::system_clock::now();
+
+        // Check waiting time only for the first call in a retry attempt.
+        if (total_calls % CALLS_PER_RETRY == 0) {
+            // Don't check the waiting time for initial call.
+            if (total_calls != 0) {
+                auto interval = now - last_call_time;
+                auto interval_ms =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(
+                        interval
+                    ).count();
+                
+                EXPECT_GE(interval_ms, WAIT_TIME);
+            }
+
+            last_call_time = now;
         }
 
-        last_call_time = now;
         total_calls++;
 
         // Fail to open a socket
         isc_throw(Unexpected, "CfgIfaceTest: cannot open a port");
     };
-    boost::shared_ptr<isc::dhcp::test::PktFilterTestStub> filter(new isc::dhcp::test::PktFilterTestStub());
+
-    boost::shared_ptr<isc::dhcp::test::PktFilter6TestStub> filter6(new isc::dhcp::test::PktFilter6TestStub());
+    boost::shared_ptr<isc::dhcp::test::PktFilter6TestStub> filter(
+        new isc::dhcp::test::PktFilter6TestStub()
+    );
     filter->setOpenSocketCallback(open_callback);
-    filter6->setOpenSocketCallback(open_callback);
     ASSERT_TRUE(filter);
-    ASSERT_TRUE(filter6);
     ASSERT_NO_THROW(IfaceMgr::instance().setPacketFilter(filter));
-    ASSERT_NO_THROW(IfaceMgr::instance().setPacketFilter(filter6));
 
     // Open an unavailable port
-    ASSERT_NO_THROW(cfg4.openSockets(AF_INET, DHCP4_SERVER_PORT));
     ASSERT_NO_THROW(cfg6.openSockets(AF_INET6, DHCP6_SERVER_PORT));
 
-    // For IPv4 bind to: eth0 and eth1 (2).
+    // Wait for a finish sockets binding (with a safe margin). 
-    // For IPv6 bind to: unicast for eth0 and multicast for eth0 and eth1 (3).
+    doWait(RETRIES * WAIT_TIME * 2);
-    // For each interface perform 1 init open and 5 retries (6).
+
-    // Perform 30 open calls ((2+3) * 6).
+    // For each interface perform 1 init open and a few retries.
-    EXPECT_EQ(30, total_calls);
+    EXPECT_EQ(CALLS_PER_RETRY * (RETRIES + 1), total_calls);
 }
 
 // This test verifies that it is possible to specify the socket

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h

@@ -626,7 +626,7 @@ public:
     void testDbLostAndFailedAfterTimeoutCallback();
 
     /// @brief Callback function registered with the lease manager
-    bool db_lost_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_lost_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_lost_callback_called_);
     }
 
@@ -634,7 +634,7 @@ public:
     uint32_t db_lost_callback_called_;
 
     /// @brief Callback function registered with the lease manager
-    bool db_recovered_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_recovered_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_recovered_callback_called_);
     }
 
@@ -642,7 +642,7 @@ public:
     uint32_t db_recovered_callback_called_;
 
     /// @brief Callback function registered with the lease manager
-    bool db_failed_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_failed_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_failed_callback_called_);
     }
 

src/lib/dhcpsrv/testutils/generic_cb_recovery_unittest.h

@@ -7,7 +7,7 @@
 #ifndef GENERIC_CONFIG_BACKEND_RECOVERY_H
 #define GENERIC_CONFIG_BACKEND_RECOVERY_H
 
-#include <database/database_connection.h>
+#include <util/reconnect_ctl.h>
 #include <database/server_collection.h>
 #include <dhcpsrv/config_backend_dhcp4_mgr.h>
 #include <dhcpsrv/testutils/generic_backend_unittest.h>
@@ -130,7 +130,7 @@ public:
     void testDbLostAndFailedAfterTimeoutCallback();
 
     /// @brief Callback function registered with the CB manager
-    bool db_lost_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_lost_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_lost_callback_called_);
     }
 
@@ -138,7 +138,7 @@ public:
     uint32_t db_lost_callback_called_;
 
     /// @brief Callback function registered with the CB manager
-    bool db_recovered_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_recovered_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_recovered_callback_called_);
     }
 
@@ -146,7 +146,7 @@ public:
     uint32_t db_recovered_callback_called_;
 
     /// @brief Callback function registered with the CB manager
-    bool db_failed_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_failed_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_failed_callback_called_);
     }
 

src/lib/dhcpsrv/testutils/generic_host_data_source_unittest.h

@@ -8,7 +8,7 @@
 #define GENERIC_HOST_DATA_SOURCE_UNITTEST_H
 
 #include <asiolink/io_address.h>
-#include <database/database_connection.h>
+#include <util/reconnect_ctl.h>
 #include <dhcpsrv/base_host_data_source.h>
 #include <dhcpsrv/cfgmgr.h>
 #include <dhcpsrv/host.h>
@@ -653,7 +653,7 @@ public:
     void testDbLostAndFailedAfterTimeoutCallback();
 
     /// @brief Callback function registered with the host manager
-    bool db_lost_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_lost_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_lost_callback_called_);
     }
 
@@ -661,7 +661,7 @@ public:
     uint32_t db_lost_callback_called_;
 
     /// @brief Callback function registered with the host manager
-    bool db_recovered_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_recovered_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_recovered_callback_called_);
     }
 
@@ -669,7 +669,7 @@ public:
     uint32_t db_recovered_callback_called_;
 
     /// @brief Callback function registered with the host manager
-    bool db_failed_callback(db::ReconnectCtlPtr /* not_used */) {
+    bool db_failed_callback(util::ReconnectCtlPtr /* not_used */) {
         return (++db_failed_callback_called_);
     }
 

src/lib/util/Makefile.am

@@ -24,6 +24,7 @@ libkea_util_la_SOURCES += pid_file.h pid_file.cc
 libkea_util_la_SOURCES += pointer_util.h
 libkea_util_la_SOURCES += range_utilities.h
 libkea_util_la_SOURCES += readwrite_mutex.h
+libkea_util_la_SOURCES += reconnect_ctl.h reconnect_ctl.cc
 libkea_util_la_SOURCES += staged_value.h
 libkea_util_la_SOURCES += state_model.cc state_model.h
 libkea_util_la_SOURCES += stopwatch.cc stopwatch.h
@@ -72,6 +73,7 @@ libkea_util_include_HEADERS = \
 	pointer_util.h \
 	range_utilities.h \
 	readwrite_mutex.h \
+	reconnect_ctl.h \
 	staged_value.h \
 	state_model.h \
 	stopwatch.h \

src/lib/util/reconnect_ctl.cc

@@ -0,0 +1,41 @@
+// Copyright (C) 2022 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <config.h>
+#include <util/reconnect_ctl.h>
+#include <exceptions/exceptions.h>
+
+namespace isc {
+namespace util {
+
+std::string
+ReconnectCtl::onFailActionToText(OnFailAction action) {
+    switch (action) {
+    case OnFailAction::STOP_RETRY_EXIT:
+        return ("stop-retry-exit");
+    case OnFailAction::SERVE_RETRY_EXIT:
+        return ("serve-retry-exit");
+    case OnFailAction::SERVE_RETRY_CONTINUE:
+        return ("serve-retry-continue");
+    }
+    return ("invalid-action-type");
+}
+
+OnFailAction
+ReconnectCtl::onFailActionFromText(const std::string& text) {
+    if (text == "stop-retry-exit") {
+        return (OnFailAction::STOP_RETRY_EXIT);
+    } else if (text == "serve-retry-exit") {
+        return (OnFailAction::SERVE_RETRY_EXIT);
+    } else if (text == "serve-retry-continue") {
+        return (OnFailAction::SERVE_RETRY_CONTINUE);
+    } else {
+        isc_throw(BadValue, "Invalid action on connection loss: " << text);
+    }
+}
+
+}
+}

src/lib/util/reconnect_ctl.h

@@ -0,0 +1,143 @@
+// Copyright (C) 2022 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef RECONNECT_CTL_H
+#define RECONNECT_CTL_H
+
+#include <string>
+#include <boost/shared_ptr.hpp>
+
+namespace isc {
+namespace util {
+
+/// @brief Type of action to take on connection loss.
+enum class OnFailAction {
+    STOP_RETRY_EXIT,
+    SERVE_RETRY_EXIT,
+    SERVE_RETRY_CONTINUE
+};
+
+/// @brief Warehouses reconnect control values
+///
+/// When any connection loses connectivity to its backend, it
+/// creates an instance of this class based on its configuration parameters and
+/// passes the instance into connection's lost callback.  This allows
+/// the layer(s) above the connection to know how to proceed.
+///
+class ReconnectCtl {
+public:
+    /// @brief Constructor.
+    ///
+    /// @param backend_type type of the caller backend.
+    /// @param timer_name timer associated to this object.
+    /// @param max_retries maximum number of reconnect attempts to make.
+    /// @param retry_interval amount of time to between reconnect attempts.
+    /// @param action which should be taken on connection loss.
+    ReconnectCtl(const std::string& backend_type, const std::string& timer_name,
+                 unsigned int max_retries, unsigned int retry_interval,
+                 OnFailAction action) :
+          backend_type_(backend_type), timer_name_(timer_name),
+          max_retries_(max_retries), retries_left_(max_retries),
+          retry_interval_(retry_interval), action_(action) {}
+
+    /// @brief Returns the type of the caller backend.
+    std::string backendType() const {
+        return (backend_type_);
+    }
+
+    /// @brief Returns the associated timer name.
+    ///
+    /// @return the associated timer.
+    std::string timerName() const {
+        return (timer_name_);
+    }
+
+    /// @brief Decrements the number of retries remaining
+    ///
+    /// Each call decrements the number of retries by one until zero is reached.
+    /// @return true the number of retries remaining is greater than zero.
+    bool checkRetries() {
+        return (retries_left_ ? --retries_left_ : false);
+    }
+
+    /// @brief Returns the maximum number of retries allowed.
+    unsigned int maxRetries() const {
+        return (max_retries_);
+    }
+
+    /// @brief Returns the number for retries remaining.
+    unsigned int retriesLeft() const {
+        return (retries_left_);
+    }
+
+    /// @brief Returns an index of current retry.
+    unsigned int retryIndex() {
+        return (max_retries_ - retries_left_);
+    }
+
+    /// @brief Returns the amount of time to wait between reconnect attempts.
+    unsigned int retryInterval() const {
+        return (retry_interval_);
+    }
+
+    /// @brief Resets the retries count.
+    void resetRetries() {
+        retries_left_ = max_retries_;
+    }
+
+    /// @brief Return true if the connection loss should affect the service,
+    /// false otherwise
+    bool alterServiceState() const {
+        return (action_ == OnFailAction::STOP_RETRY_EXIT);
+    }
+
+    /// @brief Return true if the connection recovery mechanism should shut down
+    /// the server on failure, false otherwise.
+    bool exitOnFailure() const {
+        return ((action_ == OnFailAction::STOP_RETRY_EXIT) ||
+                (action_ == OnFailAction::SERVE_RETRY_EXIT));
+    }
+
+    /// @brief Convert action to string.
+    ///
+    /// @param action The action type to be converted to text.
+    /// @return The text representation of the action type.
+    static std::string onFailActionToText(OnFailAction action);
+
+    /// @brief Convert string to action.
+    ///
+    /// @param text The text to be converted to action type.
+    /// @return The action type corresponding to the text representation.
+    static OnFailAction onFailActionFromText(const std::string& text);
+
+private:
+
+    /// @brief Caller backend type.
+    const std::string backend_type_;
+
+    /// @brief Timer associated to this object.
+    std::string timer_name_;
+
+    /// @brief Maximum number of retry attempts to make.
+    unsigned int max_retries_;
+
+    /// @brief Number of attempts remaining.
+    unsigned int retries_left_;
+
+    /// @brief The amount of time to wait between reconnect attempts.
+    unsigned int retry_interval_;
+
+    /// @brief Action to take on connection loss.
+    OnFailAction action_;
+};
+
+/// @brief Pointer to an instance of ReconnectCtl
+typedef boost::shared_ptr<ReconnectCtl> ReconnectCtlPtr;
+
+}
+}
+
+#endif // RECONNECT_CTL_H