[5477] dhcpsrv/postgres and kea-dhcp4 now support db reconnect

kea-dhcp4
    added support for max-reconnect-tries and reconnect-wait-time
    to lease and host db parsers

    Added a callback for when DB backends detect loss of connectivity

    Added a self-rescheduling method to attempt to reconnect to the
    backends if retries are enabled

dhcpsrv
    Added a callback that DatabaseConnection derivations should invoke
    when they lose connectivity.

    Added an optional callback parameter from CfgDbAccess::createManagers()
    all the way down to DatabaseConnection ctor.

    pgsql_connection.cc
        PgSqlConnection::~PgSqlConnection() - Added logic to close the
        connection only when the connect state is still OK.
        Otherwise it likes to core dump.

        PgSqlConnection::checkStatementError() - Modified to invoke the
        connectivity lost callback on "fatal" errors

    pgsql_lease_mgr_unittest.cc
    pgsql_host_data_source_unittest.cc
        Added tests to verify that the lost callback is NOT invoked on an
        open failure

src/bin/dhcp4/ctrl_dhcp4_srv.cc

@@ -18,6 +18,7 @@
 #include <hooks/hooks_manager.h>
 #include <stats/stats_mgr.h>
 #include <cfgrpt/config_report.h>
+#include <boost/lexical_cast.hpp>
 #include <signal.h>
 #include <sstream>
 
@@ -543,6 +544,83 @@ ControlledDhcpv4Srv::processCommand(const string& command,
     }
 }
 
+void
+ControlledDhcpv4Srv::dbReconnect() {
+    bool reopened = false;
+
+    // Re-open lease and host database with new parameters.
+    try {
+        CfgDbAccessPtr cfg_db = CfgMgr::instance().getCurrentCfg()->getCfgDbAccess();
+        cfg_db->createManagers(boost::bind(&ControlledDhcpv4Srv::dbLostCallback, this, _1));
+        reopened = true;
+    } catch (const std::exception& ex) {
+        LOG_ERROR(dhcp4_logger, DHCP4_DB_RECONNECT_ATTEMPT_FAILED).arg(ex.what());
+    }
+
+    if (reopened) {
+        // Cancel the timer.
+        if (TimerMgr::instance()->isTimerRegistered("Dhcp4DbReconnectTimer")) {
+            TimerMgr::instance()->cancel("Dhcp4DbReconnectTimer");
+        }
+
+        // Set network state to service enabled
+        network_state_.enableService();
+
+        // Toss the reconnct control, we're done with it
+        db_reconnect_ctl_.reset();
+    } else {
+        if (!db_reconnect_ctl_->checkRetries()) {
+            LOG_ERROR(dhcp4_logger, DHCP4_DB_RECONNECT_RETRIES_EXHAUSTED)
+            .arg(db_reconnect_ctl_->maxRetries());
+            shutdown();
+            return;
+        }
+
+        LOG_INFO(dhcp4_logger, DHCP4_DB_RECONNECT_ATTEMPT_SCHEDULE)
+                .arg(db_reconnect_ctl_->maxRetries() - db_reconnect_ctl_->retriesLeft())
+                .arg(db_reconnect_ctl_->maxRetries())
+                .arg(db_reconnect_ctl_->retryInterval());
+
+        if (!TimerMgr::instance()->isTimerRegistered("Dhcp4DbReconnectTimer")) {
+            TimerMgr::instance()->registerTimer("Dhcp4DbReconnectTimer",
+                            boost::bind(&ControlledDhcpv4Srv::dbReconnect, this),
+                            db_reconnect_ctl_->retryInterval() * 1000,
+                            asiolink::IntervalTimer::ONE_SHOT);
+        }
+
+        TimerMgr::instance()->setup("Dhcp4DbReconnectTimer");
+    }
+}
+
+bool
+ControlledDhcpv4Srv::dbLostCallback(ReconnectCtlPtr db_reconnect_ctl) {
+    // Disable service until we recover
+    network_state_.disableService();
+
+    if (!db_reconnect_ctl) {
+        // This shouldn't never happen
+        LOG_ERROR(dhcp4_logger, DHCP4_DB_RECONNECT_NO_DB_CTL);
+        return (false);
+    }
+
+    // If reconnect isn't enabled, log it and return false
+    if (!db_reconnect_ctl->retriesLeft() ||
+        !db_reconnect_ctl->retryInterval()) {
+        LOG_INFO(dhcp4_logger, DHCP4_DB_RECONNECT_DISABLED)
+            .arg(db_reconnect_ctl->retriesLeft())
+            .arg(db_reconnect_ctl->retryInterval());
+        return(false);
+    }
+
+    // Save the reconnect control
+    db_reconnect_ctl_ = db_reconnect_ctl;
+
+    // Invoke reconnect method
+    dbReconnect();
+
+    return(true);
+}
+
 isc::data::ConstElementPtr
 ControlledDhcpv4Srv::processConfig(isc::data::ConstElementPtr config) {
 
@@ -578,8 +656,7 @@ ControlledDhcpv4Srv::processConfig(isc::data::ConstElementPtr config) {
     try {
         CfgDbAccessPtr cfg_db = CfgMgr::instance().getStagingCfg()->getCfgDbAccess();
         cfg_db->setAppendedParameters("universe=4");
-        cfg_db->createManagers();
+        cfg_db->createManagers(boost::bind(&ControlledDhcpv4Srv::dbLostCallback, srv, _1));
-
     } catch (const std::exception& ex) {
         err << "Unable to open database: " << ex.what();
         return (isc::config::createAnswer(1, err.str()));
@@ -651,7 +728,8 @@ ControlledDhcpv4Srv::checkConfig(isc::data::ConstElementPtr config) {
 }
 
 ControlledDhcpv4Srv::ControlledDhcpv4Srv(uint16_t port /*= DHCP4_SERVER_PORT*/)
-    : Dhcpv4Srv(port), io_service_(), timer_mgr_(TimerMgr::instance()) {
+    : Dhcpv4Srv(port), io_service_(), timer_mgr_(TimerMgr::instance()),
+      db_reconnect_ctl_(0) {
     if (getInstance()) {
         isc_throw(InvalidOperation,
                   "There is another Dhcpv4Srv instance already.");

src/bin/dhcp4/ctrl_dhcp4_srv.h

@@ -11,6 +11,7 @@
 #include <asiolink/asiolink.h>
 #include <cc/data.h>
 #include <cc/command_interpreter.h>
+#include <dhcpsrv/database_connection.h>
 #include <dhcpsrv/timer_mgr.h>
 #include <dhcp4/dhcp4_srv.h>
 
@@ -331,6 +332,15 @@ private:
     /// Shared pointer to the instance of timer @c TimerMgr is held here to
     /// make sure that the @c TimerMgr outlives instance of this class.
     TimerMgrPtr timer_mgr_;
+
+    /// @brief Attempts to reconnect the server to the DB backend managers
+    void dbReconnect();
+
+    /// @brief Callback DB backends should invoke upon loss of connectivity
+    bool dbLostCallback(ReconnectCtlPtr db_reconnect_ctl);
+
+    /// @brief Pointer the current DB reconnect control values
+    ReconnectCtlPtr db_reconnect_ctl_;
 };
 
 }; // namespace isc::dhcp

src/bin/dhcp4/dhcp4_lexer.ll

@@ -440,6 +440,26 @@ ControlCharacterFill            [^"\\]|\\{JSONEscapeSequence}
     }
 }
 
+\"max-reconnect-tries\" {
+    switch(driver.ctx_) {
+    case isc::dhcp::Parser4Context::LEASE_DATABASE:
+    case isc::dhcp::Parser4Context::HOSTS_DATABASE:
+        return isc::dhcp::Dhcp4Parser::make_MAX_RECONNECT_TRIES(driver.loc_);
+    default:
+        return isc::dhcp::Dhcp4Parser::make_STRING("max-reconnect-tries", driver.loc_);
+    }
+}
+
+\"reconnect-wait-time\" {
+    switch(driver.ctx_) {
+    case isc::dhcp::Parser4Context::LEASE_DATABASE:
+    case isc::dhcp::Parser4Context::HOSTS_DATABASE:
+        return isc::dhcp::Dhcp4Parser::make_RECONNECT_WAIT_TIME(driver.loc_);
+    default:
+        return isc::dhcp::Dhcp4Parser::make_STRING("reconnect-wait-time", driver.loc_);
+    }
+}
+
 \"valid-lifetime\" {
     switch(driver.ctx_) {
     case isc::dhcp::Parser4Context::DHCP4:

src/bin/dhcp4/dhcp4_messages.mes

@@ -140,6 +140,33 @@ change is committed by the administrator.
 A debug message indicating that the DHCPv4 server has received an
 updated configuration from the Kea configuration system.
 
+% DHCP4_DB_RECONNECT_ATTEMPT_SCHEDULE Scheduling attempt %1 of %2 in %3 seconds
+An informational message indicating that the server is scheduling the next
+attempt to reconnect to its lease and/or host databases.  This occurs when the
+server has lost databse connectivity and is attempting to reconnect 
+automatically.
+
+% DHCP4_DB_RECONNECT_ATTEMPT_FAILED database reconnect failed: %1
+An error message indicating that an attempt to reconnect to the lease and/or
+host data bases has failed.  This occurs after connectivity to either one
+has been lost and an automatic attempt to reconnect has failed.
+
+% DHCP4_DB_RECONNECT_NO_DB_CTL unexpected error in database reconnect
+This is an error message indicating a programmatic error that should not
+occur. It will prohibit the server from attempting to reconnect to its
+databases if connectivy is lost, and the server will exit. This error
+should be reported. 
+
+% DHCP4_DB_RECONNECT_DISABLED database reconnect is disabled: max-reconnect-tries %1, reconnect-wait-time %1
+This is an informational message indicating that connetivity to either the
+lease or host database or both and that automatic reconnect is not enabled.
+
+% DHCP4_DB_RECONNECT_RETRIES_EXHAUSTED - Maximum number of Database reconnect attempts: %1, hasbeen exhausted without success, server is shutting down!
+This error indicates that the server is shutting after failing to reconnect to
+the lease and/or host database(s) after making the maximum configured number 
+of reconnect attempts.  Loss of connectivity is typically a network or database
+server issue.
+
 % DHCP4_DDNS_REQUEST_SEND_FAILED failed sending a request to kea-dhcp-ddns, error: %1,  ncr: %2
 This error message indicates that DHCP4 server attempted to send a DDNS
 update request to the DHCP-DDNS server.  This is most likely a configuration or

src/bin/dhcp4/dhcp4_parser.h

@@ -387,116 +387,118 @@ namespace isc { namespace dhcp {
         TOKEN_CONNECT_TIMEOUT = 294,
         TOKEN_CONTACT_POINTS = 295,
         TOKEN_KEYSPACE = 296,
-        TOKEN_VALID_LIFETIME = 297,
+        TOKEN_MAX_RECONNECT_TRIES = 297,
-        TOKEN_RENEW_TIMER = 298,
+        TOKEN_RECONNECT_WAIT_TIME = 298,
-        TOKEN_REBIND_TIMER = 299,
+        TOKEN_VALID_LIFETIME = 299,
-        TOKEN_DECLINE_PROBATION_PERIOD = 300,
+        TOKEN_RENEW_TIMER = 300,
-        TOKEN_SUBNET4 = 301,
+        TOKEN_REBIND_TIMER = 301,
-        TOKEN_SUBNET_4O6_INTERFACE = 302,
+        TOKEN_DECLINE_PROBATION_PERIOD = 302,
-        TOKEN_SUBNET_4O6_INTERFACE_ID = 303,
+        TOKEN_SUBNET4 = 303,
-        TOKEN_SUBNET_4O6_SUBNET = 304,
+        TOKEN_SUBNET_4O6_INTERFACE = 304,
-        TOKEN_OPTION_DEF = 305,
+        TOKEN_SUBNET_4O6_INTERFACE_ID = 305,
-        TOKEN_OPTION_DATA = 306,
+        TOKEN_SUBNET_4O6_SUBNET = 306,
-        TOKEN_NAME = 307,
+        TOKEN_OPTION_DEF = 307,
-        TOKEN_DATA = 308,
+        TOKEN_OPTION_DATA = 308,
-        TOKEN_CODE = 309,
+        TOKEN_NAME = 309,
-        TOKEN_SPACE = 310,
+        TOKEN_DATA = 310,
-        TOKEN_CSV_FORMAT = 311,
+        TOKEN_CODE = 311,
-        TOKEN_ALWAYS_SEND = 312,
+        TOKEN_SPACE = 312,
-        TOKEN_RECORD_TYPES = 313,
+        TOKEN_CSV_FORMAT = 313,
-        TOKEN_ENCAPSULATE = 314,
+        TOKEN_ALWAYS_SEND = 314,
-        TOKEN_ARRAY = 315,
+        TOKEN_RECORD_TYPES = 315,
-        TOKEN_SHARED_NETWORKS = 316,
+        TOKEN_ENCAPSULATE = 316,
-        TOKEN_POOLS = 317,
+        TOKEN_ARRAY = 317,
-        TOKEN_POOL = 318,
+        TOKEN_SHARED_NETWORKS = 318,
-        TOKEN_USER_CONTEXT = 319,
+        TOKEN_POOLS = 319,
-        TOKEN_COMMENT = 320,
+        TOKEN_POOL = 320,
-        TOKEN_SUBNET = 321,
+        TOKEN_USER_CONTEXT = 321,
-        TOKEN_INTERFACE = 322,
+        TOKEN_COMMENT = 322,
-        TOKEN_INTERFACE_ID = 323,
+        TOKEN_SUBNET = 323,
-        TOKEN_ID = 324,
+        TOKEN_INTERFACE = 324,
-        TOKEN_RAPID_COMMIT = 325,
+        TOKEN_INTERFACE_ID = 325,
-        TOKEN_RESERVATION_MODE = 326,
+        TOKEN_ID = 326,
-        TOKEN_DISABLED = 327,
+        TOKEN_RAPID_COMMIT = 327,
-        TOKEN_OUT_OF_POOL = 328,
+        TOKEN_RESERVATION_MODE = 328,
-        TOKEN_ALL = 329,
+        TOKEN_DISABLED = 329,
-        TOKEN_HOST_RESERVATION_IDENTIFIERS = 330,
+        TOKEN_OUT_OF_POOL = 330,
-        TOKEN_CLIENT_CLASSES = 331,
+        TOKEN_ALL = 331,
-        TOKEN_TEST = 332,
+        TOKEN_HOST_RESERVATION_IDENTIFIERS = 332,
-        TOKEN_CLIENT_CLASS = 333,
+        TOKEN_CLIENT_CLASSES = 333,
-        TOKEN_RESERVATIONS = 334,
+        TOKEN_TEST = 334,
-        TOKEN_DUID = 335,
+        TOKEN_CLIENT_CLASS = 335,
-        TOKEN_HW_ADDRESS = 336,
+        TOKEN_RESERVATIONS = 336,
-        TOKEN_CIRCUIT_ID = 337,
+        TOKEN_DUID = 337,
-        TOKEN_CLIENT_ID = 338,
+        TOKEN_HW_ADDRESS = 338,
-        TOKEN_HOSTNAME = 339,
+        TOKEN_CIRCUIT_ID = 339,
-        TOKEN_FLEX_ID = 340,
+        TOKEN_CLIENT_ID = 340,
-        TOKEN_RELAY = 341,
+        TOKEN_HOSTNAME = 341,
-        TOKEN_IP_ADDRESS = 342,
+        TOKEN_FLEX_ID = 342,
-        TOKEN_HOOKS_LIBRARIES = 343,
+        TOKEN_RELAY = 343,
-        TOKEN_LIBRARY = 344,
+        TOKEN_IP_ADDRESS = 344,
-        TOKEN_PARAMETERS = 345,
+        TOKEN_HOOKS_LIBRARIES = 345,
-        TOKEN_EXPIRED_LEASES_PROCESSING = 346,
+        TOKEN_LIBRARY = 346,
-        TOKEN_RECLAIM_TIMER_WAIT_TIME = 347,
+        TOKEN_PARAMETERS = 347,
-        TOKEN_FLUSH_RECLAIMED_TIMER_WAIT_TIME = 348,
+        TOKEN_EXPIRED_LEASES_PROCESSING = 348,
-        TOKEN_HOLD_RECLAIMED_TIME = 349,
+        TOKEN_RECLAIM_TIMER_WAIT_TIME = 349,
-        TOKEN_MAX_RECLAIM_LEASES = 350,
+        TOKEN_FLUSH_RECLAIMED_TIMER_WAIT_TIME = 350,
-        TOKEN_MAX_RECLAIM_TIME = 351,
+        TOKEN_HOLD_RECLAIMED_TIME = 351,
-        TOKEN_UNWARNED_RECLAIM_CYCLES = 352,
+        TOKEN_MAX_RECLAIM_LEASES = 352,
-        TOKEN_DHCP4O6_PORT = 353,
+        TOKEN_MAX_RECLAIM_TIME = 353,
-        TOKEN_CONTROL_SOCKET = 354,
+        TOKEN_UNWARNED_RECLAIM_CYCLES = 354,
-        TOKEN_SOCKET_TYPE = 355,
+        TOKEN_DHCP4O6_PORT = 355,
-        TOKEN_SOCKET_NAME = 356,
+        TOKEN_CONTROL_SOCKET = 356,
-        TOKEN_DHCP_DDNS = 357,
+        TOKEN_SOCKET_TYPE = 357,
-        TOKEN_ENABLE_UPDATES = 358,
+        TOKEN_SOCKET_NAME = 358,
-        TOKEN_QUALIFYING_SUFFIX = 359,
+        TOKEN_DHCP_DDNS = 359,
-        TOKEN_SERVER_IP = 360,
+        TOKEN_ENABLE_UPDATES = 360,
-        TOKEN_SERVER_PORT = 361,
+        TOKEN_QUALIFYING_SUFFIX = 361,
-        TOKEN_SENDER_IP = 362,
+        TOKEN_SERVER_IP = 362,
-        TOKEN_SENDER_PORT = 363,
+        TOKEN_SERVER_PORT = 363,
-        TOKEN_MAX_QUEUE_SIZE = 364,
+        TOKEN_SENDER_IP = 364,
-        TOKEN_NCR_PROTOCOL = 365,
+        TOKEN_SENDER_PORT = 365,
-        TOKEN_NCR_FORMAT = 366,
+        TOKEN_MAX_QUEUE_SIZE = 366,
-        TOKEN_ALWAYS_INCLUDE_FQDN = 367,
+        TOKEN_NCR_PROTOCOL = 367,
-        TOKEN_OVERRIDE_NO_UPDATE = 368,
+        TOKEN_NCR_FORMAT = 368,
-        TOKEN_OVERRIDE_CLIENT_UPDATE = 369,
+        TOKEN_ALWAYS_INCLUDE_FQDN = 369,
-        TOKEN_REPLACE_CLIENT_NAME = 370,
+        TOKEN_OVERRIDE_NO_UPDATE = 370,
-        TOKEN_GENERATED_PREFIX = 371,
+        TOKEN_OVERRIDE_CLIENT_UPDATE = 371,
-        TOKEN_TCP = 372,
+        TOKEN_REPLACE_CLIENT_NAME = 372,
-        TOKEN_JSON = 373,
+        TOKEN_GENERATED_PREFIX = 373,
-        TOKEN_WHEN_PRESENT = 374,
+        TOKEN_TCP = 374,
-        TOKEN_NEVER = 375,
+        TOKEN_JSON = 375,
-        TOKEN_ALWAYS = 376,
+        TOKEN_WHEN_PRESENT = 376,
-        TOKEN_WHEN_NOT_PRESENT = 377,
+        TOKEN_NEVER = 377,
-        TOKEN_LOGGING = 378,
+        TOKEN_ALWAYS = 378,
-        TOKEN_LOGGERS = 379,
+        TOKEN_WHEN_NOT_PRESENT = 379,
-        TOKEN_OUTPUT_OPTIONS = 380,
+        TOKEN_LOGGING = 380,
-        TOKEN_OUTPUT = 381,
+        TOKEN_LOGGERS = 381,
-        TOKEN_DEBUGLEVEL = 382,
+        TOKEN_OUTPUT_OPTIONS = 382,
-        TOKEN_SEVERITY = 383,
+        TOKEN_OUTPUT = 383,
-        TOKEN_FLUSH = 384,
+        TOKEN_DEBUGLEVEL = 384,
-        TOKEN_MAXSIZE = 385,
+        TOKEN_SEVERITY = 385,
-        TOKEN_MAXVER = 386,
+        TOKEN_FLUSH = 386,
-        TOKEN_DHCP6 = 387,
+        TOKEN_MAXSIZE = 387,
-        TOKEN_DHCPDDNS = 388,
+        TOKEN_MAXVER = 388,
-        TOKEN_CONTROL_AGENT = 389,
+        TOKEN_DHCP6 = 389,
-        TOKEN_TOPLEVEL_JSON = 390,
+        TOKEN_DHCPDDNS = 390,
-        TOKEN_TOPLEVEL_DHCP4 = 391,
+        TOKEN_CONTROL_AGENT = 391,
-        TOKEN_SUB_DHCP4 = 392,
+        TOKEN_TOPLEVEL_JSON = 392,
-        TOKEN_SUB_INTERFACES4 = 393,
+        TOKEN_TOPLEVEL_DHCP4 = 393,
-        TOKEN_SUB_SUBNET4 = 394,
+        TOKEN_SUB_DHCP4 = 394,
-        TOKEN_SUB_POOL4 = 395,
+        TOKEN_SUB_INTERFACES4 = 395,
-        TOKEN_SUB_RESERVATION = 396,
+        TOKEN_SUB_SUBNET4 = 396,
-        TOKEN_SUB_OPTION_DEFS = 397,
+        TOKEN_SUB_POOL4 = 397,
-        TOKEN_SUB_OPTION_DEF = 398,
+        TOKEN_SUB_RESERVATION = 398,
-        TOKEN_SUB_OPTION_DATA = 399,
+        TOKEN_SUB_OPTION_DEFS = 399,
-        TOKEN_SUB_HOOKS_LIBRARY = 400,
+        TOKEN_SUB_OPTION_DEF = 400,
-        TOKEN_SUB_DHCP_DDNS = 401,
+        TOKEN_SUB_OPTION_DATA = 401,
-        TOKEN_SUB_LOGGING = 402,
+        TOKEN_SUB_HOOKS_LIBRARY = 402,
-        TOKEN_STRING = 403,
+        TOKEN_SUB_DHCP_DDNS = 403,
-        TOKEN_INTEGER = 404,
+        TOKEN_SUB_LOGGING = 404,
-        TOKEN_FLOAT = 405,
+        TOKEN_STRING = 405,
-        TOKEN_BOOLEAN = 406
+        TOKEN_INTEGER = 406,
+        TOKEN_FLOAT = 407,
+        TOKEN_BOOLEAN = 408
       };
     };
 
@@ -771,6 +773,14 @@ namespace isc { namespace dhcp {
     symbol_type
     make_KEYSPACE (const location_type& l);
 
+    static inline
+    symbol_type
+    make_MAX_RECONNECT_TRIES (const location_type& l);
+
+    static inline
+    symbol_type
+    make_RECONNECT_WAIT_TIME (const location_type& l);
+
     static inline
     symbol_type
     make_VALID_LIFETIME (const location_type& l);
@@ -1416,12 +1426,12 @@ namespace isc { namespace dhcp {
     enum
     {
       yyeof_ = 0,
-      yylast_ = 892,     ///< Last index in yytable_.
+      yylast_ = 900,     ///< Last index in yytable_.
-      yynnts_ = 337,  ///< Number of nonterminal symbols.
+      yynnts_ = 339,  ///< Number of nonterminal symbols.
       yyfinal_ = 28, ///< Termination state number.
       yyterror_ = 1,
       yyerrcode_ = 256,
-      yyntokens_ = 152  ///< Number of tokens.
+      yyntokens_ = 154  ///< Number of tokens.
     };
 
 
@@ -1478,9 +1488,9 @@ namespace isc { namespace dhcp {
      115,   116,   117,   118,   119,   120,   121,   122,   123,   124,
      125,   126,   127,   128,   129,   130,   131,   132,   133,   134,
      135,   136,   137,   138,   139,   140,   141,   142,   143,   144,
-     145,   146,   147,   148,   149,   150,   151
+     145,   146,   147,   148,   149,   150,   151,   152,   153
     };
-    const unsigned int user_token_number_max_ = 406;
+    const unsigned int user_token_number_max_ = 408;
     const token_number_type undef_token_ = 2;
 
     if (static_cast<int>(t) <= yyeof_)
@@ -1513,30 +1523,30 @@ namespace isc { namespace dhcp {
   {
       switch (other.type_get ())
     {
-      case 167: // value
+      case 169: // value
-      case 171: // map_value
+      case 173: // map_value
-      case 209: // socket_type
+      case 211: // socket_type
-      case 212: // outbound_interface_value
+      case 214: // outbound_interface_value
-      case 222: // db_type
+      case 224: // db_type
-      case 299: // hr_mode
+      case 303: // hr_mode
-      case 444: // ncr_protocol_value
+      case 448: // ncr_protocol_value
-      case 452: // replace_client_name_value
+      case 456: // replace_client_name_value
         value.copy< ElementPtr > (other.value);
         break;
 
-      case 151: // "boolean"
+      case 153: // "boolean"
         value.copy< bool > (other.value);
         break;
 
-      case 150: // "floating point"
+      case 152: // "floating point"
         value.copy< double > (other.value);
         break;
 
-      case 149: // "integer"
+      case 151: // "integer"
         value.copy< int64_t > (other.value);
         break;
 
-      case 148: // "constant string"
+      case 150: // "constant string"
         value.copy< std::string > (other.value);
         break;
 
@@ -1557,30 +1567,30 @@ namespace isc { namespace dhcp {
     (void) v;
       switch (this->type_get ())
     {
-      case 167: // value
+      case 169: // value
-      case 171: // map_value
+      case 173: // map_value
-      case 209: // socket_type
+      case 211: // socket_type
-      case 212: // outbound_interface_value
+      case 214: // outbound_interface_value
-      case 222: // db_type
+      case 224: // db_type
-      case 299: // hr_mode
+      case 303: // hr_mode
-      case 444: // ncr_protocol_value
+      case 448: // ncr_protocol_value
-      case 452: // replace_client_name_value
+      case 456: // replace_client_name_value
         value.copy< ElementPtr > (v);
         break;
 
-      case 151: // "boolean"
+      case 153: // "boolean"
         value.copy< bool > (v);
         break;
 
-      case 150: // "floating point"
+      case 152: // "floating point"
         value.copy< double > (v);
         break;
 
-      case 149: // "integer"
+      case 151: // "integer"
         value.copy< int64_t > (v);
         break;
 
-      case 148: // "constant string"
+      case 150: // "constant string"
         value.copy< std::string > (v);
         break;
 
@@ -1660,30 +1670,30 @@ namespace isc { namespace dhcp {
     // Type destructor.
     switch (yytype)
     {
-      case 167: // value
+      case 169: // value
-      case 171: // map_value
+      case 173: // map_value
-      case 209: // socket_type
+      case 211: // socket_type
-      case 212: // outbound_interface_value
+      case 214: // outbound_interface_value
-      case 222: // db_type
+      case 224: // db_type
-      case 299: // hr_mode
+      case 303: // hr_mode
-      case 444: // ncr_protocol_value
+      case 448: // ncr_protocol_value
-      case 452: // replace_client_name_value
+      case 456: // replace_client_name_value
         value.template destroy< ElementPtr > ();
         break;
 
-      case 151: // "boolean"
+      case 153: // "boolean"
         value.template destroy< bool > ();
         break;
 
-      case 150: // "floating point"
+      case 152: // "floating point"
         value.template destroy< double > ();
         break;
 
-      case 149: // "integer"
+      case 151: // "integer"
         value.template destroy< int64_t > ();
         break;
 
-      case 148: // "constant string"
+      case 150: // "constant string"
         value.template destroy< std::string > ();
         break;
 
@@ -1710,30 +1720,30 @@ namespace isc { namespace dhcp {
     super_type::move(s);
       switch (this->type_get ())
     {
-      case 167: // value
+      case 169: // value
-      case 171: // map_value
+      case 173: // map_value
-      case 209: // socket_type
+      case 211: // socket_type
-      case 212: // outbound_interface_value
+      case 214: // outbound_interface_value
-      case 222: // db_type
+      case 224: // db_type
-      case 299: // hr_mode
+      case 303: // hr_mode
-      case 444: // ncr_protocol_value
+      case 448: // ncr_protocol_value
-      case 452: // replace_client_name_value
+      case 456: // replace_client_name_value
         value.move< ElementPtr > (s.value);
         break;
 
-      case 151: // "boolean"
+      case 153: // "boolean"
         value.move< bool > (s.value);
         break;
 
-      case 150: // "floating point"
+      case 152: // "floating point"
         value.move< double > (s.value);
         break;
 
-      case 149: // "integer"
+      case 151: // "integer"
         value.move< int64_t > (s.value);
         break;
 
-      case 148: // "constant string"
+      case 150: // "constant string"
         value.move< std::string > (s.value);
         break;
 
@@ -1807,7 +1817,7 @@ namespace isc { namespace dhcp {
      375,   376,   377,   378,   379,   380,   381,   382,   383,   384,
      385,   386,   387,   388,   389,   390,   391,   392,   393,   394,
      395,   396,   397,   398,   399,   400,   401,   402,   403,   404,
-     405,   406
+     405,   406,   407,   408
     };
     return static_cast<token_type> (yytoken_number_[type]);
   }
@@ -2052,6 +2062,18 @@ namespace isc { namespace dhcp {
     return symbol_type (token::TOKEN_KEYSPACE, l);
   }
 
+  Dhcp4Parser::symbol_type
+  Dhcp4Parser::make_MAX_RECONNECT_TRIES (const location_type& l)
+  {
+    return symbol_type (token::TOKEN_MAX_RECONNECT_TRIES, l);
+  }
+
+  Dhcp4Parser::symbol_type
+  Dhcp4Parser::make_RECONNECT_WAIT_TIME (const location_type& l)
+  {
+    return symbol_type (token::TOKEN_RECONNECT_WAIT_TIME, l);
+  }
+
   Dhcp4Parser::symbol_type
   Dhcp4Parser::make_VALID_LIFETIME (const location_type& l)
   {
@@ -2715,7 +2737,7 @@ namespace isc { namespace dhcp {
 
 #line 14 "dhcp4_parser.yy" // lalr1.cc:377
 } } // isc::dhcp
-#line 2719 "dhcp4_parser.h" // lalr1.cc:377
+#line 2741 "dhcp4_parser.h" // lalr1.cc:377
 
 
 

src/bin/dhcp4/dhcp4_parser.yy

@@ -83,6 +83,8 @@ using namespace std;
   CONNECT_TIMEOUT "connect-timeout"
   CONTACT_POINTS "contact-points"
   KEYSPACE "keyspace"
+  MAX_RECONNECT_TRIES "max-reconnect-tries"
+  RECONNECT_WAIT_TIME "reconnect-wait-time"
 
   VALID_LIFETIME "valid-lifetime"
   RENEW_TIMER "renew-timer"
@@ -583,6 +585,8 @@ database_map_param: database_type
                   | readonly
                   | connect_timeout
                   | contact_points
+                  | max_reconnect_tries
+                  | reconnect_wait_time
                   | keyspace
                   | unknown_map_entry
                   ;
@@ -673,6 +677,15 @@ keyspace: KEYSPACE {
     ctx.leave();
 };
 
+max_reconnect_tries: MAX_RECONNECT_TRIES COLON INTEGER {
+    ElementPtr n(new IntElement($3, ctx.loc2pos(@3)));
+    ctx.stack_.back()->set("max-reconnect-tries", n);
+};
+
+reconnect_wait_time: RECONNECT_WAIT_TIME COLON INTEGER {
+    ElementPtr n(new IntElement($3, ctx.loc2pos(@3)));
+    ctx.stack_.back()->set("reconnect-wait-time", n);
+};
 
 host_reservation_identifiers: HOST_RESERVATION_IDENTIFIERS {
     ElementPtr l(new ListElement(ctx.loc2pos(@1)));

src/bin/dhcp4/location.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Locations for Bison parsers in C++

src/bin/dhcp4/position.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Positions for Bison parsers in C++

src/bin/dhcp4/stack.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Stack handling for Bison parsers in C++

src/bin/dhcp6/location.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Locations for Bison parsers in C++

src/bin/dhcp6/position.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Positions for Bison parsers in C++

src/bin/dhcp6/stack.hh

@@ -1,4 +1,3 @@
-// Generated 201712311009
 // A Bison parser, made by GNU Bison 3.0.4.
 
 // Stack handling for Bison parsers in C++

src/lib/dhcpsrv/cfg_db_access.cc

@@ -38,15 +38,15 @@ CfgDbAccess::getHostDbAccessString() const {
 
 
 void
-CfgDbAccess::createManagers() const {
+CfgDbAccess::createManagers(DatabaseConnection::Callback db_lost_callback) const {
     // Recreate lease manager.
     LeaseMgrFactory::destroy();
-    LeaseMgrFactory::create(getLeaseDbAccessString());
+    LeaseMgrFactory::create(getLeaseDbAccessString(), db_lost_callback);
 
     // Recreate host data source.
     HostDataSourceFactory::destroy();
     if (!host_db_access_.empty()) {
-        HostMgr::create(getHostDbAccessString());
+        HostMgr::create(getHostDbAccessString(), db_lost_callback);
     }
 }
 

src/lib/dhcpsrv/cfg_db_access.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2016-2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2016-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -8,6 +8,8 @@
 #define CFG_DBACCESS_H
 
 #include <cc/cfg_to_element.h>
+#include <dhcpsrv/database_connection.h>
+
 #include <boost/shared_ptr.hpp>
 #include <string>
 
@@ -62,7 +64,11 @@ public:
 
     /// @brief Creates instance of lease manager and host data source
     /// according to the configuration specified.
-    void createManagers() const;
+    ///
+    /// @param db_lost_callback function to invoke if connectivity to
+    /// either the lease or host managers, once established, is subsequently
+    /// lost.
+    void createManagers(DatabaseConnection::Callback db_lost_callback = NULL) const;
 
     /// @brief Unparse an access string
     ///

src/lib/dhcpsrv/database_connection.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2015-2016 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2015-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -106,5 +106,41 @@ DatabaseConnection::configuredReadOnly() const {
     return (readonly_value == "true");
 }
 
+ReconnectCtlPtr
+DatabaseConnection::makeReconnectCtl() const {
+    ReconnectCtlPtr retry;
+    unsigned int retries = 0;
+    unsigned int interval = 0;
+
+    // Assumes that parsing ensurse only valid values are present
+    std::string parm_str;
+    try {
+        parm_str = getParameter("max-reconnect-tries");
+        retries = boost::lexical_cast<unsigned int>(parm_str);
+    } catch (...) {
+        // Wasn't specified so  so we'll use default of 0;
+    }
+
+    try {
+        parm_str = getParameter("reconnect-wait-time");
+        interval = boost::lexical_cast<unsigned int>(parm_str);
+    } catch (...) {
+        // Wasn't specified so  so we'll use default of 0;
+    }
+
+    retry.reset(new ReconnectCtl(retries, interval));
+    return (retry);
+}
+
+bool
+DatabaseConnection::invokeDbLostCallback() const {
+    if (db_lost_callback_ != NULL) {
+        // Invoke the callback, passing in a new instance of ReconnectCtl
+        return (db_lost_callback_)(makeReconnectCtl());
+    }
+
+    return (false);
+}
+
 };
 };

src/lib/dhcpsrv/database_connection.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2015-2016 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2015-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -8,6 +8,8 @@
 #define DATABASE_CONNECTION_H
 
 #include <boost/noncopyable.hpp>
+#include <boost/function.hpp>
+#include <boost/shared_ptr.hpp>
 #include <exceptions/exceptions.h>
 #include <map>
 #include <string>
@@ -63,6 +65,58 @@ public:
         isc::Exception(file, line, what) {}
 };
 
+/// @brief Warehouses DB reconnect control values
+///
+/// When a DatabaseConnection loses connectivity to its backend, it
+/// creates an of this class based on its configuration parameters and
+/// passes the instance into connection's DB lost callback.  This allows
+/// the layer(s) above the connection to know how to proceed.
+///
+class ReconnectCtl {
+public:
+    /// @brief Constructor
+    /// @param max_retries maximum number of reconnect attempts to make
+    /// @param retry_interval amount of time to between reconnect attempts
+    ReconnectCtl(unsigned int max_retries, unsigned int retry_interval)
+        : max_retries_(max_retries), retries_left_(max_retries), 
+          retry_interval_(retry_interval) {}
+
+    /// @brief Decrements the number of retries remaining
+    ///
+    /// Each call decrements the number of retries by one until zero is reached.
+    /// @return true the number of retries remaining is greater than zero.
+    bool checkRetries() {
+        return (retries_left_ ? --retries_left_ : false);
+    }
+
+    /// @brief Returns the maximum number for retries allowed 
+    unsigned int maxRetries() {
+        return (max_retries_);
+    }
+
+    /// @brief Returns the number for retries remaining
+    unsigned int retriesLeft() {
+        return (retries_left_);
+    }
+
+    /// @brief Returns the amount of time to wait between reconnect attempts
+    unsigned int retryInterval() {
+        return (retry_interval_);
+    }
+
+private:
+    /// @brief Maximum number of retry attempts to make
+    unsigned int max_retries_;
+
+    /// @brief Number of attempts remaining
+    unsigned int retries_left_;
+
+    /// @brief The amount of time to wait between reconnect attempts
+    unsigned int retry_interval_;
+};
+
+/// @brief Pointer to an instance of ReconnectCtl
+typedef boost::shared_ptr<ReconnectCtl> ReconnectCtlPtr;
 
 /// @brief Common database connection class.
 ///
@@ -86,14 +140,29 @@ public:
     /// @brief Database configuration parameter map
     typedef std::map<std::string, std::string> ParameterMap;
 
+    /// @brief Defines a callback prototype for propogating events upward
+    /// typedef boost::function<bool (const ParameterMap& parameters)> Callback;
+    typedef boost::function<bool (ReconnectCtlPtr db_retry)> Callback;
+
     /// @brief Constructor
     ///
     /// @param parameters A data structure relating keywords and values
     ///        concerned with the database.
-    DatabaseConnection(const ParameterMap& parameters)
+    /// @param db_lost_callback  Optional call back function to invoke if a
-        :parameters_(parameters) {
+    ///        successfully open connection subsequently fails
+    DatabaseConnection(const ParameterMap& parameters,
+        Callback db_lost_callback = NULL)
+        :parameters_(parameters), db_lost_callback_(db_lost_callback) {
     }
 
+    /// @brief Destructor
+    virtual ~DatabaseConnection(){};
+
+    /// @brief Instantiates a ReconnectCtl based on the connection's
+    /// reconnect parameters
+    /// @return pointer to the new ReconnectCtl object
+    virtual ReconnectCtlPtr makeReconnectCtl() const;
+
     /// @brief Returns value of a connection parameter.
     ///
     /// @param name Name of the parameter which value should be returned.
@@ -129,6 +198,17 @@ public:
     /// and set to false.
     bool configuredReadOnly() const;
 
+    /// @brief Invokes the connection's lost connectivity callback
+    ///
+    /// This function is may be called by derivations when the connectivity
+    /// to their data server is lost.  If connectivity callback was specified,
+    /// this function will instantiate a ReconnectCtl and pass it to the
+    /// callback.
+    ///
+    /// @return Returns the result of the callback or false if there is no
+    /// callback.
+    bool invokeDbLostCallback() const;
+
 private:
 
     /// @brief List of parameters passed in dbconfig
@@ -138,6 +218,10 @@ private:
     /// intended to keep any DHCP-related parameters.
     ParameterMap parameters_;
 
+protected:
+
+    /// @brief Optional function to invoke if the connectivity is lost
+    Callback db_lost_callback_;
 };
 
 }; // end of isc::dhcp namespace

src/lib/dhcpsrv/host_data_source_factory.cc

@@ -45,7 +45,8 @@ HostDataSourceFactory::getHostDataSourcePtr() {
 }
 
 void
-HostDataSourceFactory::create(const std::string& dbaccess) {
+HostDataSourceFactory::create(const std::string& dbaccess,
+                              DatabaseConnection::Callback db_lost_callback) {
     // Parse the access string and create a redacted string for logging.
     DatabaseConnection::ParameterMap parameters =
             DatabaseConnection::parse(dbaccess);
@@ -72,7 +73,8 @@ HostDataSourceFactory::create(const std::string& dbaccess) {
     if (db_type == "postgresql") {
         LOG_INFO(dhcpsrv_logger, DHCPSRV_PGSQL_HOST_DB)
             .arg(DatabaseConnection::redactedAccessString(parameters));
-        getHostDataSourcePtr().reset(new PgSqlHostDataSource(parameters));
+        getHostDataSourcePtr().reset(new PgSqlHostDataSource(parameters,
+                                     db_lost_callback));
         return;
     }
 #endif

src/lib/dhcpsrv/host_data_source_factory.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2015-2016 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2015-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -29,9 +29,10 @@ public:
 
 /// @brief Host Data Source Factory
 ///
-/// This class comprises nothing but static methods used to create a host data source object.
+/// This class comprises nothing but static methods used to create a host data
-/// It analyzes the database information passed to the creation function and instantiates
+/// source object. It analyzes the database information passed to the creation
-/// an appropriate host data source object based on the type requested.
+/// function and instantiates an appropriate host data source object based on
+/// the type requested.
 ///
 /// Strictly speaking these functions could be stand-alone functions.  However,
 /// it is convenient to encapsulate them in a class for naming purposes.
@@ -58,11 +59,15 @@ public:
     ///        -end specific, although must include the "type" keyword which
     ///        gives the backend in use.
     ///
+    /// @param db_lost_callback function to invoke if connectivity to host
+    /// data source is lost.
+    ///
     /// @throw isc::InvalidParameter dbaccess string does not contain the "type"
     ///        keyword.
     /// @throw isc::dhcp::InvalidType The "type" keyword in dbaccess does not
     ///        identify a supported backend.
-    static void create(const std::string& dbaccess);
+    static void create(const std::string& dbaccess,
+                       DatabaseConnection::Callback db_lost_callback = NULL);
 
     /// @brief Destroy host data source
     ///

src/lib/dhcpsrv/host_mgr.cc

@@ -37,14 +37,15 @@ HostMgr::getHostMgrPtr() {
 }
 
 void
-HostMgr::create(const std::string& access) {
+HostMgr::create(const std::string& access,
+                DatabaseConnection::Callback db_lost_callback) {
     getHostMgrPtr().reset(new HostMgr());
 
     if (!access.empty()) {
         // If the user specified parameters, let's pass them to the create
         // method. It will destroy any prior instances and will create
         // the new one.
-        HostDataSourceFactory::create(access);
+        HostDataSourceFactory::create(access, db_lost_callback);
     } else {
         // Ok, no parameters were specified. We should destroy the existing
         // instance.

src/lib/dhcpsrv/host_mgr.h

@@ -10,6 +10,7 @@
 #include <dhcp/duid.h>
 #include <dhcp/hwaddr.h>
 #include <dhcpsrv/base_host_data_source.h>
+#include <dhcpsrv/database_connection.h>
 #include <dhcpsrv/host.h>
 #include <dhcpsrv/subnet_id.h>
 #include <boost/noncopyable.hpp>
@@ -68,7 +69,11 @@ public:
     /// However, the "type" parameter will be common and it will specify which
     /// data source is to be used. Currently, no parameters are supported
     /// and the parameter is ignored.
-    static void create(const std::string& access = "");
+    ///
+    /// @param db_lost_callback function to invoke if connectivity to
+    /// the host database is lost.
+    static void create(const std::string& access = "",
+                       DatabaseConnection::Callback db_lost_callback = NULL);
 
     /// @brief Returns a sole instance of the @c HostMgr.
     ///

src/lib/dhcpsrv/lease_mgr_factory.cc

@@ -41,7 +41,8 @@ LeaseMgrFactory::getLeaseMgrPtr() {
 }
 
 void
-LeaseMgrFactory::create(const std::string& dbaccess) {
+LeaseMgrFactory::create(const std::string& dbaccess,
+                        DatabaseConnection::Callback db_lost_callback) {
     const std::string type = "type";
 
     // Parse the access string and create a redacted string for logging.
@@ -67,7 +68,7 @@ LeaseMgrFactory::create(const std::string& dbaccess) {
 #ifdef HAVE_PGSQL
     if (parameters[type] == string("postgresql")) {
         LOG_INFO(dhcpsrv_logger, DHCPSRV_PGSQL_DB).arg(redacted);
-        getLeaseMgrPtr().reset(new PgSqlLeaseMgr(parameters));
+        getLeaseMgrPtr().reset(new PgSqlLeaseMgr(parameters, db_lost_callback));
         return;
     }
 #endif
@@ -115,6 +116,5 @@ LeaseMgrFactory::instance() {
     return (*lmptr);
 }
 
-
 }; // namespace dhcp
 }; // namespace isc

src/lib/dhcpsrv/lease_mgr_factory.h

@@ -63,11 +63,15 @@ public:
     ///        -end specific, although must include the "type" keyword which
     ///        gives the backend in use.
     ///
+    /// @param db_lost_callback function to invoke if connectivity to lease
+    /// database is lost.
+    ///
     /// @throw isc::InvalidParameter dbaccess string does not contain the "type"
     ///        keyword.
     /// @throw isc::dhcp::InvalidType The "type" keyword in dbaccess does not
     ///        identify a supported backend.
-    static void create(const std::string& dbaccess);
+    static void create(const std::string& dbaccess, 
+                       DatabaseConnection::Callback db_lost_callback = NULL);
 
     /// @brief Destroy lease manager
     ///

src/lib/dhcpsrv/parsers/dbaccess_parser.cc

@@ -56,6 +56,9 @@ DbAccessParser::parse(CfgDbAccessPtr& cfg_db,
     int64_t lfc_interval = 0;
     int64_t timeout = 0;
     int64_t port = 0;
+    int64_t max_reconnect_tries = 0;
+    int64_t reconnect_wait_time = 0;
+    
     // 2. Update the copy with the passed keywords.
     BOOST_FOREACH(ConfigPair param, database_config->mapValue()) {
         try {
@@ -74,10 +77,15 @@ DbAccessParser::parse(CfgDbAccessPtr& cfg_db,
                 values_copy[param.first] =
                     boost::lexical_cast<std::string>(timeout);
 
-            } else if (param.first == "reconnect-wait-time") {
+            } else if (param.first == "max-reconnect-tries") {
-                timeout = param.second->intValue();
+                max_reconnect_tries = param.second->intValue();
                 values_copy[param.first] =
-                    boost::lexical_cast<std::string>(timeout);
+                    boost::lexical_cast<std::string>(max_reconnect_tries);
+
+            } else if (param.first == "reconnect-wait-time") {
+                reconnect_wait_time = param.second->intValue();
+                values_copy[param.first] =
+                    boost::lexical_cast<std::string>(reconnect_wait_time);
 
             } else if (param.first == "request-timeout") {
                 timeout = param.second->intValue();
@@ -160,6 +168,21 @@ DbAccessParser::parse(CfgDbAccessPtr& cfg_db,
                   << " (" << value->getPosition() << ")");
     }
 
+    // Check that the max-reconnect-retries reasonable.
+    if (max_reconnect_tries < 0) {
+        ConstElementPtr value = database_config->get("max-reconnect-tries");
+        isc_throw(DhcpConfigError, "max-reconnect-tries cannot be less than zero: " 
+                  << " (" << value->getPosition() << ")");
+    }
+
+    // Check that the reconnect-wait-time reasonable.
+    if ((reconnect_wait_time < 0) ||
+        (port > std::numeric_limits<uint16_t>::max())) {
+        ConstElementPtr value = database_config->get("reconnect-wait-time");
+        isc_throw(DhcpConfigError, "reconnect-wait-time cannot be less than zero: " 
+                  << " (" << value->getPosition() << ")");
+    }
+
     // 4. If all is OK, update the stored keyword/value pairs.  We do this by
     // swapping contents - values_copy is destroyed immediately after the
     // operation (when the method exits), so we are not interested in its new

src/lib/dhcpsrv/pgsql_connection.cc

@@ -111,11 +111,13 @@ PgSqlTransaction::commit() {
 PgSqlConnection::~PgSqlConnection() {
     if (conn_) {
         // Deallocate the prepared queries.
-        PgSqlResult r(PQexec(conn_, "DEALLOCATE all"));
+        if (PQstatus(conn_) == CONNECTION_OK) {
-        if(PQresultStatus(r) != PGRES_COMMAND_OK) {
+            PgSqlResult r(PQexec(conn_, "DEALLOCATE all"));
-            // Highly unlikely but we'll log it and go on.
+            if(PQresultStatus(r) != PGRES_COMMAND_OK) {
-            LOG_ERROR(dhcpsrv_logger, DHCPSRV_PGSQL_DEALLOC_ERROR)
+                // Highly unlikely but we'll log it and go on.
-                      .arg(PQerrorMessage(conn_));
+                LOG_ERROR(dhcpsrv_logger, DHCPSRV_PGSQL_DEALLOC_ERROR)
+                          .arg(PQerrorMessage(conn_));
+            }
         }
     }
 }
@@ -287,25 +289,30 @@ PgSqlConnection::checkStatementError(const PgSqlResult& r,
     if (s != PGRES_COMMAND_OK && s != PGRES_TUPLES_OK) {
         // We're testing the first two chars of SQLSTATE, as this is the
         // error class. Note, there is a severity field, but it can be
-        // misleadingly returned as fatal.
+        // misleadingly returned as fatal. However, a loss of connectivity
+        // can lead to a NULL sqlstate with a status of PGRES_FATAL_ERROR.
         const char* sqlstate = PQresultErrorField(r, PG_DIAG_SQLSTATE);
-        if ((sqlstate != NULL) &&
+        if  ((sqlstate == NULL) || 
             ((memcmp(sqlstate, "08", 2) == 0) ||  // Connection Exception
              (memcmp(sqlstate, "53", 2) == 0) ||  // Insufficient resources
              (memcmp(sqlstate, "54", 2) == 0) ||  // Program Limit exceeded
              (memcmp(sqlstate, "57", 2) == 0) ||  // Operator intervention
-             (memcmp(sqlstate, "58", 2) == 0))) { // System error
+             (memcmp(sqlstate, "58", 2) == 0))) {// System error
             LOG_ERROR(dhcpsrv_logger, DHCPSRV_PGSQL_FATAL_ERROR)
                          .arg(statement.name)
                          .arg(PQerrorMessage(conn_))
-                         .arg(sqlstate);
+                         .arg(sqlstate ? sqlstate : "<sqlstate null>");
-            exit (-1);
+            // If there's no lost db callback, then exit
+            if (!invokeDbLostCallback()) {
+                exit (-1);
+            }
         }
 
         const char* error_message = PQerrorMessage(conn_);
         isc_throw(DbOperationError, "Statement exec failed:" << " for: "
-                  << statement.name << ", reason: "
+                  << statement.name << ", status: " << s
-                  << error_message);
+                  << "sqlstate:[ " << (sqlstate ? sqlstate : "<null>") 
+                  <<  "], reason: " << error_message);
     }
 }
 

src/lib/dhcpsrv/pgsql_connection.h

@@ -303,6 +303,11 @@ public:
         : DatabaseConnection(parameters) {
     }
 
+    PgSqlConnection(const ParameterMap& parameters, 
+                    Callback db_lost_callback)
+        : DatabaseConnection(parameters, db_lost_callback) {
+    }
+
     /// @brief Destructor
     virtual ~PgSqlConnection();
 
@@ -392,7 +397,7 @@ public:
     ///
     /// @throw isc::dhcp::DbOperationError Detailed PostgreSQL failure
     void checkStatementError(const PgSqlResult& r,
-                             PgSqlTaggedStatement& statement) const;
+                             PgSqlTaggedStatement& statement) const; 
 
     /// @brief PgSql connection handle
     ///

src/lib/dhcpsrv/pgsql_host_data_source.cc

@@ -1284,7 +1284,8 @@ public:
     ///
     /// This constructor opens database connection and initializes prepared
     /// statements used in the queries.
-    PgSqlHostDataSourceImpl(const PgSqlConnection::ParameterMap& parameters);
+    PgSqlHostDataSourceImpl(const PgSqlConnection::ParameterMap& parameters,
+                            DatabaseConnection::Callback db_lost_callback);
 
     /// @brief Destructor.
     ~PgSqlHostDataSourceImpl();
@@ -1699,14 +1700,15 @@ TaggedStatementArray tagged_statements = { {
 }; // end anonymous namespace
 
 PgSqlHostDataSourceImpl::
-PgSqlHostDataSourceImpl(const PgSqlConnection::ParameterMap& parameters)
+PgSqlHostDataSourceImpl(const PgSqlConnection::ParameterMap& parameters,
+                        DatabaseConnection::Callback db_lost_callback)
     : host_exchange_(new PgSqlHostWithOptionsExchange(PgSqlHostWithOptionsExchange::DHCP4_ONLY)),
       host_ipv6_exchange_(new PgSqlHostIPv6Exchange(PgSqlHostWithOptionsExchange::DHCP6_ONLY)),
       host_ipv46_exchange_(new PgSqlHostIPv6Exchange(PgSqlHostWithOptionsExchange::
                                                      DHCP4_AND_DHCP6)),
       host_ipv6_reservation_exchange_(new PgSqlIPv6ReservationExchange()),
       host_option_exchange_(new PgSqlOptionExchange()),
-      conn_(parameters),
+      conn_(parameters, db_lost_callback),
       is_readonly_(false) {
 
     // Open the database.
@@ -1926,8 +1928,9 @@ PgSqlHostDataSourceImpl::checkReadOnly() const {
 
 
 PgSqlHostDataSource::
-PgSqlHostDataSource(const PgSqlConnection::ParameterMap& parameters)
+PgSqlHostDataSource(const PgSqlConnection::ParameterMap& parameters,
-    : impl_(new PgSqlHostDataSourceImpl(parameters)) {
+                    DatabaseConnection::Callback db_lost_callback)
+    : impl_(new PgSqlHostDataSourceImpl(parameters, db_lost_callback)) {
 }
 
 PgSqlHostDataSource::~PgSqlHostDataSource() {

src/lib/dhcpsrv/pgsql_host_data_source.h

@@ -53,11 +53,15 @@ public:
     /// @param parameters A data structure relating keywords and values
     ///        concerned with the database.
     ///
+    /// @param db_lost_callback function to invoke if connectivity to
+    /// to host database is lost.
+    ///
     /// @throw isc::dhcp::NoDatabaseName Mandatory database name not given
     /// @throw isc::dhcp::DbOpenError Error opening the database
     /// @throw isc::dhcp::DbOperationError An operation on the open database has
     ///        failed.
-    PgSqlHostDataSource(const DatabaseConnection::ParameterMap& parameters);
+    PgSqlHostDataSource(const DatabaseConnection::ParameterMap& parameters,
+                        DatabaseConnection::Callback db_lost_callback = NULL); 
 
     /// @brief Virtual destructor.
     /// Frees database resources and closes the database connection through

src/lib/dhcpsrv/pgsql_lease_mgr.cc

@@ -820,9 +820,10 @@ protected:
     bool fetch_type_;
 };
 
-PgSqlLeaseMgr::PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters)
+PgSqlLeaseMgr::PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters,
+    DatabaseConnection::Callback db_lost_callback)
     : LeaseMgr(), exchange4_(new PgSqlLease4Exchange()),
-    exchange6_(new PgSqlLease6Exchange()), conn_(parameters) {
+    exchange6_(new PgSqlLease6Exchange()), conn_(parameters, db_lost_callback) {
     conn_.openDatabase();
     int i = 0;
     for( ; tagged_statements[i].text != NULL ; ++i) {

src/lib/dhcpsrv/pgsql_lease_mgr.h

@@ -51,11 +51,15 @@ public:
     /// @param parameters A data structure relating keywords and values
     ///        concerned with the database.
     ///
+    /// @param db_lost_callback function to invoke if connectivity to
+    /// to lease database is lost.
+    ///
     /// @throw isc::dhcp::NoDatabaseName Mandatory database name not given
     /// @throw isc::dhcp::DbOpenError Error opening the database
     /// @throw isc::dhcp::DbOperationError An operation on the open database has
     ///        failed.
-    PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters);
+    PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters,
+                  DatabaseConnection::Callback db_lost_callback = NULL);
 
     /// @brief Destructor (closes database)
     virtual ~PgSqlLeaseMgr();
@@ -65,8 +69,7 @@ public:
 
     /// @brief Adds an IPv4 lease
     ///
-    /// @param lease lease to be added
+    /// @param lease lease to be added ///
-    ///
     /// @result true if the lease was added, false if not (because a lease
     ///         with the same address was already there).
     ///

src/lib/dhcpsrv/tests/database_connection_unittest.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2015-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -9,8 +9,29 @@
 #include <dhcpsrv/database_connection.h>
 #include <gtest/gtest.h>
 
+#include <boost/bind.hpp>
+
 using namespace isc::dhcp;
 
+class DatabaseConnectionCallbackTest : public ::testing::Test {
+public:
+    DatabaseConnectionCallbackTest()
+        : db_reconnect_ctl_(0) {
+    }
+
+    bool dbLostCallback(ReconnectCtlPtr db_conn_retry) {
+        if (!db_conn_retry) {
+            isc_throw(isc::BadValue, "db_conn_retry should not null");
+        }
+
+        db_reconnect_ctl_ = db_conn_retry;
+        return (true);
+    }
+
+    ReconnectCtlPtr db_reconnect_ctl_;
+};
+
+
 
 /// @brief getParameter test
 ///
@@ -28,6 +49,43 @@ TEST(DatabaseConnectionTest, getParameter) {
     EXPECT_THROW(datasrc.getParameter("param3"), isc::BadValue);
 }
 
+TEST_F(DatabaseConnectionCallbackTest, NoDbLostCallback) {
+    DatabaseConnection::ParameterMap pmap;
+    pmap[std::string("max-reconnect-tries")] = std::string("3");
+    pmap[std::string("reconnect-wait-time")] = std::string("60");
+    DatabaseConnection datasrc(pmap);
+
+    bool ret;
+    ASSERT_NO_THROW(ret = datasrc.invokeDbLostCallback());
+    EXPECT_FALSE(ret);
+    EXPECT_FALSE(db_reconnect_ctl_);
+}
+
+TEST_F(DatabaseConnectionCallbackTest, dbLostCallback) {
+    DatabaseConnection::ParameterMap pmap;
+    pmap[std::string("max-reconnect-tries")] = std::string("3");
+    pmap[std::string("reconnect-wait-time")] = std::string("60");
+    DatabaseConnection datasrc(pmap, boost::bind(&DatabaseConnectionCallbackTest
+                                                 ::dbLostCallback, this, _1));
+    bool ret;
+    ASSERT_NO_THROW(ret = datasrc.invokeDbLostCallback());
+    EXPECT_TRUE(ret);
+    ASSERT_TRUE(db_reconnect_ctl_);
+    ASSERT_EQ(3, db_reconnect_ctl_->maxRetries());
+    ASSERT_EQ(3, db_reconnect_ctl_->retriesLeft());
+    EXPECT_EQ(60, db_reconnect_ctl_->retryInterval());
+
+    for (int i = 3; i > 1 ; --i) {
+        ASSERT_EQ(i, db_reconnect_ctl_->retriesLeft());
+        ASSERT_TRUE(db_reconnect_ctl_->checkRetries());
+    }
+
+    EXPECT_FALSE(db_reconnect_ctl_->checkRetries());
+    EXPECT_EQ(0, db_reconnect_ctl_->retriesLeft());
+    EXPECT_EQ(3, db_reconnect_ctl_->maxRetries());
+}
+
+
 // This test checks that a database access string can be parsed correctly.
 TEST(DatabaseConnectionTest, parse) {
 

src/lib/dhcpsrv/tests/pgsql_host_data_source_unittest.cc

@@ -216,6 +216,35 @@ TEST(PgSqlHostDataSource, OpenDatabase) {
     destroyPgSQLSchema();
 }
 
+/// @brief Flag used to detect calls to db_lost_callback function
+bool callback_called = false;
+
+/// @brief Callback function used in open database testing
+bool db_lost_callback(ReconnectCtlPtr /* db_conn_retry */) {
+    return (callback_called = true);
+}
+
+/// @brief Make sure open failures do NOT invoke db lost callback
+/// The db lost callback should only be invoked after succesfully
+/// opening the DB and then subsequently losing it. Failing to
+/// open should be handled directly by the application layer.
+/// There is simply no good way to break the connection in a
+/// unit test environment.  So testing the callback invocation
+/// in a unit test is next to impossible. That has to be done
+/// as a system test.
+TEST(PgSqlHostDataSource, NoCallbackOnOpenFail) {
+    // Schema needs to be created for the test to work.
+    destroyPgSQLSchema();
+    createPgSQLSchema();
+
+    callback_called = false;
+    EXPECT_THROW(HostDataSourceFactory::create(connectionString(
+        PGSQL_VALID_TYPE, VALID_NAME, INVALID_HOST, VALID_USER, VALID_PASSWORD),
+        db_lost_callback), DbOpenError);
+
+    EXPECT_FALSE(callback_called);
+    destroyPgSQLSchema();
+}
 
 // This test verifies that database backend can operate in Read-Only mode.
 TEST_F(PgSqlHostDataSourceTest, testReadOnlyDatabase) {

src/lib/dhcpsrv/tests/pgsql_lease_mgr_unittest.cc

@@ -172,6 +172,37 @@ TEST(PgSqlOpenTest, OpenDatabase) {
     destroyPgSQLSchema(true);
 }
 
+/// @brief Flag used to detect calls to db_lost_callback function
+bool callback_called = false;
+
+/// @brief Callback function used in open database testing
+bool db_lost_callback(ReconnectCtlPtr /* db_conn_retry */) {
+    return (callback_called = true);
+}
+
+/// @brief Make sure open failures do NOT invoke db lost callback
+/// The db lost callback should only be invoked after succesfully
+/// opening the DB and then subsequently losing it. Failing to
+/// open should be handled directly by the application layer.
+/// There is simply no good way to break the connection in a
+/// unit test environment.  So testing the callback invocation
+/// in a unit test is next to impossible. That has to be done
+/// as a system test.
+TEST(PgSqlOpenTest, NoCallbackOnOpenFail) {
+    // Schema needs to be created for the test to work.
+    destroyPgSQLSchema();
+    createPgSQLSchema();
+
+    callback_called = false;
+    EXPECT_THROW(LeaseMgrFactory::create(connectionString(
+        PGSQL_VALID_TYPE, VALID_NAME, INVALID_HOST, VALID_USER, VALID_PASSWORD),
+        db_lost_callback),
+        DbOpenError);
+    EXPECT_FALSE(callback_called);
+
+    destroyPgSQLSchema();
+}
+
 /// @brief Check the getType() method
 ///
 /// getType() returns a string giving the type of the backend, which should