[4294] Memfile and MySql now support recalulating IPv4 lease statistics

src/lib/dhcpsrv/cfg_subnets4.cc
    CfgSubnets4::removeStatistics()
    - added removal of all lease statistics per subnet, and global declined address
    stats

    CfgSubnets4::updateStatistics()
    - added call to LeaseMgr::recountAddressStats4

src/lib/dhcpsrv/lease.cc
src/lib/dhcpsrv/lease.h
    Replaces lease state constants with LeaseState enumeration.

src/lib/dhcpsrv/lease_mgr.cc
src/lib/dhcpsrv/lease_mgr.h
    struct AddressStatsRow4 - contains the content of one row of the IPv4
    lease statistical data result set

    class AddressStatsQuery4 - base class for constructing the IPv4
    lease statistical data result set for an IPv4 lease storage

    LeaseMgr::recountAddressStats4() -  new method which recalculates
    per-subnet and global stats for IPv4 leases

    LeaseMgr::startAddressStatsQuery4() - new virtual method that fetches
    the IPv4 lease statistical data result set

src/lib/dhcpsrv/lease_mgr_factory.h
src/lib/dhcpsrv/lease_mgr_factory.cc
    LeaseMgrFactory::haveInstance() - new static method which indicates
    whether or not the lease manager singleton exists

src/lib/dhcpsrv/memfile_lease_mgr.h
src/lib/dhcpsrv/memfile_lease_mgr.cc
    MemfileAddressStatsQuery4 - Derivation of AddressStatsQuery4, it constructs
    the IPv4 lease statistical data by iterating over IPv4 lease storage

    Memfile_LeaseMgr::startAddressStatsQuery4() - new virtual method which
    creates, starts, and returns a MemfileAddressStatsQuery4

src/lib/dhcpsrv/memfile_lease_storage.h
    Added an a per subnet_ID index to IPv4 storage

src/lib/dhcpsrv/mysql_lease_mgr.h
src/lib/dhcpsrv/mysql_lease_mgr.cc
    - Added RECOUNT_LEASE4_STATS query

    MySqlAddressStatsQuery4 Derivation of AddressStatsQuery4, it constructs
    the IPv4 lease statistical data by executing RECOUNT_LEASE4_STATS

    MySqlLeaseMgr::startAddressStatsQuery4() - new virtual method which
    creates, starts, and returns a MySqlAddressStatsQuery4

src/lib/dhcpsrv/tests/alloc_engine_utils.cc
    AllocEngine6Test::AllocEngine6Test()
    AllocEngine4Test::AllocEngine4Test()
    - moved lease mgr create up above configuration commit

src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc
     ~CfgMySQLDbAccessTest() - added destruction of lease manager singleton,
    otherwise subsequent tests can fail

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h
src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc
    GenericLeaseMgrTest::checkStat() - new method for comparing a stat
    GenericLeaseMgrTest::checkAddressStats4() - new method for comparing a list
    of stats
    GenericLeaseMgrTest::makeLease4() - new method for making a minimal lease
    GenericLeaseMgrTest::testRecountAddressStats4() - new method which tests
    a lease manager's ability to recalculate the IPv4 lease statistics

src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc
    TEST_F(MemfileLeaseMgrTest, recountAddressStats4) - new test which tests
    Memfile_LeaseMgr's ability to recalculate IPv4 lease statistics

src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc
    TEST_F(MySqlLeaseMgrTest, recountAddressStats4) - new test which tests
    MySqlLeaseMgr's ability to recalculate IPv4 lease statistics

src/lib/dhcpsrv/cfg_subnets4.cc

@@ -8,6 +8,7 @@
 #include <dhcp/iface_mgr.h>
 #include <dhcpsrv/cfg_subnets4.h>
 #include <dhcpsrv/dhcpsrv_log.h>
+#include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/subnet_id.h>
 #include <dhcpsrv/addr_utilities.h>
 #include <asiolink/io_address.h>
@@ -232,18 +233,21 @@ CfgSubnets4::removeStatistics() {
     using namespace isc::stats;
 
     // For each v4 subnet currently configured, remove the statistic.
-    /// @todo: May move this to CfgSubnets4 class if there will be more
+    StatsMgr& stats_mgr = StatsMgr::instance();
-    /// statistics here.
     for (Subnet4Collection::const_iterator subnet4 = subnets_.begin();
          subnet4 != subnets_.end(); ++subnet4) {
+        SubnetID subnet_id = (*subnet4)->getID();
+        stats_mgr.del(StatsMgr::generateName("subnet", subnet_id,
+                                             "total-addresses"));
 
-        StatsMgr::instance().del(StatsMgr::generateName("subnet",
+        stats_mgr.del(StatsMgr::generateName("subnet", subnet_id,
-                                                        (*subnet4)->getID(),
+                                             "assigned-addresses"));
-                                                        "total-addresses"));
 
-        StatsMgr::instance().del(StatsMgr::generateName("subnet",
+        stats_mgr.del(StatsMgr::generateName("subnet", subnet_id,
-                                                        (*subnet4)->getID(),
+                                             "declined-addresses"));
-                                                        "assigned-addresses"));
+
+        stats_mgr.del(StatsMgr::generateName("subnet", subnet_id,
+                                             "declined-reclaimed-addresses"));
     }
 }
 
@@ -251,14 +255,21 @@ void
 CfgSubnets4::updateStatistics() {
     using namespace isc::stats;
 
-    /// @todo: May move this to CfgSubnets4 class if there will be more
+    StatsMgr& stats_mgr = StatsMgr::instance();
-    /// statistics here.
+    for (Subnet4Collection::const_iterator subnet4 = subnets_.begin();
-    for (Subnet4Collection::const_iterator subnet = subnets_.begin();
+         subnet4 != subnets_.end(); ++subnet4) {
-         subnet != subnets_.end(); ++subnet) {
+        SubnetID subnet_id = (*subnet4)->getID();
 
-        StatsMgr::instance().setValue(
+        stats_mgr.setValue(StatsMgr::
-            StatsMgr::generateName("subnet", (*subnet)->getID(), "total-addresses"),
+                           generateName("subnet", subnet_id, "total-addresses"),
-            static_cast<int64_t>((*subnet)->getPoolCapacity(Lease::TYPE_V4)));
+                                        static_cast<int64_t>
+                                        ((*subnet4)->getPoolCapacity(Lease::
+                                                                     TYPE_V4)));
+    }
+
+    // If we have subnets and a lease mgr, recount the least statistics
+    if (subnets_.begin() != subnets_.end() && LeaseMgrFactory::haveInstance()) {
+            LeaseMgrFactory::instance().recountAddressStats4();
     }
 }
 

src/lib/dhcpsrv/dhcpsrv_messages.mes

@@ -923,3 +923,6 @@ lease from the Cassandra database for the specified address.
 % DHCPSRV_CQL_UPDATE_ADDR6 updating IPv6 lease for address %1
 A debug message issued when the server is attempting to update IPv6
 lease from the Cassandra database for the specified address.
+
+% TOMS_UTILITY_MESSAGE %1
+Handy log message that should be deleted

src/lib/dhcpsrv/lease.cc

@@ -15,10 +15,6 @@ using namespace std;
 namespace isc {
 namespace dhcp {
 
-const uint32_t Lease::STATE_DEFAULT = 0x0;
-const uint32_t Lease::STATE_DECLINED = 0x1;
-const uint32_t Lease::STATE_EXPIRED_RECLAIMED = 0x2;
-
 Lease::Lease(const isc::asiolink::IOAddress& addr, uint32_t t1, uint32_t t2,
              uint32_t valid_lft, SubnetID subnet_id, time_t cltt,
              const bool fqdn_fwd, const bool fqdn_rev,

src/lib/dhcpsrv/lease.h

@@ -40,18 +40,18 @@ struct Lease {
     /// @return text decription
     static std::string typeToText(Type type);
 
-    /// @name Common lease states constants.
+    /// @name Enumeration of lease states
     //@{
-    ///
+    typedef enum {
-    /// @brief A lease in the default state.
+        /// @brief A lease in the default (assigned) state.
-    static const uint32_t STATE_DEFAULT;
+        STATE_DEFAULT,
-
+        /// @brief Declined lease.
-    /// @brief Declined lease.
+        STATE_DECLINED,
-    static const uint32_t STATE_DECLINED;
+        /// @brief Expired and reclaimed lease.
-
+        STATE_EXPIRED_RECLAIMED,
-    /// @brief Expired and reclaimed lease.
+        /// @brief The number of defined lease states.
-    static const uint32_t STATE_EXPIRED_RECLAIMED;
+        NUM_LEASE_STATES
-
+    } LeaseState;
     //@}
 
     /// @brief Returns name(s) of the basic lease state(s).

src/lib/dhcpsrv/lease_mgr.cc

@@ -6,8 +6,11 @@
 
 #include <config.h>
 
+#include <dhcpsrv/cfgmgr.h>
+#include <dhcpsrv/dhcpsrv_log.h>
 #include <dhcpsrv/lease_mgr.h>
 #include <exceptions/exceptions.h>
+#include <stats/stats_mgr.h>
 
 #include <boost/foreach.hpp>
 #include <boost/algorithm/string.hpp>
@@ -44,6 +47,82 @@ LeaseMgr::getLease6(Lease::Type type, const DUID& duid,
     return (*col.begin());
 }
 
+void 
+LeaseMgr::recountAddressStats4() {
+    using namespace stats;
+
+    StatsMgr& stats_mgr = StatsMgr::instance();
+
+    AddressStatsQuery4Ptr query = startAddressStatsQuery4();
+    if (!query) {
+        /// NULL means not backend does not support recounting.
+        return;
+    }
+
+    // Zero out the global stats. (Ok, so currently there's only one
+    // that should be cleared.  "reclaimed-declined-addresses" never
+    // gets zeroed. @todo discuss with Tomek the rational of not
+    // clearing it when we clear the rest.
+    int64_t zero = 0;
+    stats_mgr.setValue("declined-addresses", zero);
+    stats_mgr.setValue("declined-reclaimed-addresses", zero);
+
+    // Clear subnet level stats.  This ensures we don't end up with corner
+    // cases that leave stale values in place. 
+    const Subnet4Collection* subnets = 
+        CfgMgr::instance().getCurrentCfg()->getCfgSubnets4()->getAll();
+
+    for (Subnet4Collection::const_iterator subnet = subnets->begin();
+         subnet != subnets->end(); ++subnet) {
+        SubnetID subnet_id = (*subnet)->getID();
+        stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id,
+                                                  "assigned-addresses"),
+                           zero);
+
+        stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id,
+                                                  "declined-addresses"),
+                           zero);
+        stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id,
+                                                  "declined-reclaimed-addresses"),
+                           zero);
+    }
+   
+    // Get counts per state per subnet. Iterate over the result set
+    // updating the subnet and global values. 
+    AddressStatsRow4 row;
+    while (query->getNextRow(row)) {
+        switch(row.lease_state_) {
+            case Lease::STATE_DEFAULT:
+                // Set subnet level value.
+                stats_mgr.setValue(StatsMgr::generateName("subnet", 
+                                                          row.subnet_id_,
+                                                          "assigned-addresses"),
+                                   row.state_count_);
+                break;
+
+            case Lease::STATE_DECLINED:
+                // Set subnet level value.
+                stats_mgr.setValue(StatsMgr::generateName("subnet",
+                                                          row.subnet_id_,
+                                                          "declined-addresses"),
+                                   row.state_count_);
+
+                // Add to the global value.
+                stats_mgr.addValue("declined-addresses", row.state_count_);
+                break;
+
+            default:
+                // Not one we're tracking.
+                break;
+        }
+    }
+}
+
+AddressStatsQuery4Ptr
+LeaseMgr::startAddressStatsQuery4() {
+    return(AddressStatsQuery4Ptr());
+}
+
 std::string
 LeaseMgr::getDBVersion() {
     isc_throw(NotImplemented, "LeaseMgr::getDBVersion() called");

src/lib/dhcpsrv/lease_mgr.h

@@ -146,6 +146,70 @@ public:
     virtual ~SqlExchange() {};
     ExchangeColumnInfoContainer parameters_;   ///< Column names and types
 };
+
+/// @brief Contains a single row of IPv4 lease statistical data
+///
+/// The contents of the row consist of a subnet ID, a lease state,
+/// and the number of leases in that state for that subnet ID.
+struct AddressStatsRow4 {
+    /// @brief Default constructor
+    AddressStatsRow4() : 
+        subnet_id_(0), lease_state_(Lease::STATE_DEFAULT), state_count_(0) {
+    }
+
+    /// @brief Constructor
+    ///
+    /// @param subnet_id The subnet id to which this data applies
+    /// @param lease_state The lease state counted
+    /// @param state_count The count of leases in the lease state
+    AddressStatsRow4(const SubnetID& subnet_id,
+                     const Lease::LeaseState& lease_state,
+                     const int64_t state_count) 
+        : subnet_id_(subnet_id), lease_state_(lease_state), 
+          state_count_(state_count) {
+    }
+
+    /// @brief The subnet ID to which this data applies
+    SubnetID subnet_id_;
+    /// @brief The lease_state to which the count applies
+    uint32_t lease_state_;
+    /// @brief state_count The count of leases in the lease state
+    int64_t state_count_;
+};
+
+/// @brief Base class for fulfilling IPv4 statistical lease data query
+///
+/// LeaseMgr derivations implement this class such that it provides
+/// upto date IPv4 statistical lease data organized as rows of
+/// AddressStatsRow4 instances.  The rows must be accessible in
+/// ascending order by subnet id.
+class AddressStatsQuery4 {
+public:
+    /// @brief Default constructor
+    AddressStatsQuery4() {};
+
+    /// @brief virtual destructor 
+    virtual ~AddressStatsQuery4() {};
+
+    /// @brief Executes the query
+    ///
+    /// This method should conduct whatever steps are required to 
+    /// calculate the IPv4 lease statistical data by examining the
+    /// IPv4 lease data and making that results available row by row.
+    virtual void start() {};
+
+    /// @brief Fetches the next row of data
+    ///
+    /// @param[out] row Storage into which the row is fetched
+    ///
+    /// @return True if a row was fetched, false if there are no
+    /// more rows.
+    virtual bool getNextRow(AddressStatsRow4& row) { return(false); };
+};
+
+/// @brief Defines a pointer to an AddressStatsQuery4.
+typedef boost::shared_ptr<AddressStatsQuery4> AddressStatsQuery4Ptr;
+
 /// @brief Abstract Lease Manager
 ///
 /// This is an abstract API for lease database backends. It provides unified
@@ -397,6 +461,36 @@ public:
     /// @return Number of leases deleted.
     virtual uint64_t deleteExpiredReclaimedLeases6(const uint32_t secs) = 0;
 
+    /// @brief Recalculates per-subnet and global stats for IPv4 leases
+    ///
+    /// This method recalculates the following statistics: 
+    /// per-subnet:
+    /// - assigned-addresses
+    /// - declined-addresses
+    /// - declined-reclaimed-addresses (reset to zero)
+    /// global:
+    /// - declined-addresses
+    /// - declined-reclaimed-addresses (reset to zero)
+    ///
+    /// It invokes the virtual method, startAddressStatsQuery4(), which
+    /// returns an instance of an AddressStats4Qry.  The query
+    /// query contains a "result set"  where each row is an AddressStatRow4
+    /// that contains a subnet id, a lease state, the number of leases in that
+    /// state and is ordered by subnet id.  The method iterates over the 
+    /// result set rows, setting the appropriate statistic per subnet and
+    /// adding to the approporate global statistic. 
+    void recountAddressStats4();
+
+    /// @brief Virtual method which creates and runs the IPv4 lease stats query 
+    ///
+    /// LeaseMgr derivations implement this method such that it creates and
+    /// returns an instance of an AddressStatsQuery whose result set has been 
+    /// populated with upto date IPv4 lease statistical data.  Each row of the 
+    /// result set is an AddressStatRow4 which ordered ascending by subnet ID.  
+    ///
+    /// @return A populated AddressStatsQuery4
+    virtual AddressStatsQuery4Ptr startAddressStatsQuery4();
+
     /// @brief Return backend type
     ///
     /// Returns the type of the backend (e.g. "mysql", "memfile" etc.)

src/lib/dhcpsrv/lease_mgr_factory.cc

@@ -101,6 +101,11 @@ LeaseMgrFactory::destroy() {
     getLeaseMgrPtr().reset();
 }
 
+bool 
+LeaseMgrFactory::haveInstance() {
+    return (getLeaseMgrPtr().get());
+}
+
 LeaseMgr&
 LeaseMgrFactory::instance() {
     LeaseMgr* lmptr = getLeaseMgrPtr().get();

src/lib/dhcpsrv/lease_mgr_factory.h

@@ -85,7 +85,10 @@ public:
     ///        create() to create one before calling this method.
     static LeaseMgr& instance();
 
-
+    /// @brief Indicates if the lease manager has been instantiated.
+    ///
+    /// @return True if the lease manager instance exists, false otherwise.
+    static bool haveInstance();
 
 private:
     /// @brief Hold pointer to lease manager

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -256,6 +256,143 @@ LFCSetup::getExitStatus() const {
     return (process_->getExitStatus(pid_));
 }
 
+
+/// @brief Memfile derivation of the IPv4 statistical lease data query
+///
+/// This class is used to recalculate IPv4 lease statistics for Memfile
+/// lease storage.  It does so by iterating over the given storage,
+/// accumulating counts of leases in each of the monitored lease states
+/// for each subnet and storing these counts in an internal collection.
+/// The populated result set will contain one entry per monitored state
+/// per subnet.
+///
+class MemfileAddressStatsQuery4 : public AddressStatsQuery4 {
+public:
+    /// @brief Constructor
+    ///
+    /// @param storage4 A pointer to the v4 lease storage to be counted
+    MemfileAddressStatsQuery4(Lease4Storage& storage4);
+
+    /// @brief Destructor
+    virtual ~MemfileAddressStatsQuery4() {};
+
+    /// @brief Creates the IPv4 lease statistical data result set
+    ///
+    /// The result is populated by iterating over the IPv4 leases in storage,
+    /// in ascending order by subnet ID, accumulating the lease state counts.
+    /// At the completion of all entries for a given subnet, the counts are 
+    /// used to create AddressStatsRow4 instances which are appended to an 
+    /// internal vector.  The process results in a vector containing one entry
+    /// per state per subnet.
+    ///
+    /// Currently the states counted are:
+    ///
+    /// - Lease::STATE_DEFAULT (i.e. assigned) 
+    /// - Lease::STATE_DECLINED 
+    virtual void start();
+
+    /// @brief Fetches the next row in the result set
+    ///
+    /// Once the internal result set has been populated by invoking the
+    /// the start() method, this method is used to iterate over the
+    /// result set rows.  Once the last row has been fetched, subsequent
+    /// calls will return false. 
+    /// @param row Storage for the fetched row
+    ///
+    /// @return True if the fetch succeeded, false if there are no more
+    /// rows to fetch.
+    virtual bool getNextRow(AddressStatsRow4& row);
+
+    /// @brief Returns the number of rows in the result set
+    /// @todo, should this be a virtual member of the base class?
+    int getRowCount();
+
+private:
+    /// @brief The Memfile storage containing the IPv4 leases to analyze
+    Lease4Storage& storage4_;
+
+    /// @brief A vector containing the "result set" 
+    std::vector<AddressStatsRow4> rows_;
+
+    /// @brief An iterator for accessing the next row within the result set
+    std::vector<AddressStatsRow4>::iterator next_pos_;
+};
+
+MemfileAddressStatsQuery4::MemfileAddressStatsQuery4(Lease4Storage& storage4) 
+    : storage4_(storage4), rows_(0), next_pos_(rows_.end()) {};
+
+void 
+MemfileAddressStatsQuery4::start() {
+    // Get the subnet_id index
+    const Lease4StorageSubnetIdIndex& idx = storage4_.get<SubnetIdIndexTag>();
+
+    // Iterate over the leases in order by subnet, accumulating per
+    // subnet counts for each state of interest.  As we finish each
+    // subnet, add the appropriate rows to our result set.
+    SubnetID cur_id = 0;
+    int64_t assigned = 0;
+    int64_t declined = 0;
+    for(Lease4StorageSubnetIdIndex::const_iterator lease = idx.begin();
+        lease != idx.end(); ++lease) {
+
+        // If we've hit the next subnet, add rows for the current subnet
+        // and wipe the accumulators
+        if ((*lease)->subnet_id_ > cur_id) {
+            if (cur_id > 0) {
+                rows_.push_back(AddressStatsRow4(cur_id,Lease::STATE_DEFAULT, 
+                                                 assigned));
+                assigned = 0;
+                rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, 
+                                                 declined));
+                declined = 0;
+            }
+
+            // Update current subnet id
+            cur_id = (*lease)->subnet_id_;
+        }
+
+        // Bump the appropriate accumulator
+        switch ((*lease)->state_) {
+        case Lease::STATE_DEFAULT:
+            ++assigned;
+            break;
+        case Lease::STATE_DECLINED:
+            ++declined;
+            break;
+        default:
+            // Not one we're tracking.
+            break;
+        }
+    }
+
+    // Make the rows for last subnet, unless there were no rows
+    if (idx.begin() != idx.end()) {
+        rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DEFAULT,
+                                         assigned));
+        rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, 
+                                         declined));
+    }
+
+    // Set the next row position to the beginning of the rows.
+    next_pos_ = rows_.begin();
+}
+
+bool 
+MemfileAddressStatsQuery4::getNextRow(AddressStatsRow4& row) { 
+    if (next_pos_ == rows_.end()) {
+        return (false);
+    }
+
+    row = *next_pos_;
+    ++next_pos_;
+    return (true);
+}
+
+int 
+MemfileAddressStatsQuery4::getRowCount() {
+    return (rows_.size());
+}
+
 // Explicit definition of class static constants.  Values are given in the
 // declaration so they're not needed here.
 const int Memfile_LeaseMgr::MAJOR_VERSION;
@@ -299,6 +436,7 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& param
         }
         lfcSetup(conversion_needed);
     }
+
 }
 
 Memfile_LeaseMgr::~Memfile_LeaseMgr() {
@@ -1048,5 +1186,12 @@ void Memfile_LeaseMgr::lfcExecute(boost::shared_ptr<LeaseFileType>& lease_file)
     }
 }
 
+AddressStatsQuery4Ptr
+Memfile_LeaseMgr::startAddressStatsQuery4() {
+    AddressStatsQuery4Ptr query(new MemfileAddressStatsQuery4(storage4_));
+    query->start();
+    return(query);
+}
+
 } // end of namespace isc::dhcp
 } // end of namespace isc

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -92,6 +92,7 @@ public:
 
     /// @}
 
+
     /// @brief Specifies universe (V4, V6)
     ///
     /// This enumeration is used by various functions in Memfile %Lease Manager,
@@ -594,6 +595,14 @@ public:
     int getLFCExitStatus() const;
     //@}
 
+    /// @brief Creates and runs the IPv4 lease stats query 
+    ///
+    /// It creates an instance of a MemfileAddressStatsQuery4 and then 
+    /// invokes it's start method in which the query constructs its 
+    /// statistical data result set.  The query object is then returned.
+    /// 
+    /// @return The populated query as a pointer to an AddressStatsQuery4
+    virtual AddressStatsQuery4Ptr startAddressStatsQuery4();
 
     /// @name Protected methods used for %Lease File Cleanup.
     /// The following methods are protected so as they can be accessed and

src/lib/dhcpsrv/memfile_lease_storage.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -27,6 +27,9 @@ namespace dhcp {
 /// @brief Tag for indexes by address.
 struct AddressIndexTag { };
 
+/// @brief Tag for indexes by subnet id.
+struct SubnetIdIndexTag { };
+
 /// @brief Tag for indexes by DUID, IAID, lease type tuple.
 struct DuidIaidTypeIndexTag { };
 
@@ -135,6 +138,15 @@ typedef boost::multi_index_container<
             boost::multi_index::member<Lease, isc::asiolink::IOAddress, &Lease::addr_>
         >,
 
+        boost::multi_index::ordered_non_unique<
+            boost::multi_index::tag<SubnetIdIndexTag>,
+            // The subnet id is held in the subnet_id_ member of Lease4
+            // class. Note that the subnet_id_ is defined in the base
+            // class (Lease) so we have to point to this class rather
+            // than derived class: Lease4.
+            boost::multi_index::member<Lease, SubnetID, &Lease::subnet_id_>
+        >,
+
         // Specification of the second index starts here.
         boost::multi_index::ordered_non_unique<
             boost::multi_index::tag<HWAddressSubnetIdIndexTag>,
@@ -232,6 +244,9 @@ typedef Lease6Storage::index<ExpirationIndexTag>::type Lease6StorageExpirationIn
 /// @brief DHCPv4 lease storage index by address.
 typedef Lease4Storage::index<AddressIndexTag>::type Lease4StorageAddressIndex;
 
+/// @brief DHCPv4 lease storage index by subnet id.
+typedef Lease4Storage::index<SubnetIdIndexTag>::type Lease4StorageSubnetIdIndex;
+
 /// @brief DHCPv4 lease storage index by exiration time.
 typedef Lease4Storage::index<ExpirationIndexTag>::type Lease4StorageExpirationIndex;
 

src/lib/dhcpsrv/mysql_lease_mgr.cc

@@ -205,6 +205,9 @@ TaggedStatement tagged_statements[] = {
                         "hostname = ?, hwaddr = ?, hwtype = ?, hwaddr_source = ?, "
                         "state = ? "
                             "WHERE address = ?"},
+    {MySqlLeaseMgr::RECOUNT_LEASE4_STATS,
+                    "SELECT subnet_id, state, count(state) as state_count "
+                        "FROM lease4 group by subnet_id, state"},
     // End of list sentinel
     {MySqlLeaseMgr::NUM_STATEMENTS, NULL}
 };
@@ -1214,6 +1217,143 @@ private:
     uint32_t        state_;             ///< Lease state.
 };
 
+/// @brief MySql derivation of the IPv4 statistical lease data query
+///
+/// This class is used to recalculate IPv4 lease statistics for MySQL
+/// lease storage.  It does so by executing a query which returns a result
+/// containining contain one row per monitored state per subnet, ordered
+/// by subnet id in ascending order.
+///
+class MySqlAddressStatsQuery4 : public AddressStatsQuery4 {
+public:
+    /// @brief Constructor
+    ///
+    /// @param conn A open connection to the database housing the lease data
+    MySqlAddressStatsQuery4(MySqlConnection& conn);
+
+    /// @brief Destructor
+    virtual ~MySqlAddressStatsQuery4();
+
+    /// @brief Creates the IPv4 lease statistical data result set
+    ///
+    /// The result set is populated by executing an SQL query against the
+    /// lease4 table which sums the leases per lease state per subnet id.
+    /// The query used is the prepared statement identified by
+    /// MySqlLeaseMgr::RECOUNT_LEASE4_STATS.  This method creates the binds
+    /// the statement to the output bind array  and then executes the
+    /// statement.
+    void start();
+
+    /// @brief Fetches the next row in the result set
+    ///
+    /// Once the internal result set has been populated by invoking the
+    /// the start() method, this method is used to iterate over the
+    /// result set rows. Once the last row has been fetched, subsequent
+    /// calls will return false.
+    ///
+    /// @param row Storage for the fetched row
+    ///
+    /// @return True if the fetch succeeded, false if there are no more
+    /// rows to fetch.
+    bool getNextRow(AddressStatsRow4& row);
+
+private:
+
+    /// @brief Analyzes the given statement outcome status
+    ///
+    /// Wrapper method around the MySqlConnection:checkError() that is
+    /// used to generate the appropriate exception if the status indicates
+    /// an error.
+    ////
+    /// a DbOperation error
+    /// @param status The MySQL statement execution outcome status
+    /// @param what invocation context message which will be included in
+    /// any exception
+    void checkError(int status, const char* what) const;
+
+    /// @brief Database connection to use to execute the query
+    MySqlConnection& conn_;
+
+    /// @brief The query's prepared statement
+    MYSQL_STMT *statement_;
+
+    /// @brief Bind array used to store the query result set;
+    std::vector<MYSQL_BIND> bind_;
+
+    /// @brief Member struct that is bound to the statement;
+    AddressStatsRow4 stat_row_;
+};
+
+MySqlAddressStatsQuery4::MySqlAddressStatsQuery4(MySqlConnection& conn)
+    : conn_(conn), statement_(conn_.statements_[MySqlLeaseMgr
+                                                ::RECOUNT_LEASE4_STATS]),
+      bind_(3) {
+}
+
+MySqlAddressStatsQuery4::~MySqlAddressStatsQuery4() {
+    (void) mysql_stmt_free_result(statement_);
+}
+
+
+void
+MySqlAddressStatsQuery4::start() {
+    // subnet_id: unsigned int
+    bind_[0].buffer_type = MYSQL_TYPE_LONG;
+    bind_[0].buffer = reinterpret_cast<char*>(&stat_row_.subnet_id_);
+    bind_[0].is_unsigned = MLM_TRUE;
+
+    // state:  uint32_t
+    bind_[1].buffer_type = MYSQL_TYPE_LONG;
+    bind_[1].buffer = reinterpret_cast<char*>(&stat_row_.lease_state_);
+    bind_[1].is_unsigned = MLM_TRUE;
+
+    // state_count_: uint32_t
+    bind_[2].buffer_type = MYSQL_TYPE_LONG;
+    bind_[2].buffer = reinterpret_cast<char*>(&stat_row_.state_count_);
+    bind_[2].is_unsigned = MLM_TRUE;
+
+    // Set up the MYSQL_BIND array for the data being returned
+    // and bind it to the statement.
+    int status = mysql_stmt_bind_result(statement_, &bind_[0]);
+    checkError(status, "RECOUNT_LEASE4_STATS: outbound binding failed");
+
+    // Execute the statement
+    status = mysql_stmt_execute(statement_);
+    checkError(status, "RECOUNT_LEASE4_STATS: unable to execute");
+
+    // Ensure that all the lease information is retrieved in one go to avoid
+    // overhead of going back and forth between client and server.
+    status = mysql_stmt_store_result(statement_);
+    checkError(status, "RECOUNT_LEASE4_STATS: results storage setup failed");
+}
+
+bool
+MySqlAddressStatsQuery4::getNextRow(AddressStatsRow4& row) {
+    bool have_row = false;
+    int status = mysql_stmt_fetch(statement_);
+    if (status == MLM_MYSQL_FETCH_SUCCESS) {
+        row = stat_row_;
+        have_row = true;
+    } else if (status != MYSQL_NO_DATA) {
+        checkError(status, "RECOUNT_LEASE4_STATS: getNextRow failed");
+    }
+
+    return (have_row);
+}
+
+void
+MySqlAddressStatsQuery4::checkError(int status, const char* what) const {
+    conn_.checkError(status, MySqlLeaseMgr::RECOUNT_LEASE4_STATS, what);
+}
+
+AddressStatsQuery4Ptr
+MySqlLeaseMgr::startAddressStatsQuery4() {
+    AddressStatsQuery4Ptr query(new MySqlAddressStatsQuery4(conn_));
+    query->start();
+    return(query);
+}
+
+
 
 
 
@@ -2025,7 +2165,6 @@ MySqlLeaseMgr::getVersion() const {
     return (std::make_pair(major, minor));
 }
 
-
 void
 MySqlLeaseMgr::commit() {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MYSQL_COMMIT);

src/lib/dhcpsrv/mysql_lease_mgr.h

@@ -25,7 +25,6 @@ namespace dhcp {
 class MySqlLease4Exchange;
 class MySqlLease6Exchange;
 
-
 /// @brief MySQL Lease Manager
 ///
 /// This class provides the \ref isc::dhcp::LeaseMgr interface to the MySQL
@@ -410,6 +409,7 @@ public:
         INSERT_LEASE6,               // Add entry to lease6 table
         UPDATE_LEASE4,               // Update a Lease4 entry
         UPDATE_LEASE6,               // Update a Lease6 entry
+        RECOUNT_LEASE4_STATS,        // Fetches address statisics
         NUM_STATEMENTS               // Number of statements
     };
 
@@ -590,6 +590,15 @@ private:
     uint64_t deleteExpiredReclaimedLeasesCommon(const uint32_t secs,
                                                 StatementIndex statement_index);
 
+    /// @brief Creates and runs the IPv4 lease stats query
+    ///
+    /// It creates an instance of a MySqlAddressStatsQuery4 and then
+    /// invokes its start method, which fetches its statistical data
+    /// result set by executing the RECOUNT_LEASE_STATS4 query.
+    /// The query object is then returned.
+    /// 
+    /// @return The populated query as a pointer to an AddressStatsQuery4
+    virtual AddressStatsQuery4Ptr startAddressStatsQuery4();
 
     /// @brief Check Error and Throw Exception
     ///

src/lib/dhcpsrv/tests/alloc_engine_utils.cc

@@ -123,6 +123,8 @@ AllocEngine4Test::generateDeclinedLease(const std::string& addr,
 AllocEngine6Test::AllocEngine6Test() {
     CfgMgr::instance().clear();
 
+    factory_.create("type=memfile universe=6 persist=false");
+
     duid_ = DuidPtr(new DUID(std::vector<uint8_t>(8, 0x42)));
     iaid_ = 42;
 
@@ -141,7 +143,6 @@ AllocEngine6Test::AllocEngine6Test() {
 
     initFqdn("", false, false);
 
-    factory_.create("type=memfile universe=6 persist=false");
 }
 
 void
@@ -525,6 +526,9 @@ AllocEngine4Test::initSubnet(const asiolink::IOAddress& pool_start,
 }
 
 AllocEngine4Test::AllocEngine4Test() {
+
+    factory_.create("type=memfile universe=4 persist=false");
+
     // Create fresh instance of the HostMgr, and drop any previous HostMgr state.
     HostMgr::instance().create();
 
@@ -548,7 +552,6 @@ AllocEngine4Test::AllocEngine4Test() {
     initSubnet(IOAddress("192.0.2.100"), IOAddress("192.0.2.109"));
     cfg_mgr.commit();
 
-    factory_.create("type=memfile universe=4 persist=false");
 
     // Create a default context. Note that remaining parameters must be
     // assigned when needed.

src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc

@@ -88,6 +88,7 @@ public:
     /// @brief Destructor.
     virtual ~CfgMySQLDbAccessTest() {
         destroyMySQLSchema();
+        LeaseMgrFactory::destroy();
     }
 };
 

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc

@@ -5,11 +5,18 @@
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 #include <config.h>
+
+#include <asiolink/io_address.h>
+#include <dhcpsrv/cfgmgr.h>
+#include <dhcpsrv/database_connection.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <dhcpsrv/tests/test_utils.h>
-#include <dhcpsrv/database_connection.h>
+#include <stats/stats_mgr.h>
-#include <asiolink/io_address.h>
+
+#include <boost/foreach.hpp>
+
 #include <gtest/gtest.h>
+
 #include <sstream>
 
 using namespace std;
@@ -57,6 +64,7 @@ GenericLeaseMgrTest::GenericLeaseMgrTest()
         /// a template
         leasetype6_.push_back(LEASETYPE6[i]);
     }
+
 }
 
 GenericLeaseMgrTest::~GenericLeaseMgrTest() {
@@ -2381,6 +2389,169 @@ GenericLeaseMgrTest::testGetDeclinedLeases6() {
     }
 }
 
+void
+GenericLeaseMgrTest::checkStat(const std::string& name,
+                               const int64_t expected_value) {
+    stats::ObservationPtr obs =
+        stats::StatsMgr::instance().getObservation(name);
+
+    ASSERT_TRUE(obs) << " stat: " << name << " not found ";
+    ASSERT_EQ(expected_value, obs->getInteger().first)
+                << " stat: " << name << " value wrong";
+}
+
+void
+GenericLeaseMgrTest::checkAddressStats4(const StatValMapList& expectedStats) {
+    // Global accumulators
+    int64_t declined_addresses = 0;
+    int64_t declined_reclaimed_addresses = 0;
+
+#if 0
+    isc::data::ConstElementPtr allstats = stats::StatsMgr::instance().getAll();
+    std::cout << "ALL: ";
+    allstats->toJSON(std::cout);
+    std::cout << std::endl;
+#endif
+
+    // Iterate over all stats for each subnet
+    for (int subnet_idx = 0; subnet_idx < expectedStats.size(); ++subnet_idx) {
+        BOOST_FOREACH(StatValPair expectedStat, expectedStats[subnet_idx]) {
+            // Verify the per subnet value.
+            checkStat(stats::StatsMgr::generateName("subnet", subnet_idx+1,
+                                                    expectedStat.first),
+                      expectedStat.second);
+
+            // Add the value to globals as needed.
+            if (expectedStat.first == "declined-addresses") {
+                declined_addresses += expectedStat.second;
+            } else if (expectedStat.first == "declined-reclaimed-addresses") {
+                declined_reclaimed_addresses += expectedStat.second;
+            }
+        }
+    }
+
+    // Verify the globals.
+    checkStat("declined-addresses", declined_addresses);
+    checkStat("declined-reclaimed-addresses", declined_reclaimed_addresses);
+}
+
+void
+GenericLeaseMgrTest::makeLease4(const std::string& address,
+                                const SubnetID& subnet_id,
+                                const Lease::LeaseState& state) {
+    Lease4Ptr lease(new Lease4());
+
+    // set the address
+    lease->addr_ = IOAddress(address);
+
+    // make a MAC from the address
+    std::vector<uint8_t> hwaddr = lease->addr_.toBytes();
+    hwaddr.push_back(0);
+    hwaddr.push_back(0);
+
+    lease->hwaddr_.reset(new HWAddr(hwaddr, HTYPE_ETHER));
+    lease->valid_lft_ = 86400;
+    lease->cltt_ = 168256;
+    lease->subnet_id_ = subnet_id;
+    lease->state_ = state;
+    ASSERT_TRUE(lmptr_->addLease(lease));
+}
+
+void
+GenericLeaseMgrTest::testRecountAddressStats4() {
+    using namespace stats;
+
+    StatsMgr::instance().removeAll();
+
+    // create subnets
+    CfgSubnets4Ptr cfg =
+        CfgMgr::instance().getStagingCfg()->getCfgSubnets4();
+
+    // Create 3 subnets.
+    Subnet4Ptr subnet;
+    Pool4Ptr pool;
+
+    subnet.reset(new Subnet4(IOAddress("192.0.1.0"), 24, 1, 2, 3, 1));
+    pool.reset(new Pool4(IOAddress("192.0.1.0"), 24));
+    subnet->addPool(pool);
+    cfg->add(subnet);
+
+    subnet.reset(new Subnet4(IOAddress("192.0.2.0"), 24, 1, 2, 3, 2));
+    pool.reset(new Pool4(IOAddress("192.0.2.0"), 24));
+    subnet->addPool(pool);
+    cfg->add(subnet);
+
+    int num_subnets = 2;
+
+    ASSERT_NO_THROW(CfgMgr::instance().commit());
+
+    // Create the expected stats list.  At this point, the only stat
+    // that should be non-zero is total-addresses.
+    StatValMapList expectedStats(num_subnets);
+    for (int i = 0; i < num_subnets; ++i) {
+        expectedStats[i]["total-addresses"] = 256;
+        expectedStats[i]["assigned-addresses"] = 0;
+        expectedStats[i]["declined-addresses"] = 0;
+        expectedStats[i]["declined-reclaimed-addresses"] = 0;
+    }
+
+    // Make sure stats are as expected.
+    ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats));
+
+    // Recount stats.  We should have the same results.
+    ASSERT_NO_THROW(lmptr_->recountAddressStats4());
+
+    // Make sure stats are as expected.
+    ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats));
+
+    // Now let's insert some leases into subnet 1.
+    int subnet_id = 1;
+
+    // Insert one lease in default state, i.e. assigned.
+    makeLease4("192.0.1.1", subnet_id);
+
+    // Insert one lease in declined state.
+    makeLease4("192.0.1.2", subnet_id, Lease::STATE_DECLINED);
+
+    // Insert one lease in the expired state.
+    makeLease4("192.0.1.3", subnet_id, Lease::STATE_EXPIRED_RECLAIMED);
+
+    // Insert another lease in default state, i.e. assigned.
+    makeLease4("192.0.1.4", subnet_id);
+
+    // Update the expected stats list for subnet 1.
+    expectedStats[subnet_id - 1]["assigned-addresses"] = 2;
+    expectedStats[subnet_id - 1]["declined-addresses"] = 1;
+
+    // Now let's add leases to subnet 2.
+    subnet_id = 2;
+
+    // Insert one delined lease.
+    makeLease4("192.0.2.2", subnet_id, Lease::STATE_DECLINED);
+
+    // Update the expected stats.
+    expectedStats[subnet_id - 1]["declined-addresses"] = 1;
+
+    // Now Recount the stats.
+    ASSERT_NO_THROW(lmptr_->recountAddressStats4());
+
+    // Make sure stats are as expected.
+    ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats));
+
+    // Delete some leases from subnet, and update the expected stats.
+    EXPECT_TRUE(lmptr_->deleteLease(IOAddress("192.0.1.1")));
+    expectedStats[0]["assigned-addresses"] = 1;
+
+    EXPECT_TRUE(lmptr_->deleteLease(IOAddress("192.0.1.2")));
+    expectedStats[0]["declined-addresses"] = 0;
+
+    // Recount the stats.
+    ASSERT_NO_THROW(lmptr_->recountAddressStats4());
+
+    // Make sure stats are as expected.
+    ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats));
+}
+
 }; // namespace test
 }; // namespace dhcp
 }; // namespace isc

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2014-2015 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2014-2016 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -15,6 +15,12 @@ namespace isc {
 namespace dhcp {
 namespace test {
 
+
+/// @brief typedefs to simplify lease statistic testing
+typedef std::map<std::string, int64_t> StatValMap;
+typedef std::pair<std::string, int64_t> StatValPair;
+typedef std::vector<StatValMap> StatValMapList;
+
 /// @brief Test Fixture class with utility functions for LeaseMgr backends
 ///
 /// It contains utility functions, like dummy lease creation.
@@ -94,6 +100,34 @@ public:
     /// @return vector<Lease6Ptr> Vector of pointers to leases
     std::vector<Lease6Ptr> createLeases6();
 
+    /// @brief Compares a StatsMgr statistic to an expected value
+    ///
+    /// Attempt to fetch the named statistic from the StatsMg and if
+    /// found, compare its observed value to the given value.
+    /// Fails if the stat is not found or if the values do not match.
+    ///
+    /// @param name StatsMgr name for the statistic to check
+    /// @param expected_value expected value of the statistic
+    void checkStat(const std::string& name, const int64_t expected_value);
+
+    /// @brief Compares StatsMgr statistics against an expected list of values
+    ///
+    /// Iterates over a list of statistic names and expectec values, attempting
+    /// to fetch each from the StatsMgr and if found, compare its observed value
+    /// to the expected value.  Fails any the stat is not found or if the values
+    /// do not match.
+    ///
+    /// @param expected_stats Map of expected static names and values.
+    void checkAddressStats4(const StatValMapList& expected_stats);
+
+    /// @brief Constructs a minimal IPv4 lease and adds it to the lease storage
+    ///
+    /// @param address - IPv4 address for the lease
+    /// @param subnet_id - subnet ID to which the lease belongs
+    /// @param state - the state of the lease
+    void makeLease4(const std::string& address, const SubnetID& subnet_id,
+                    const Lease::LeaseState& state = Lease::STATE_DEFAULT);
+
     /// @brief checks that addLease, getLease4(addr) and deleteLease() works
     void testBasicLease4();
 
@@ -313,6 +347,13 @@ public:
     /// leases can be removed.
     void testDeleteExpiredReclaimedLeases4();
 
+    /// @brief Check that the IPv4 lease statistics can be recounted
+    ///
+    /// This test creates two subnets and several leases associated with
+    /// them, then verifies that lease statistics are recalculated correctly
+    /// after altering the lease states in various ways.
+    void testRecountAddressStats4();
+
     /// @brief String forms of IPv4 addresses
     std::vector<std::string>  straddress4_;
 

src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc

@@ -1883,6 +1883,10 @@ TEST_F(MemfileLeaseMgrTest, lease6ContainerIndexUpdate) {
     }
 }
 
-
+// Verifies that IPv4 lease statistics can be recalculated.
+TEST_F(MemfileLeaseMgrTest, recountAddressStats4) {
+    startBackend(V4);
+    testRecountAddressStats4();
+}
 
 }; // end of anonymous namespace

src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc

@@ -477,4 +477,9 @@ TEST_F(MySqlLeaseMgrTest, deleteExpiredReclaimedLeases4) {
     testDeleteExpiredReclaimedLeases4();
 }
 
+// Verifies that IPv4 lease statistics can be recalculated.
+TEST_F(MySqlLeaseMgrTest, recountAddressStats4) {
+    testRecountAddressStats4();
+}
+
 }; // Of anonymous namespace