[optimize_query] Add new index for query by DUID

Fixes #98

AUTHORS

@@ -172,6 +172,7 @@ We have received the following contributions:
  - Sunil Mayya
    2018-07: support for Authentication option in DHCPv6
    2018-07: support storage of Authentication keys in host structure
+   2018-07: Optimized query for host reservation from the backends
 
 Kea uses log4cplus (http://sourceforge.net/projects/log4cplus/) for logging,
 Boost (http://www.boost.org/) library for almost everything, and can use Botan

ChangeLog

@@ -1,3 +1,7 @@
+1436.   [func]      MayyaSunil
+	Implemented new index for queries host reservations by DUID.
+	(Github #98, git c20b5248da1283e596e35ad057ae242f4d613965)
+
 1435.	[func]		marcin
 	Implemented ha-continue command in HA hooks library and
 	updated the Kea User's Guide with the information how to

src/lib/dhcpsrv/cql_lease_mgr.cc

@@ -2306,10 +2306,10 @@ CqlLeaseMgr::getLeases6(const DUID& duid) const {
     // Get the data.
     Lease6Collection result;
     std::unique_ptr<CqlLease6Exchange> exchange6(new CqlLease6Exchange(dbconn_));
-    exchange6->getLeaseCollection(CqlLease6Exchange::GET_LEASE6_DUID, data, result);
+    exchange6->getLeaseCollection(CqlLease6Exchange::GET_LEASE6_DUID,
+                                  data, result);
 
     return (result);
-    
 }
 
 Lease6Collection

src/lib/dhcpsrv/lease_mgr.h

@@ -460,9 +460,10 @@ public:
 
     /// @brief Returns collection of lease for matching DUID
     ///
-    /// @return Lease collection (may be empty if no IPv6 lease found for the DUID).
+    /// @return Lease collection 
+    /// (may be empty if no IPv6 lease found for the DUID).
     virtual Lease6Collection getLeases6(const DUID& duid) const = 0; 
-    
+
     /// @brief Returns range of IPv6 leases using paging.
     ///
     /// This method implements paged browsing of the lease database. The first

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -1020,13 +1020,17 @@ Memfile_LeaseMgr::getLeases6(const DUID& duid) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MEMFILE_GET6_DUID)
        .arg(duid.toText());
 
-   Lease6Collection collection;
+    Lease6Collection collection;
-   for (auto lease = storage6_.begin(); lease != storage6_.end(); ++lease ) {
+    const Lease6StorageDuidIndex& idx = storage6_.get<DuidIndexTag>();
-       if ( (**lease).duid_->getDuid() == duid.getDuid() )
+    std::pair<Lease6StorageDuidIndex::const_iterator,
-            collection.push_back(Lease6Ptr(new Lease6(**lease)));
+              Lease6StorageDuidIndex::const_iterator> l =
-   }
+        idx.equal_range(duid.getDuid());
 
-   return (collection);
+    for (auto lease = l.first; lease != l.second; ++lease) {
+        collection.push_back(Lease6Ptr(new Lease6(**lease)));
+    }
+
+    return (collection);
 }
 
 Lease6Collection

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -315,7 +315,8 @@ public:
     /// @brief Returns IPv6 leases for the DUID.
     ///
     /// @todo: implement an optimised of the query using index.
-    /// @return Lease collection (may be empty if no IPv6 lease found) for the DUID.
+    /// @return Lease collection (may be empty if no IPv6 lease found) 
+    /// for the DUID.
     virtual Lease6Collection getLeases6(const DUID& duid) const;
     
     /// @brief Returns range of IPv6 leases using paging.

src/lib/dhcpsrv/memfile_lease_storage.h

@@ -45,6 +45,8 @@ struct ClientIdHWAddressSubnetIdIndexTag { };
 /// @brief Tag for indexs by subnet-id.
 struct SubnetIdIndexTag { };
 
+/// @brief Tag for index using DUID.
+struct DuidIndexTag { }; 
 /// @name Multi index containers holding DHCPv4 and DHCPv6 leases.
 ///
 //@{
@@ -112,9 +114,19 @@ typedef boost::multi_index_container<
         // This index sorts leases by SubnetID.
         boost::multi_index::ordered_non_unique<
             boost::multi_index::tag<SubnetIdIndexTag>,
-            boost::multi_index::member<Lease, isc::dhcp::SubnetID, &Lease::subnet_id_>
+            boost::multi_index::member<Lease, isc::dhcp::SubnetID,
+            &Lease::subnet_id_>
+        >,
+
+        // Specification of the fifth index starts here
+        // This index is used to retrieve leases for matching duid.
+        boost::multi_index::ordered_non_unique<
+            boost::multi_index::tag<DuidIndexTag>,
+            boost::multi_index::const_mem_fun<Lease6,
+                                              const std::vector<uint8_t>&,
+                                              &Lease6::getDuidVector>
         >
-     >
+    >
 > Lease6Storage; // Specify the type name of this container.
 
 /// @brief A multi index container holding DHCPv4 leases.
@@ -250,6 +262,9 @@ typedef Lease6Storage::index<ExpirationIndexTag>::type Lease6StorageExpirationIn
 /// @brief DHCPv6 lease storage index by Subnet-id.
 typedef Lease6Storage::index<SubnetIdIndexTag>::type Lease6StorageSubnetIdIndex;
 
+/// @brief DHCPv6 lease storage index by Subnet-id.
+typedef Lease6Storage::index<DuidIndexTag>::type Lease6StorageDuidIndex;
+
 /// @brief DHCPv4 lease storage index by address.
 typedef Lease4Storage::index<AddressIndexTag>::type Lease4StorageAddressIndex;
 

src/lib/dhcpsrv/mysql_lease_mgr.cc

@@ -223,6 +223,15 @@ tagged_statements = { {
                         "state, user_context "
                             "FROM lease6 "
                             "WHERE subnet_id = ?"},
+    {MySqlLeaseMgr::GET_LEASE6_DUID,
+                    "SELECT address, duid, valid_lifetime, "
+                        "expire, subnet_id, pref_lifetime, "
+                        "lease_type, iaid, prefix_len, "
+                        "fqdn_fwd, fqdn_rev, hostname, "
+                        "hwaddr, hwtype, hwaddr_source, "
+                        "state, user_context "
+                            "FROM lease6 "
+                            "WHERE duid = ?"},
     {MySqlLeaseMgr::GET_LEASE6_EXPIRE,
                     "SELECT address, duid, valid_lifetime, "
                         "expire, subnet_id, pref_lifetime, "
@@ -2224,15 +2233,22 @@ Lease6Collection
 MySqlLeaseMgr::getLeases6(const DUID& duid) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MYSQL_GET_DUID);
    
-    Lease6Collection result =  getLeases6();
+    // Set up the WHERE clause value
+    MYSQL_BIND inbind[1];
+    memset(inbind, 0, sizeof(inbind));
+
+    const vector<uint8_t>& duid_vector = duid.getDuid();
+    unsigned long duid_length = duid_vector.size();
+
+    inbind[0].buffer_type = MYSQL_TYPE_BLOB;
+    inbind[0].buffer = reinterpret_cast<char*>(
+            const_cast<uint8_t*>(&duid_vector[0]));
+    inbind[0].buffer_length = duid_length;
+    inbind[0].length = &duid_length;
     
-    //erase the ones not containing the matching DUID
+    Lease6Collection result;
-    for (auto iter = result.begin(); iter != result.end();
+    
-            iter++) {
+    getLeaseCollection(GET_LEASE6_DUID, inbind, result);
-        if ((*iter)->duid_->getDuid() != duid.getDuid()) {
-            result.erase(iter);
-        }
-    }
 
     return result;
 }

src/lib/dhcpsrv/mysql_lease_mgr.h

@@ -312,7 +312,8 @@ public:
     /// @brief Returns all IPv6 leases for the DUID.
     ///
     /// @todo: implement an optimised of the query using index.
-    /// @return Lease collection (may be empty if no IPv6 lease found) for the DUID.
+    /// @return Lease collection (may be empty if no IPv6 lease found)
+    /// for the DUID.
     virtual Lease6Collection getLeases6(const DUID& duid) const;
     
     /// @brief Returns range of IPv6 leases using paging.
@@ -586,6 +587,7 @@ public:
         GET_LEASE6_DUID_IAID_SUBID,  // Get lease6 by DUID, IAID and subnet ID
         GET_LEASE6_PAGE,             // Get page of leases beginning with an address
         GET_LEASE6_SUBID,            // Get IPv6 leases by subnet ID
+        GET_LEASE6_DUID,             // Get IPv6 leases by DUID
         GET_LEASE6_EXPIRE,           // Get lease6 by expiration.
         INSERT_LEASE4,               // Add entry to lease4 table
         INSERT_LEASE6,               // Add entry to lease6 table

src/lib/dhcpsrv/pgsql_lease_mgr.cc

@@ -215,6 +215,17 @@ PgSqlTaggedStatement tagged_statements[] = {
       "FROM lease6 "
       "WHERE subnet_id = $1"},
 
+    // GET_LEASE6_DUID
+    { 1, { OID_BYTEA },
+      "get_lease6_duid",
+      "SELECT address, duid, valid_lifetime, "
+        "extract(epoch from expire)::bigint, subnet_id, pref_lifetime, "
+        "lease_type, iaid, prefix_len, fqdn_fwd, fqdn_rev, hostname, "
+        "hwaddr, hwtype, hwaddr_source, "
+        "state, user_context "
+      "FROM lease6 "
+      "WHERE duid = $1"},
+
     // GET_LEASE6_EXPIRE
     { 3, { OID_INT8, OID_TIMESTAMP, OID_INT8 },
       "get_lease6_expire",
@@ -1507,20 +1518,20 @@ PgSqlLeaseMgr::getLeases6(SubnetID subnet_id) const {
 }
 
 Lease6Collection
-PgSqlLeaseMgr::getLeases6( const DUID& duid ) const {
+PgSqlLeaseMgr::getLeases6(const DUID& duid) const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
               DHCPSRV_PGSQL_GET_DUID)
               .arg(duid.toText());
 
-    Lease6Collection result = getLeases6();
+    // Set up the WHERE clause value
+    PsqlBindArray bind_array;
 
-    //erase the ones not containing the matching DUID
+    // DUID
-    for (auto iter = result.begin(); iter != result.end();
+    bind_array.add(duid.getDuid());
-            iter++) {
+    Lease6Collection result;
-        if ((*iter)->duid_->getDuid() != duid.getDuid()) {
+
-            result.erase(iter);
+    // query to fetch the data
-        }
+    getLeaseCollection(GET_LEASE6_DUID, bind_array, result);
-    }
 
     return (result);
 }

src/lib/dhcpsrv/pgsql_lease_mgr.h

@@ -284,7 +284,8 @@ public:
     /// @brief Returns IPv6 leases for the DUID.
     ///
     /// @todo: implement an optimised of the query using index.
-    /// @return Lease collection (may be empty if no IPv6 lease found) for the DUID
+    /// @return Lease collection (may be empty if no IPv6 lease found)
+    /// for the DUID
     virtual Lease6Collection getLeases6(const DUID& duid) const;
     
     /// @brief Returns range of IPv6 leases using paging.
@@ -554,6 +555,7 @@ public:
         GET_LEASE6_DUID_IAID_SUBID, // Get lease6 by DUID, IAID and subnet ID
         GET_LEASE6_PAGE,            // Get page of IPv6 leases beginning with an address
         GET_LEASE6_SUBID,           // Get IPv6 leases by subnet ID
+        GET_LEASE6_DUID,           // Get IPv6 leases by DUID
         GET_LEASE6_EXPIRE,          // Get expired lease6
         INSERT_LEASE4,              // Add entry to lease4 table
         INSERT_LEASE6,              // Add entry to lease6 table

src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc

@@ -679,6 +679,14 @@ TEST_F(CqlLeaseMgrTest, lease6LeaseTypeCheck) {
     testLease6LeaseTypeCheck();
 }
 
+/// @brief Verifies the getLeases6(DUID) method
+///
+/// Adds 3 lease and verifies fetch by DUID.
+/// Verifies retrival of non existant DUID fails
+TEST_F(CqlLeaseMgrTest, getLeases6Duid) {
+   testGetLeases6Duid(); 
+}
+
 /// @brief Check GetLease6 methods - access by DUID/IAID/SubnetID
 ///
 /// Adds leases to the database and checks that they can be accessed via

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc

@@ -1620,6 +1620,62 @@ GenericLeaseMgrTest::testGetLease6DuidIaidSubnetId() {
     EXPECT_FALSE(returned);
 }
 
+/// @brief verifies getLeases6(DUID)
+void
+GenericLeaseMgrTest::testGetLeases6Duid() {
+    //add leases
+    IOAddress addr1(std::string("2001:db8:1::111"));
+    IOAddress addr2(std::string("2001:db8:1::222"));
+    IOAddress addr3(std::string("2001:db8:1::333"));
+
+    DuidPtr duid1(new DUID({0, 1, 1, 1, 1, 1, 1, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}));
+    DuidPtr duid2(new DUID({0, 2, 2, 2, 2, 2, 2, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}));
+    DuidPtr duid3(new DUID({0, 3, 3, 3, 3, 3, 3, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}));
+    DuidPtr duid4(new DUID({0, 4, 4, 4, 4, 4, 4, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}));
+
+    uint32_t iaid = 7; // random number
+
+    SubnetID subnet_id = 8; // radom number
+
+    Lease6Ptr lease1(new Lease6(Lease::TYPE_NA, addr1, duid1, iaid, 100, 200, 50,
+                               80, subnet_id));
+    Lease6Ptr lease2(new Lease6(Lease::TYPE_NA, addr2, duid2, iaid, 100, 200, 50,
+                               80, subnet_id));
+    Lease6Ptr lease3(new Lease6(Lease::TYPE_NA, addr3, duid3, iaid, 100, 200, 50,
+                               80, subnet_id));
+
+    EXPECT_TRUE(lmptr_->addLease(lease1));
+    EXPECT_TRUE(lmptr_->addLease(lease2));
+    EXPECT_TRUE(lmptr_->addLease(lease3));
+    
+    Lease6Collection returned1 = lmptr_->getLeases6(*(lease1->duid_));
+    Lease6Collection returned2 = lmptr_->getLeases6(*(lease2->duid_));
+    Lease6Collection returned3 = lmptr_->getLeases6(*(lease3->duid_));
+  
+    //verify if the returned lease mathces 
+    EXPECT_EQ(returned1.size(), 1);
+    EXPECT_EQ(returned2.size(), 1);
+    EXPECT_EQ(returned3.size(), 1);
+
+    //verify that the returned lease are same
+    EXPECT_TRUE(returned1[0]->addr_ == lease1->addr_); 
+    EXPECT_TRUE(returned2[0]->addr_ == lease2->addr_);
+    EXPECT_TRUE(returned3[0]->addr_ == lease3->addr_);
+    
+    //now verify we return empty for a lease that has not been stored
+    returned3 = lmptr_->getLeases6(*duid4);
+    EXPECT_TRUE(returned3.empty());
+   
+    //clean up
+    (void) lmptr_->deleteLease(addr1);
+    (void) lmptr_->deleteLease(addr2);
+    (void) lmptr_->deleteLease(addr3);
+   
+    //now verify we return empty for a lease that has not been stored
+    returned3 = lmptr_->getLeases6(*duid4);
+    EXPECT_TRUE(returned3.empty());
+}
+
 /// @brief Checks that getLease6() works with different DUID sizes
 void
 GenericLeaseMgrTest::testGetLease6DuidIaidSubnetIdSize() {

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h

@@ -278,6 +278,12 @@ public:
     /// Adds leases to the database and checks that they can be accessed via
     /// a combination of DIUID and IAID.
     void testGetLease6DuidIaidSubnetId();
+    
+    /// @brief verifies getLeases6 method by DUID
+    ///
+    /// Adds 3 leases to backend and retrieves, verifes empty
+    /// retrival of non existent DUID.
+    void testGetLeases6Duid();
 
     /// @brief Checks that getLease6() works with different DUID sizes
     void testGetLease6DuidIaidSubnetIdSize();

src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc

@@ -942,6 +942,13 @@ TEST_F(MemfileLeaseMgrTest, getLeases6SubnetId) {
     testGetLeases6SubnetId();
 }
 
+// This test adds 3 leases  and verifies fetch by DUID.
+// Verifies retrival of non existant DUID fails
+TEST_F(MemfileLeaseMgrTest, getLeases6Duid) {
+    startBackend(V6);
+    testGetLeases6Duid();
+}
+
 // This test checks that all IPv6 leases are returned.
 TEST_F(MemfileLeaseMgrTest, getLeases6) {
     startBackend(V6);

src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc

@@ -475,6 +475,14 @@ TEST_F(MySqlLeaseMgrTest, getLease6DuidIaidSubnetIdSize) {
     testGetLease6DuidIaidSubnetIdSize();
 }
 
+// @brief check leases could be retrieved by DUID
+///
+/// Create leases, add them to backend and verify if it can be queried
+/// using DUID index
+TEST_F(MySqlLeaseMgrTest, getLeases6Duid) {
+    testGetLeases6Duid();
+}
+
 /// @brief Lease6 update tests
 ///
 /// Checks that we are able to update a lease in the database.

src/lib/dhcpsrv/tests/pgsql_lease_mgr_unittest.cc

@@ -451,6 +451,14 @@ TEST_F(PgSqlLeaseMgrTest, lease6LeaseTypeCheck) {
     testLease6LeaseTypeCheck();
 }
 
+/// @brief Verifies the getLeases6(DUID) method
+///
+/// Adds 3 lease and verifies fetch by DUID.
+/// Verifies retrival of non existant DUID fails
+TEST_F(PgSqlLeaseMgrTest, getLeases6Duid) {
+   testGetLeases6Duid(); 
+}
+
 /// @brief Check GetLease6 methods - access by DUID/IAID/SubnetID
 ///
 /// Adds leases to the database and checks that they can be accessed via