[#1423] Checkpoint: test keyword sets

doc/examples/kea4/all-keys.json

@@ -11,6 +11,9 @@
 {
     // Kea DHCPv4 server configuration begins here.
     "Dhcp4": {
+        // TODO (here and other levels).
+        "authoritative": false,
+
         // Global bootfile name to be set in the 'file' field.
         "boot-file-name": "/dev/null",
 
@@ -284,7 +287,10 @@
                 "type": "mysql",
 
                 // User name to be used to access the database.
-                "user": "kea"
+                "user": "kea",
+
+                // Read only mode.
+                "readonly": false
             },
             {
                 // Name of the database to connect to.
@@ -337,6 +343,9 @@
                 // Connection reconnect wait time.
                 "reconnect-wait-time": 100,
 
+                // Connection maximum reconnect tries.
+                "max-reconnect-tries": 3,
+
                 // Connection connect timeout.
                 "connect-timeout": 100,
 
@@ -853,6 +862,9 @@
                                 // Reserved IP address.
                                 "ip-address": "192.0.2.204",
 
+                                // Hostname.
+                                "hostname": "foo.example.org",
+
                                 // Reservation specific option data.
                                 "option-data": [
                                     {
@@ -945,7 +957,10 @@
             "enable-queue": true,
 
             // Queue type was mandatory.
-            "queue-type": "kea-ring4"
+            "queue-type": "kea-ring4",
+
+            // Capacity is optional.
+            "capacity": 64
         },
 
         // Fetches host reservations.
@@ -1009,6 +1024,9 @@
                 // Specifies logging severity, i.e. "ERROR", "WARN", "INFO", "DEBUG".
                 "severity": "INFO"
             }
-        ]
+        ],
+
+        // Look at advanced example for the use of user-contexts.
+        "user-context": { }
     }
 }

doc/examples/kea6/all-keys.json

@@ -240,7 +240,10 @@
                 "type": "mysql",
 
                 // User name to be used to access the database.
-                "user": "kea"
+                "user": "kea",
+
+                // Read only mode.
+                "readonly": false
             },
             {
                 // Name of the database to connect to.
@@ -293,6 +296,9 @@
                 // Connection reconnect wait time.
                 "reconnect-wait-time": 100,
 
+                // Connection maximum reconnect tries.
+                "max-reconnect-tries": 3,
+
                 // Connection connect timeout.
                 "connect-timeout": 100,
 
@@ -937,7 +943,10 @@
             "enable-queue": true,
 
             // Queue type was mandatory.
-            "queue-type": "kea-ring6"
+            "queue-type": "kea-ring6",
+
+            // Capacity is optional.
+            "capacity": 64
         },
 
         // Fetches host reservations.
@@ -1004,6 +1013,9 @@
                 // Specifies logging severity, i.e. "ERROR", "WARN", "INFO", "DEBUG".
                 "severity": "INFO"
             }
-        ]
+        ],
+
+        // Look at advanced example for the use of user-contexts.
+        "user-context": { }
     }
 }

src/bin/agent/tests/Makefile.am

@@ -23,6 +23,7 @@ AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/agent/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
 AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/agent\"
+AM_CPPFLAGS += -DSYNTAX_FILE=\"$(srcdir)/../agent_parser.yy\"
 
 CLEANFILES = $(builddir)/interfaces.txt $(builddir)/logger_lockfile
 

src/bin/agent/tests/parser_unittests.cc

@@ -6,12 +6,14 @@
 
 #include <config.h>
 
-#include <gtest/gtest.h>
 #include <cc/data.h>
 #include <agent/parser_context.h>
 #include <cc/dhcp_config_error.h>
 #include <testutils/io_utils.h>
 #include <testutils/user_context_utils.h>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <set>
 
 using namespace isc::data;
 using namespace isc::test;
@@ -684,6 +686,76 @@ TEST(ParserTest, unicodeSlash) {
     EXPECT_EQ("////", result->stringValue());
 }
 
+// This test checks that all map entries are in the sample file.
+TEST(ParserTest, mapEntries) {
+    // Type of keyword set.
+    typedef set<string> KeywordSet;
+
+    // Get keywords from the syntax file (agent_parser.yy).
+    ifstream syntax_file(SYNTAX_FILE);
+    EXPECT_TRUE(syntax_file.is_open());
+    string line;
+    KeywordSet syntax_keys = { "user-context" };
+    // Code setting the map entry.
+    const string pattern = "ctx.stack_.back()->set(\"";
+    while (getline(syntax_file, line)) {
+        // Skip comments.
+        size_t comment = line.find("//");
+        if (comment <= pattern.size()) {
+            continue;
+        }
+        if (comment != string::npos) {
+            line.resize(comment);
+        }
+        // Search for the code pattern.
+        size_t key_begin = line.find(pattern);
+        if (key_begin == string::npos) {
+            continue;
+        }
+        // Extract keywords.
+        line = line.substr(key_begin + pattern.size());
+        size_t key_end = line.find_first_of('"');
+        EXPECT_NE(string::npos, key_end);
+        string keyword = line.substr(0, key_end);
+        // Ignore result when adding the keyword to the syntax keyword set.
+        static_cast<void>(syntax_keys.insert(keyword));
+    }
+    syntax_file.close();
+
+    // Get keywords from the sample file
+    string sample_fname(CFG_EXAMPLES);
+    sample_fname += "/simple.json";
+    ParserContext ctx;
+    ElementPtr sample_json;
+    EXPECT_NO_THROW(sample_json =
+        ctx.parseFile(sample_fname, ParserContext::PARSER_AGENT));
+    ASSERT_TRUE(sample_json);
+    KeywordSet sample_keys;
+    // Recursively extract keywords.
+    static void (*extract)(ConstElementPtr, KeywordSet&) =
+        [] (ConstElementPtr json, KeywordSet& set) {
+            if (json->getType() == Element::list) {
+                // Handle lists.
+                for (auto elem : json->listValue()) {
+                    extract(elem, set);
+                }
+            } else if (json->getType() == Element::map) {
+                // Handle maps.
+                for (auto elem : json->mapValue()) {
+                    static_cast<void>(set.insert(elem.first));
+                    if ((elem.first != "user-context") &&
+                        (elem.first != "parameters")) {
+                        extract(elem.second, set);
+                    }
+                }
+            }
+        };
+    extract(sample_json, sample_keys);
+
+    // Compare.
+    EXPECT_EQ(syntax_keys, sample_keys);
+}
+
 }
 }
 }

src/bin/d2/tests/Makefile.am

@@ -23,6 +23,7 @@ AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/d2/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
 AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/ddns\"
+AM_CPPFLAGS += -DSYNTAX_FILE=\"$(srcdir)/../d2_parser.yy\"
 
 CLEANFILES = $(builddir)/interfaces.txt $(builddir)/logger_lockfile
 

src/bin/d2/tests/parser_unittest.cc

@@ -5,12 +5,15 @@
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #include <config.h>
 
-#include <gtest/gtest.h>
 #include <cc/data.h>
 #include <d2/parser_context.h>
 #include <d2/tests/parser_unittest.h>
 #include <testutils/io_utils.h>
 #include <testutils/user_context_utils.h>
+#include <gtest/gtest.h>
+#include <fstream>
+
+#include "test_data_files_config.h"
 
 using namespace isc::data;
 using namespace isc::test;
@@ -628,6 +631,72 @@ TEST(ParserTest, unicodeSlash) {
     EXPECT_EQ("////", result->stringValue());
 }
 
+// This test checks that all map entries are in the sample file.
+TEST(ParserTest, mapEntries) {
+    // Get keywords from the syntax file (d2_parser.yy).
+    ifstream syntax_file(SYNTAX_FILE);
+    EXPECT_TRUE(syntax_file.is_open());
+    string line;
+    KeywordSet syntax_keys = { "user-context" };
+    // Code setting the map entry.
+    const string pattern = "ctx.stack_.back()->set(\"";
+    while (getline(syntax_file, line)) {
+        // Skip comments.
+        size_t comment = line.find("//");
+        if (comment <= pattern.size()) {
+            continue;
+        }
+        if (comment != string::npos) {
+            line.resize(comment);
+        }
+        // Search for the code pattern.
+        size_t key_begin = line.find(pattern);
+        if (key_begin == string::npos) {
+            continue;
+        }
+        // Extract keywords.
+        line = line.substr(key_begin + pattern.size());
+        size_t key_end = line.find_first_of('"');
+        EXPECT_NE(string::npos, key_end);
+        string keyword = line.substr(0, key_end);
+        // Ignore result when adding the keyword to the syntax keyword set.
+        static_cast<void>(syntax_keys.insert(keyword));
+    }
+    syntax_file.close();
+
+    // Get keywords from the sample file
+    string sample_fname(D2_TEST_DATA_DIR);
+    sample_fname += "/get_config.json";
+    D2ParserContext ctx;
+    ElementPtr sample_json;
+    EXPECT_NO_THROW(sample_json =
+        ctx.parseFile(sample_fname, D2ParserContext::PARSER_DHCPDDNS));
+    ASSERT_TRUE(sample_json);
+    KeywordSet sample_keys;
+    // Recursively extract keywords.
+    static void (*extract)(ConstElementPtr, KeywordSet&) =
+        [] (ConstElementPtr json, KeywordSet& set) {
+            if (json->getType() == Element::list) {
+                // Handle lists.
+                for (auto elem : json->listValue()) {
+                    extract(elem, set);
+                }
+            } else if (json->getType() == Element::map) {
+                // Handle maps.
+                for (auto elem : json->mapValue()) {
+                    static_cast<void>(set.insert(elem.first));
+                    if (elem.first != "user-context") {
+                        extract(elem.second, set);
+                    }
+                }
+            }
+        };
+    extract(sample_json, sample_keys);
+
+    // Compare.
+    EXPECT_EQ(syntax_keys, sample_keys);
+}
+
 }
 }
 }

src/bin/d2/tests/parser_unittest.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2017-2020 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -10,6 +10,7 @@
 #include <gtest/gtest.h>
 #include <cc/data.h>
 #include <d2/parser_context.h>
+#include <set>
 
 using namespace isc::data;
 using namespace std;
@@ -29,8 +30,11 @@ parseJSON(const std::string& in)
     return (ctx.parseString(in, isc::d2::D2ParserContext::PARSER_JSON));
 }
 
-};
-};
-};
+/// @brief Type of keyword set
+typedef std::set<std::string> KeywordSet;
+
+}
+}
+}
 
 #endif // PARSER_UNITTEST_H

src/bin/dhcp4/tests/Makefile.am

@@ -23,6 +23,7 @@ AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/dhcp4/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
 AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/kea4\"
+AM_CPPFLAGS += -DSYNTAX_FILE=\"$(srcdir)/../dhcp4_parser.yy\"
 
 CLEANFILES = $(builddir)/interfaces.txt $(builddir)/logger_lockfile
 CLEANFILES += $(builddir)/load_marker.txt $(builddir)/unload_marker.txt

src/bin/dhcp4/tests/parser_unittest.cc

@@ -6,12 +6,14 @@
 
 #include <config.h>
 
-#include <gtest/gtest.h>
 #include <dhcp4/parser_context.h>
 #include <dhcpsrv/parsers/simple_parser4.h>
 #include <testutils/gtest_utils.h>
 #include <testutils/io_utils.h>
 #include <testutils/user_context_utils.h>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <set>
 
 using namespace isc::data;
 using namespace isc::test;
@@ -681,6 +683,94 @@ TEST(ParserTest, unicodeSlash) {
     EXPECT_EQ("////", result->stringValue());
 }
 
+// This test checks that all map entries are in the all-keys file.
+TEST(ParserTest, mapEntries) {
+    // Type of keyword set.
+    typedef set<string> KeywordSet;
+
+    // Get keywords from the syntax file (dhcp4_parser.yy).
+    ifstream syntax_file(SYNTAX_FILE);
+    EXPECT_TRUE(syntax_file.is_open());
+    string line;
+    KeywordSet syntax_keys = { "user-context" };
+    // Code setting the map entry.
+    const string pattern = "ctx.stack_.back()->set(\"";
+    while (getline(syntax_file, line)) {
+        // Skip comments.
+        size_t comment = line.find("//");
+        if (comment <= pattern.size()) {
+            continue;
+        }
+        if (comment != string::npos) {
+            line.resize(comment);
+        }
+        // Search for the code pattern.
+        size_t key_begin = line.find(pattern);
+        if (key_begin == string::npos) {
+            continue;
+        }
+        // Extract keywords.
+        line = line.substr(key_begin + pattern.size());
+        size_t key_end = line.find_first_of('"');
+        EXPECT_NE(string::npos, key_end);
+        string keyword = line.substr(0, key_end);
+        // Ignore result when adding the keyword to the syntax keyword set.
+        static_cast<void>(syntax_keys.insert(keyword));
+    }
+    syntax_file.close();
+
+    // Get keywords from the all keys file
+    string sample_fname(CFG_EXAMPLES);
+    sample_fname += "/all-keys.json";
+    Parser4Context ctx;
+    ElementPtr sample_json;
+    EXPECT_NO_THROW(sample_json =
+        ctx.parseFile(sample_fname, Parser4Context::PARSER_DHCP4));
+    ASSERT_TRUE(sample_json);
+    KeywordSet sample_keys = {
+        "hw-address", "duid", "client-id", "flex-id",
+        "hosts-database"
+    };
+    // Recursively extract keywords.
+    static void (*extract)(ConstElementPtr, KeywordSet&) =
+        [] (ConstElementPtr json, KeywordSet& set) {
+            if (json->getType() == Element::list) {
+                // Handle lists.
+                for (auto elem : json->listValue()) {
+                    extract(elem, set);
+                }
+            } else if (json->getType() == Element::map) {
+                // Handle maps.
+                for (auto elem : json->mapValue()) {
+                    static_cast<void>(set.insert(elem.first));
+                    if (elem.first != "user-context") {
+                        extract(elem.second, set);
+                    }
+                }
+            }
+        };
+    extract(sample_json, sample_keys);
+
+    // Compare.
+    auto print_keys = [](const KeywordSet& keys) {
+        string s = "{";
+        bool first = true;
+        for (auto key : keys) {
+            if (first) {
+                first = false;
+                s += " ";
+            } else {
+                s += ", ";
+            }
+            s += "\"" + key + "\"";
+        }
+        return (s + " }");
+    };
+    EXPECT_EQ(syntax_keys, sample_keys)
+        << "syntax has: " << print_keys(syntax_keys) << endl
+        << "sample has: " << print_keys(sample_keys) << endl;
+}
+
 }
 }
 }

src/bin/dhcp6/tests/Makefile.am

@@ -24,6 +24,7 @@ AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/dhcp6/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
 AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/kea6\"
+AM_CPPFLAGS += -DSYNTAX_FILE=\"$(srcdir)/../dhcp6_parser.yy\"
 
 CLEANFILES  = $(builddir)/logger_lockfile
 CLEANFILES += $(builddir)/load_marker.txt $(builddir)/unload_marker.txt

src/bin/dhcp6/tests/parser_unittest.cc

@@ -6,11 +6,13 @@
 
 #include <config.h>
 
-#include <gtest/gtest.h>
 #include <dhcp6/parser_context.h>
 #include <dhcpsrv/parsers/simple_parser6.h>
 #include <testutils/io_utils.h>
 #include <testutils/user_context_utils.h>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <set>
 
 using namespace isc::data;
 using namespace std;
@@ -670,6 +672,95 @@ TEST(ParserTest, unicodeSlash) {
     EXPECT_EQ("////", result->stringValue());
 }
 
+// This test checks that all map entries are in the all-keys file.
+TEST(ParserTest, mapEntries) {
+    // Type of keyword set.
+    typedef set<string> KeywordSet;
+
+    // Get keywords from the syntax file (dhcp6_parser.yy).
+    ifstream syntax_file(SYNTAX_FILE);
+    EXPECT_TRUE(syntax_file.is_open());
+    string line;
+    KeywordSet syntax_keys = { "user-context" };
+    // Code setting the map entry.
+    const string pattern = "ctx.stack_.back()->set(\"";
+    while (getline(syntax_file, line)) {
+        // Skip comments.
+        size_t comment = line.find("//");
+        if (comment <= pattern.size()) {
+            continue;
+        }
+        if (comment != string::npos) {
+            line.resize(comment);
+        }
+        // Search for the code pattern.
+        size_t key_begin = line.find(pattern);
+        if (key_begin == string::npos) {
+            continue;
+        }
+        // Extract keywords.
+        line = line.substr(key_begin + pattern.size());
+        size_t key_end = line.find_first_of('"');
+        EXPECT_NE(string::npos, key_end);
+        string keyword = line.substr(0, key_end);
+        // Ignore result when adding the keyword to the syntax keyword set.
+        static_cast<void>(syntax_keys.insert(keyword));
+    }
+    syntax_file.close();
+
+    // Get keywords from the all keys file
+    string sample_fname(CFG_EXAMPLES);
+    sample_fname += "/all-keys.json";
+    Parser6Context ctx;
+    ElementPtr sample_json;
+    EXPECT_NO_THROW(sample_json =
+        ctx.parseFile(sample_fname, Parser6Context::PARSER_DHCP6));
+    ASSERT_TRUE(sample_json);
+    KeywordSet sample_keys = {
+        "hw-address", "flex-id",
+        "hosts-database",
+        "htype", "ip-address", "time"
+    };
+    // Recursively extract keywords.
+    static void (*extract)(ConstElementPtr, KeywordSet&) =
+        [] (ConstElementPtr json, KeywordSet& set) {
+            if (json->getType() == Element::list) {
+                // Handle lists.
+                for (auto elem : json->listValue()) {
+                    extract(elem, set);
+                }
+            } else if (json->getType() == Element::map) {
+                // Handle maps.
+                for (auto elem : json->mapValue()) {
+                    static_cast<void>(set.insert(elem.first));
+                    if (elem.first != "user-context") {
+                        extract(elem.second, set);
+                    }
+                }
+            }
+        };
+    extract(sample_json, sample_keys);
+
+    // Compare.
+    auto print_keys = [](const KeywordSet& keys) {
+        string s = "{";
+        bool first = true;
+        for (auto key : keys) {
+            if (first) {
+                first = false;
+                s += " ";
+            } else {
+                s += ", ";
+            }
+            s += "\"" + key + "\"";
+        }
+        return (s + " }");
+    };
+    EXPECT_EQ(syntax_keys, sample_keys)
+        << "syntax has: " << print_keys(syntax_keys) << endl
+        << "sample has: " << print_keys(sample_keys) << endl;
+}
+
 }
 }
 }

src/bin/netconf/tests/Makefile.am

@@ -8,6 +8,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/bin -I$(top_builddir)/src/bin
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/netconf/tests\"
 AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/netconf\"
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/netconf/tests\"
+AM_CPPFLAGS += -DSYNTAX_FILE=\"$(srcdir)/../netconf_parser.yy\"
 AM_CPPFLAGS += $(BOOST_INCLUDES) $(SYSREPO_INCLUDEDIR)
 
 CLEANFILES = *.json *.log

src/bin/netconf/tests/parser_unittests.cc

@@ -6,12 +6,14 @@
 
 #include <config.h>
 
-#include <gtest/gtest.h>
 #include <cc/data.h>
 #include <netconf/parser_context.h>
 #include <cc/dhcp_config_error.h>
 #include <testutils/io_utils.h>
 #include <testutils/user_context_utils.h>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <set>
 
 using namespace isc::data;
 using namespace isc::test;
@@ -740,6 +742,79 @@ TEST(ParserTest, unicodeSlash) {
     EXPECT_EQ("////", result->stringValue());
 }
 
+// This test checks that all map entries are in the sample file.
+TEST(ParserTest, mapEntries) {
+    // Type of keyword set.
+    typedef set<string> KeywordSet;
+
+    // Get keywords from the syntax file (netconf_parser.yy).
+    ifstream syntax_file(SYNTAX_FILE);
+    EXPECT_TRUE(syntax_file.is_open());
+    string line;
+    KeywordSet syntax_keys = { "user-context" };
+    // Code setting the map entry.
+    const string pattern = "ctx.stack_.back()->set(\"";
+    while (getline(syntax_file, line)) {
+        // Skip comments.
+        size_t comment = line.find("//");
+        if (comment <= pattern.size()) {
+            continue;
+        }
+        if (comment != string::npos) {
+            line.resize(comment);
+        }
+        // Search for the code pattern.
+        size_t key_begin = line.find(pattern);
+        if (key_begin == string::npos) {
+            continue;
+        }
+        // Extract keywords.
+        line = line.substr(key_begin + pattern.size());
+        size_t key_end = line.find_first_of('"');
+        EXPECT_NE(string::npos, key_end);
+        string keyword = line.substr(0, key_end);
+        // Ignore result when adding the keyword to the syntax keyword set.
+        static_cast<void>(syntax_keys.insert(keyword));
+    }
+    syntax_file.close();
+
+    // Get keywords from the sample file
+    string sample_fname(CFG_EXAMPLES);
+    sample_fname += "/simple-dhcp4.json";
+    ParserContext ctx;
+    ElementPtr sample_json;
+    EXPECT_NO_THROW(sample_json =
+        ctx.parseFile(sample_fname, ParserContext::PARSER_NETCONF));
+    ASSERT_TRUE(sample_json);
+    KeywordSet sample_keys = {
+      "ca", "d2", "dhcp6",
+      "hooks-libraries", "library", "parameters",
+      "socket-url"
+    };
+    // Recursively extract keywords.
+    static void (*extract)(ConstElementPtr, KeywordSet&) =
+        [] (ConstElementPtr json, KeywordSet& set) {
+            if (json->getType() == Element::list) {
+                // Handle lists.
+                for (auto elem : json->listValue()) {
+                    extract(elem, set);
+                }
+            } else if (json->getType() == Element::map) {
+                // Handle maps.
+                for (auto elem : json->mapValue()) {
+                    static_cast<void>(set.insert(elem.first));
+                    if (elem.first != "user-context") {
+                        extract(elem.second, set);
+                    }
+                }
+            }
+        };
+    extract(sample_json, sample_keys);
+
+    // Compare.
+    EXPECT_EQ(syntax_keys, sample_keys);
+}
+
 }
 }
 }