From c27421b7e7b41247b41aca0de903f328048ab920 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Fri, 7 Jun 2013 14:07:42 -0700 Subject: [PATCH 01/35] [2854] add "bind10_server" module, a mixin providing common server behavior. --- configure.ac | 1 + src/lib/python/isc/server_common/Makefile.am | 4 +- .../isc/server_common/bind10_server.py.in | 115 ++++++++++++++++++ .../server_common/server_common_messages.mes | 16 +++ .../isc/server_common/tests/Makefile.am | 1 + .../server_common/tests/bind10_server_test.py | 92 ++++++++++++++ 6 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 src/lib/python/isc/server_common/bind10_server.py.in create mode 100755 src/lib/python/isc/server_common/tests/bind10_server_test.py diff --git a/configure.ac b/configure.ac index b01be12851..559178ba6d 100644 --- a/configure.ac +++ b/configure.ac @@ -1395,6 +1395,7 @@ AC_OUTPUT([doc/version.ent src/lib/python/isc/notify/tests/notify_out_test src/lib/python/isc/log/tests/log_console.py src/lib/python/isc/log_messages/work/__init__.py + src/lib/python/isc/server_common/bind10_server.py src/lib/dns/gen-rdatacode.py src/lib/python/bind10_config.py src/lib/cc/session_config.h.pre diff --git a/src/lib/python/isc/server_common/Makefile.am b/src/lib/python/isc/server_common/Makefile.am index 596d6cd24b..54b288584e 100644 --- a/src/lib/python/isc/server_common/Makefile.am +++ b/src/lib/python/isc/server_common/Makefile.am @@ -1,12 +1,14 @@ SUBDIRS = tests python_PYTHON = __init__.py tsig_keyring.py auth_command.py dns_tcp.py -python_PYTHON += datasrc_clients_mgr.py +python_PYTHON += datasrc_clients_mgr.py bind10_server.py python_PYTHON += logger.py pythondir = $(pyexecdir)/isc/server_common BUILT_SOURCES = $(PYTHON_LOGMSGPKG_DIR)/work/server_common_messages.py +BUILT_SOURCES += bind10_server.py + nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/server_common_messages.py pylogmessagedir = $(pyexecdir)/isc/log_messages/ diff --git a/src/lib/python/isc/server_common/bind10_server.py.in b/src/lib/python/isc/server_common/bind10_server.py.in new file mode 100644 index 0000000000..51b3ba81dc --- /dev/null +++ b/src/lib/python/isc/server_common/bind10_server.py.in @@ -0,0 +1,115 @@ +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import os +import signal + +import isc.log +from isc.server_common.logger import logger +from isc.log_messages.server_common_messages import * + +class BIND10Server: + """A mixin class for common BIND 10 server implementations. + + Methods to be implemented in the actual class: + + """ + # Will be set to True when the server should stop and shut down. + # Can be read via accessor method 'shutdown', mainly for testing. + __shutdown = False + __module_name = None + + @property + def shutdown(self): + return self.__shutdown + + def _setup_ccsession(self): + self._cc = isc.config.ModuleCCSession(self._get_specfile_location(), + self._config_handler, + self._command_handler) + + def _get_specfile_location(self): + """Return the path to the module spec file following common convetion. + + This method generates the path commonly used by most BIND 10 modules, + determined by a well known prefix and the module name. + + A specific module can override this method if it uses a different + path for the spec file. + + """ + if 'B10_FROM_SOURCE' in os.environ: + specfile_path = os.environ['B10_FROM_SOURCE'] + '/src/bin/' + \ + self.__module_name + else: + specfile_path = '@datadir@/@PACKAGE@'\ + .replace('${datarootdir}', '@datarootdir@')\ + .replace('${prefix}', '@prefix@') + return specfile_path + '/' + self.__module_name + + def _trigger_shutdown(self): + """Initiate a shutdown sequence. + + This method is expected to be called in various ways including + in the middle of a signal handler, and is designed to be as simple + as possible to minimize side effects. Actual shutdown will take + place in a normal control flow. + + This method is defined as 'protected' so tests can call it; otherwise + it's private. + + """ + self.__shutdown = True + + def _run_internal(self): + while not self.__shutdown: + pass + + def _config_handler(self): + pass + + def _command_handler(self): + pass + + def run(self, module_name): + """Start the server and let it run until it's told to stop. + + Usually this must be the first method of this class that is called + from its user. + + Parameter: + module_name (str): the Python module name for the actual server + implementation. Often identical to the directory name in which + the implementation files are placed. + + Returns: values expected to be used as program's exit code. + 0: server has run and finished successfully. + 1: some error happens + + """ + try: + self.__module_name = module_name + shutdown_sighandler = \ + lambda signal, frame: self._trigger_shutdown() + signal.signal(signal.SIGTERM, shutdown_sighandler) + signal.signal(signal.SIGINT, shutdown_sighandler) + self._setup_ccsession() + self._run_internal() + return 0 + except Exception as ex: + logger.error(PYSERVER_COMMON_UNCAUGHT_EXCEPTION, type(ex).__name__, + str(ex)) + + return 1 diff --git a/src/lib/python/isc/server_common/server_common_messages.mes b/src/lib/python/isc/server_common/server_common_messages.mes index f22ce65210..24e221e3f5 100644 --- a/src/lib/python/isc/server_common/server_common_messages.mes +++ b/src/lib/python/isc/server_common/server_common_messages.mes @@ -21,6 +21,9 @@ # have that at this moment. So when adding a message, make sure that # the name is not already used in src/lib/config/config_messages.mes +% PYSERVER_COMMON_COMMAND %1 server has received '%2' command +The server process received the shown name of command from other module. + % PYSERVER_COMMON_DNS_TCP_SEND_DONE completed sending TCP message to %1 (%2 bytes in total) Debug message. A complete DNS message has been successfully transmitted over a TCP connection, possibly after multiple send @@ -44,6 +47,14 @@ The destination address and the total size of the message that has been transmitted so far (including the 2-byte length field) are shown in the log message. +% PYSERVER_COMMON_SERVER_STARTED %1 server has started +The server process has successfully started and is now ready to receive +commands and updates. + +% PYSERVER_COMMON_SERVER_STOPPED %1 server has stopped +The server process has successfully stopped and is no longer listening for or +handling commands. Normally the process will soon exit. + % PYSERVER_COMMON_TSIG_KEYRING_DEINIT Deinitializing global TSIG keyring A debug message noting that the global TSIG keyring is being removed from memory. Most programs don't do that, they just exit, which is OK. @@ -57,3 +68,8 @@ to be loaded from configuration. A debug message. The TSIG keyring is being (re)loaded from configuration. This happens at startup or when the configuration changes. The old keyring is removed and new one created with all the keys. + +% PYSERVER_COMMON_UNCAUGHT_EXCEPTION uncaught exception of type %1: %2 +The BIND 10 server process encountered an uncaught exception and will now shut +down. This is indicative of a programming error and should not happen under +normal circumstances. The exception type and message are printed. diff --git a/src/lib/python/isc/server_common/tests/Makefile.am b/src/lib/python/isc/server_common/tests/Makefile.am index 86006d598e..c2667fdad6 100644 --- a/src/lib/python/isc/server_common/tests/Makefile.am +++ b/src/lib/python/isc/server_common/tests/Makefile.am @@ -1,5 +1,6 @@ PYCOVERAGE_RUN = @PYCOVERAGE_RUN@ PYTESTS = tsig_keyring_test.py dns_tcp_test.py datasrc_clients_mgr_test.py +PYTESTS += bind10_server_test.py EXTRA_DIST = $(PYTESTS) # If necessary (rare cases), explicitly specify paths to dynamic libraries diff --git a/src/lib/python/isc/server_common/tests/bind10_server_test.py b/src/lib/python/isc/server_common/tests/bind10_server_test.py new file mode 100755 index 0000000000..19cf3356cf --- /dev/null +++ b/src/lib/python/isc/server_common/tests/bind10_server_test.py @@ -0,0 +1,92 @@ +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import unittest +import os +import signal + +import isc.log +import isc.config +from isc.server_common.bind10_server import * +from isc.testutils.ccsession_mock import MockModuleCCSession + +class TestException(Exception): + """A generic exception class specific in this test module.""" + pass + +class MyCCSession(MockModuleCCSession, isc.config.ConfigData): + def __init__(self, specfile, config_handler, command_handler): + pass + +class MockServer(BIND10Server): + def _setup_ccsession(self): + orig_cls = isc.config.ModuleCCSession + isc.config.ModuleCCSession = MyCCSession + try: + super()._setup_ccsession() + except Exception: + raise + finally: + isc.config.ModuleCCSession = orig_cls + +class TestBIND10Server(unittest.TestCase): + def setUp(self): + self.__server = MockServer() + + def test_init(self): + """Check initial conditions""" + self.assertFalse(self.__server.shutdown) + + def test_trigger_shutdown(self): + self.__server._trigger_shutdown() + self.assertTrue(self.__server.shutdown) + + def test_sigterm_handler(self): + """Check the signal handler behavior. + + SIGTERM and SIGINT should be caught and should call memmgr's + _trigger_shutdown(). This test also indirectly confirms main() calls + run_internal(). + + """ + def checker(): + self.__shutdown_called = True + + self.__server._run_internal = lambda: os.kill(os.getpid(), + signal.SIGTERM) + self.__server._trigger_shutdown = lambda: checker() + self.assertEqual(0, self.__server.run('test')) + self.assertTrue(self.__shutdown_called) + + self.__shutdown_called = False + self.__server._run_internal = lambda: os.kill(os.getpid(), + signal.SIGINT) + self.assertEqual(0, self.__server.run('test')) + self.assertTrue(self.__shutdown_called) + + def test_exception(self): + """Check exceptions are handled, not leaked.""" + def exception_raiser(ex_cls): + raise ex_cls('test') + + # Test all possible exceptions that are explicitly caught + for ex in [TestException]: + self.__server._run_internal = lambda: exception_raiser(ex) + self.assertEqual(1, self.__server.run('test')) + +if __name__== "__main__": + isc.log.init("bind10_server_test") + isc.log.resetUnitTestRootLogger() + unittest.main() From 8ec2caa761eac2681485217645e7e328924c3c7f Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Fri, 7 Jun 2013 17:26:19 -0700 Subject: [PATCH 02/35] [2854] added more behavior to BIND10Server class --- .../isc/server_common/bind10_server.py.in | 113 +++++++++++-- .../server_common/server_common_messages.mes | 6 +- .../isc/server_common/tests/Makefile.am | 1 + .../server_common/tests/bind10_server_test.py | 151 +++++++++++++++++- 4 files changed, 252 insertions(+), 19 deletions(-) diff --git a/src/lib/python/isc/server_common/bind10_server.py.in b/src/lib/python/isc/server_common/bind10_server.py.in index 51b3ba81dc..e810aec262 100644 --- a/src/lib/python/isc/server_common/bind10_server.py.in +++ b/src/lib/python/isc/server_common/bind10_server.py.in @@ -13,32 +13,77 @@ # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import errno import os +import select import signal import isc.log +import isc.config from isc.server_common.logger import logger from isc.log_messages.server_common_messages import * +class BIND10ServerFatal(Exception): + """Exception raised when the server program encounters a fatal error.""" + pass + class BIND10Server: """A mixin class for common BIND 10 server implementations. + It takes care of common initialization such as setting up a module CC + session, and running main event loop. It also handles the "shutdown" + command for its normal behavior. If a specific server class wants to + handle this command differently or if it does not support the command, + it should override the _command_handler method. + + Specific modules can define module-specific class inheriting this class, + instantiate it, and call run() with the module name. + Methods to be implemented in the actual class: + _config_handler: config handler method as specified in ModuleCCSession. + must be exception free; errors should be signaled by + the return value. + _mod_command_handler: can be optionally defined to handle + module-specific commands. should conform to + command handlers as specified in ModuleCCSession. + must be exception free; errors should be signaled + by the return value. """ # Will be set to True when the server should stop and shut down. # Can be read via accessor method 'shutdown', mainly for testing. __shutdown = False - __module_name = None + + # ModuleCCSession used in the server. Defined as 'protectd' so tests + # can refer to it directly; others should access it via the + # 'mod_ccsession' accessor. + _mod_cc = None + + # Will be set in run(). Define a tentative value so other methods can + # be tested directly. + __module_name = '' + + # Basically constant, but allow tests to override it. + _select_fn = select.select @property def shutdown(self): return self.__shutdown + @property + def mod_ccsession(self): + return self._mod_cc + def _setup_ccsession(self): - self._cc = isc.config.ModuleCCSession(self._get_specfile_location(), - self._config_handler, - self._command_handler) + """Create and start module CC session. + + This is essentially private, but allows tests to override it. + + """ + self._mod_cc = isc.config.ModuleCCSession( + self._get_specfile_location(), self._config_handler, + self._command_handler) + self._mod_cc.start() def _get_specfile_location(self): """Return the path to the module spec file following common convetion. @@ -54,10 +99,10 @@ class BIND10Server: specfile_path = os.environ['B10_FROM_SOURCE'] + '/src/bin/' + \ self.__module_name else: - specfile_path = '@datadir@/@PACKAGE@'\ - .replace('${datarootdir}', '@datarootdir@')\ - .replace('${prefix}', '@prefix@') - return specfile_path + '/' + self.__module_name + specfile_path = '${datarootdir}/bind10'\ + .replace('${datarootdir}', '${prefix}/share')\ + .replace('${prefix}', '/Users/jinmei/opt') + return specfile_path + '/' + self.__module_name + '.spec' def _trigger_shutdown(self): """Initiate a shutdown sequence. @@ -67,21 +112,53 @@ class BIND10Server: as possible to minimize side effects. Actual shutdown will take place in a normal control flow. - This method is defined as 'protected' so tests can call it; otherwise - it's private. + This method is defined as 'protected'. User classes can use it + to shut down the server. """ self.__shutdown = True def _run_internal(self): + """Main event loop. + + This method is essentially private, but allows tests to override it. + + """ + + logger.info(PYSERVER_COMMON_SERVER_STARTED, self.__module_name) + cc_fileno = self._mod_cc.get_socket().fileno() while not self.__shutdown: - pass + try: + (reads, _, _) = self._select_fn([cc_fileno], [], []) + except select.error as ex: + # ignore intterruption by signal; regard other select errors + # fatal. + if ex.args[0] == errno.EINTR: + continue + else: + raise + for fileno in reads: + if fileno == cc_fileno: + # this shouldn't raise an exception (if it does, we'll + # propagate it) + self._mod_cc.check_command(True) - def _config_handler(self): - pass + self._mod_cc.send_stopping() - def _command_handler(self): - pass + def _command_handler(self, cmd, args): + logger.debug(logger.DBGLVL_TRACE_BASIC, PYSERVER_COMMON_COMMAND, + self.__module_name, cmd) + if cmd == 'shutdown': + self._trigger_shutdown() + answer = isc.config.create_answer(0) + else: + answer = self._mod_command_handler(cmd, args) + + return answer + + def _mod_command_handler(self, cmd, args): + """The default implementation of the module specific command handler""" + return isc.config.create_answer(1, "Unknown command: " + str(cmd)) def run(self, module_name): """Start the server and let it run until it's told to stop. @@ -107,9 +184,13 @@ class BIND10Server: signal.signal(signal.SIGINT, shutdown_sighandler) self._setup_ccsession() self._run_internal() + logger.info(PYSERVER_COMMON_SERVER_STOPPED, self.__module_name) return 0 + except BIND10ServerFatal as ex: + logger.error(PYSERVER_COMMON_SERVER_FATAL, self.__module_name, + ex) except Exception as ex: logger.error(PYSERVER_COMMON_UNCAUGHT_EXCEPTION, type(ex).__name__, - str(ex)) + ex) return 1 diff --git a/src/lib/python/isc/server_common/server_common_messages.mes b/src/lib/python/isc/server_common/server_common_messages.mes index 24e221e3f5..ac3f6a9c45 100644 --- a/src/lib/python/isc/server_common/server_common_messages.mes +++ b/src/lib/python/isc/server_common/server_common_messages.mes @@ -47,11 +47,15 @@ The destination address and the total size of the message that has been transmitted so far (including the 2-byte length field) are shown in the log message. +% PYSERVER_COMMON_SERVER_FATAL %1 server has encountered a fatal error: %2 +The BIND 10 server process encountered a fatal error (normally specific to +the particular program), and is forcing itself to shut down. + % PYSERVER_COMMON_SERVER_STARTED %1 server has started The server process has successfully started and is now ready to receive commands and updates. -% PYSERVER_COMMON_SERVER_STOPPED %1 server has stopped +% PYSERVER_COMMON_SERVER_STOPPED %1 server has started The server process has successfully stopped and is no longer listening for or handling commands. Normally the process will soon exit. diff --git a/src/lib/python/isc/server_common/tests/Makefile.am b/src/lib/python/isc/server_common/tests/Makefile.am index c2667fdad6..bf6b26ce20 100644 --- a/src/lib/python/isc/server_common/tests/Makefile.am +++ b/src/lib/python/isc/server_common/tests/Makefile.am @@ -30,6 +30,7 @@ endif $(LIBRARY_PATH_PLACEHOLDER) \ PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/lib/dns/python/.libs \ B10_LOCKFILE_DIR_FROM_BUILD=$(abs_top_builddir) \ + B10_FROM_SOURCE=$(abs_top_srcdir) \ B10_FROM_BUILD=$(abs_top_builddir) \ $(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \ done diff --git a/src/lib/python/isc/server_common/tests/bind10_server_test.py b/src/lib/python/isc/server_common/tests/bind10_server_test.py index 19cf3356cf..14b94de802 100755 --- a/src/lib/python/isc/server_common/tests/bind10_server_test.py +++ b/src/lib/python/isc/server_common/tests/bind10_server_test.py @@ -14,6 +14,7 @@ # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import unittest +import errno import os import signal @@ -22,15 +23,47 @@ import isc.config from isc.server_common.bind10_server import * from isc.testutils.ccsession_mock import MockModuleCCSession +TEST_FILENO = 42 # arbitrarily chosen + class TestException(Exception): """A generic exception class specific in this test module.""" pass class MyCCSession(MockModuleCCSession, isc.config.ConfigData): def __init__(self, specfile, config_handler, command_handler): + # record parameter for later inspection + self.specfile_param = specfile + self.config_handler_param = config_handler + self.command_handler_param = command_handler + + self.check_command_param = None # used in check_command() + + # Initialize some local attributes of MockModuleCCSession, including + # 'stopped' + MockModuleCCSession.__init__(self) + + def start(self): pass + def check_command(self, nonblock): + """Mock check_command(). Just record the param for later inspection.""" + self.check_command_param = nonblock + + def get_socket(self): + return self + + def fileno(self): + """Pretending get_socket().fileno() + + Returing an arbitrarily chosen constant. + + """ + return TEST_FILENO + class MockServer(BIND10Server): + def __init__(self): + self._select_fn = self.select_wrapper + def _setup_ccsession(self): orig_cls = isc.config.ModuleCCSession isc.config.ModuleCCSession = MyCCSession @@ -41,6 +74,19 @@ class MockServer(BIND10Server): finally: isc.config.ModuleCCSession = orig_cls + def _config_handler(self): + pass + + def mod_command_handler(self, cmd, args): + """A sample _mod_command_handler implementation.""" + self.command_handler_params = (cmd, args) # for inspection + return isc.config.create_answer(0) + + def select_wrapper(self, reads, writes, errors): + self._trigger_shutdown() # make sure the loop will stop + self.select_params = (reads, writes, errors) # record for inspection + return [], [], [] + class TestBIND10Server(unittest.TestCase): def setUp(self): self.__server = MockServer() @@ -57,7 +103,7 @@ class TestBIND10Server(unittest.TestCase): """Check the signal handler behavior. SIGTERM and SIGINT should be caught and should call memmgr's - _trigger_shutdown(). This test also indirectly confirms main() calls + _trigger_shutdown(). This test also indirectly confirms run() calls run_internal(). """ @@ -82,10 +128,111 @@ class TestBIND10Server(unittest.TestCase): raise ex_cls('test') # Test all possible exceptions that are explicitly caught - for ex in [TestException]: + for ex in [TestException, BIND10ServerFatal]: self.__server._run_internal = lambda: exception_raiser(ex) self.assertEqual(1, self.__server.run('test')) + def test_run(self): + """Check other behavior of run()""" + self.__server._run_internal = lambda: None # prevent looping + self.assertEqual(0, self.__server.run('test')) + # module CC session should have been setup. + self.assertEqual(self.__server.mod_ccsession.specfile_param, + os.environ['B10_FROM_SOURCE'] + + '/src/bin/test/test.spec') + self.assertEqual(self.__server.mod_ccsession.config_handler_param, + self.__server._config_handler) + self.assertEqual(self.__server.mod_ccsession.command_handler_param, + self.__server._command_handler) + + def test_shutdown_command(self): + answer = self.__server._command_handler('shutdown', None) + self.assertTrue(self.__server.shutdown) + self.assertEqual((0, None), isc.config.parse_answer(answer)) + + def test_other_command(self): + self.__server._mod_command_handler = self.__server.mod_command_handler + answer = self.__server._command_handler('other command', None) + # shouldn't be confused with shutdown + self.assertFalse(self.__server.shutdown) + self.assertEqual((0, None), isc.config.parse_answer(answer)) + self.assertEqual(('other command', None), + self.__server.command_handler_params) + + def test_other_command_nohandler(self): + """Similar to test_other_command, but without explicit handler""" + # In this case "unknown command" error should be returned. + answer = self.__server._command_handler('other command', None) + self.assertEqual(1, isc.config.parse_answer(answer)[0]) + + def test_run_internal(self): + self.__server._setup_ccsession() + self.__server._run_internal() + self.assertEqual(([TEST_FILENO], [], []), self.__server.select_params) + + def select_wrapper(self, r, w, e, ex=None, ret=None): + """Mock select() function used some of the tests below. + + If ex is not None and it's first call to this method, it raises ex + assuming it's an exception. + + If ret is not None, it returns the given value; otherwise it returns + all empty lists. + + """ + self.select_params.append((r, w, e)) + if ex is not None and len(self.select_params) == 1: + raise ex + else: + self.__server._trigger_shutdown() + if ret is not None: + return ret + return [], [], [] + + def test_select_for_command(self): + """A normal event iteration, handling one command.""" + self.select_params = [] + self.__server._select_fn = \ + lambda r, w, e: self.select_wrapper(r, w, e, + ret=([TEST_FILENO], [], [])) + self.__server._setup_ccsession() + self.__server._run_internal() + # select should be called only once. + self.assertEqual([([TEST_FILENO], [], [])], self.select_params) + # check_command should have been called. + self.assertTrue(self.__server.mod_ccsession.check_command_param) + # module CC session should have been stopped explicitly. + self.assertTrue(self.__server.mod_ccsession.stopped) + + def test_select_interrupted(self): + """Emulating case select() raises EINTR.""" + self.select_params = [] + self.__server._select_fn = \ + lambda r, w, e: self.select_wrapper(r, w, e, + ex=select.error(errno.EINTR)) + self.__server._setup_ccsession() + self.__server._run_internal() + # EINTR will be ignored and select() will be called again. + self.assertEqual([([TEST_FILENO], [], []), ([TEST_FILENO], [], [])], + self.select_params) + # check_command() shouldn't have been called + self.assertIsNone(self.__server.mod_ccsession.check_command_param) + self.assertTrue(self.__server.mod_ccsession.stopped) + + def test_select_other_exception(self): + """Emulating case select() raises other select error.""" + self.select_params = [] + self.__server._select_fn = \ + lambda r, w, e: self.select_wrapper(r, w, e, + ex=select.error(errno.EBADF)) + self.__server._setup_ccsession() + # the exception will be propagated. + self.assertRaises(select.error, self.__server._run_internal) + self.assertEqual([([TEST_FILENO], [], [])], self.select_params) + # in this case module CC session hasn't been stopped explicitly + # others will notice it due to connection reset. + self.assertFalse(self.__server.mod_ccsession.stopped) + if __name__== "__main__": isc.log.init("bind10_server_test") isc.log.resetUnitTestRootLogger() From 94f2a0246ef707722666dc81c68cb58ac85414d4 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Fri, 7 Jun 2013 18:17:04 -0700 Subject: [PATCH 03/35] [2854] added to BIND10Server a hook for module-specific initialization. --- src/lib/python/isc/server_common/bind10_server.py.in | 11 +++++++++++ .../isc/server_common/tests/bind10_server_test.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/lib/python/isc/server_common/bind10_server.py.in b/src/lib/python/isc/server_common/bind10_server.py.in index e810aec262..a1f59ae5f9 100644 --- a/src/lib/python/isc/server_common/bind10_server.py.in +++ b/src/lib/python/isc/server_common/bind10_server.py.in @@ -48,6 +48,12 @@ class BIND10Server: command handlers as specified in ModuleCCSession. must be exception free; errors should be signaled by the return value. + _setup_module: can be optionally defined for module-specific + initialization. This is called after the module CC + session has started, and can be used for registering + interest on remote modules, etc. If it raises an + exception, the server will be immediatelly stopped. + Parameter: None, Return: None """ # Will be set to True when the server should stop and shut down. @@ -160,6 +166,10 @@ class BIND10Server: """The default implementation of the module specific command handler""" return isc.config.create_answer(1, "Unknown command: " + str(cmd)) + def _setup_module(self): + """The default implementation of the module specific initilization""" + pass + def run(self, module_name): """Start the server and let it run until it's told to stop. @@ -183,6 +193,7 @@ class BIND10Server: signal.signal(signal.SIGTERM, shutdown_sighandler) signal.signal(signal.SIGINT, shutdown_sighandler) self._setup_ccsession() + self._setup_module() self._run_internal() logger.info(PYSERVER_COMMON_SERVER_STOPPED, self.__module_name) return 0 diff --git a/src/lib/python/isc/server_common/tests/bind10_server_test.py b/src/lib/python/isc/server_common/tests/bind10_server_test.py index 14b94de802..d380a3fb47 100755 --- a/src/lib/python/isc/server_common/tests/bind10_server_test.py +++ b/src/lib/python/isc/server_common/tests/bind10_server_test.py @@ -145,6 +145,16 @@ class TestBIND10Server(unittest.TestCase): self.assertEqual(self.__server.mod_ccsession.command_handler_param, self.__server._command_handler) + def test_run_with_setup_module(self): + """Check run() with module specific setup method.""" + self.setup_called = False + def check_called(): + self.setup_called = True + self.__server._run_internal = lambda: None + self.__server._setup_module = check_called + self.assertEqual(0, self.__server.run('test')) + self.assertTrue(self.setup_called) + def test_shutdown_command(self): answer = self.__server._command_handler('shutdown', None) self.assertTrue(self.__server.shutdown) From 2379e411e9671d4ecbee71ca5a9663053a40d091 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Fri, 7 Jun 2013 22:31:27 -0700 Subject: [PATCH 04/35] [2854] updated specfile path guessing more reliable, trying FROM_BUILD. also loosen the check for the corresponding test so it reasonably works. --- .../isc/server_common/bind10_server.py.in | 20 ++++++++++++------- .../server_common/tests/bind10_server_test.py | 8 +++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/lib/python/isc/server_common/bind10_server.py.in b/src/lib/python/isc/server_common/bind10_server.py.in index a1f59ae5f9..2bac145bef 100644 --- a/src/lib/python/isc/server_common/bind10_server.py.in +++ b/src/lib/python/isc/server_common/bind10_server.py.in @@ -101,13 +101,19 @@ class BIND10Server: path for the spec file. """ - if 'B10_FROM_SOURCE' in os.environ: - specfile_path = os.environ['B10_FROM_SOURCE'] + '/src/bin/' + \ - self.__module_name - else: - specfile_path = '${datarootdir}/bind10'\ - .replace('${datarootdir}', '${prefix}/share')\ - .replace('${prefix}', '/Users/jinmei/opt') + # First check if it's running under an 'in-source' environment, + # then try commonly used paths and file names. If found, use it. + for ev in ['B10_FROM_SOURCE', 'B10_FROM_BUILD']: + if ev in os.environ: + specfile = os.environ[ev] + '/src/bin/' + self.__module_name +\ + '/' + self.__module_name + '.spec' + if os.path.exists(specfile): + return specfile + # Otherwise, just use the installed path, whether or not it really + # exists; leave error handling to the caller. + specfile_path = '${datarootdir}/bind10'\ + .replace('${datarootdir}', '${prefix}/share')\ + .replace('${prefix}', '/Users/jinmei/opt') return specfile_path + '/' + self.__module_name + '.spec' def _trigger_shutdown(self): diff --git a/src/lib/python/isc/server_common/tests/bind10_server_test.py b/src/lib/python/isc/server_common/tests/bind10_server_test.py index d380a3fb47..e20d59febd 100755 --- a/src/lib/python/isc/server_common/tests/bind10_server_test.py +++ b/src/lib/python/isc/server_common/tests/bind10_server_test.py @@ -137,9 +137,11 @@ class TestBIND10Server(unittest.TestCase): self.__server._run_internal = lambda: None # prevent looping self.assertEqual(0, self.__server.run('test')) # module CC session should have been setup. - self.assertEqual(self.__server.mod_ccsession.specfile_param, - os.environ['B10_FROM_SOURCE'] + - '/src/bin/test/test.spec') + # The exact path to the spec file can vary, so we simply check + # it works and it's the expected name stripping the path. + self.assertEqual( + self.__server.mod_ccsession.specfile_param.split('/')[-1], + 'test.spec') self.assertEqual(self.__server.mod_ccsession.config_handler_param, self.__server._config_handler) self.assertEqual(self.__server.mod_ccsession.command_handler_param, From 1b1c061696198766e9f6821e69cb1658f83c2030 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Mon, 10 Jun 2013 15:11:05 -0700 Subject: [PATCH 05/35] [2854] added get_clients_map to DataSrcClientsMgr to use in memmgr. --- .../isc/server_common/datasrc_clients_mgr.py | 19 +++++++++++++ .../tests/datasrc_clients_mgr_test.py | 27 +++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/lib/python/isc/server_common/datasrc_clients_mgr.py b/src/lib/python/isc/server_common/datasrc_clients_mgr.py index 75e6827574..946f8d974a 100644 --- a/src/lib/python/isc/server_common/datasrc_clients_mgr.py +++ b/src/lib/python/isc/server_common/datasrc_clients_mgr.py @@ -61,6 +61,25 @@ class DataSrcClientsMgr: self.__clients_map = {} self.__map_lock = threading.Lock() + def get_clients_map(self): + """Returns a dict from RR class to ConfigurableClientList. + + It corresponds to the generation of data source configuration at the + time of the call. It can be safely called while reconfigure() is + called from another thread. + + The mapping of the dict should be considered "frozen"; the caller + shouldn't modify the mapping (it can use the mapped objects in a + way modifying its internal state). + + Note: in a future version we may also need to return the + "generation ID" of the corresponding configuration so the caller + application can handle migration between generations gradually. + + """ + with self.__map_lock: + return self.__clients_map + def get_client_list(self, rrclass): """Return the configured ConfigurableClientList for the RR class. diff --git a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py index 827a417d73..9883256d56 100644 --- a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py +++ b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py @@ -79,12 +79,11 @@ class DataSrcClientsMgrTest(unittest.TestCase): self.__mgr.reconfigure, {"classes": {"IN": 42}}) self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH)) - def test_reconfig_while_using_old(self): - """Check datasrc client and finder can work even after list is gone.""" - self.__mgr.reconfigure(DEFAULT_CONFIG) - clist = self.__mgr.get_client_list(RRClass.CH) - self.__mgr.reconfigure({"classes": {"IN": []}}) + def check_client_list_content(self, clist): + """Some set of checks on given data source client list. + Used by a couple of tests below. + """ datasrc_client, finder, exact = clist.find(Name('bind')) self.assertTrue(exact) @@ -104,6 +103,24 @@ class DataSrcClientsMgrTest(unittest.TestCase): rrsets = datasrc_client.get_iterator(Name('bind')) self.assertNotEqual(0, len(list(rrsets))) + def test_reconfig_while_using_old(self): + """Check datasrc client and finder can work even after list is gone.""" + self.__mgr.reconfigure(DEFAULT_CONFIG) + clist = self.__mgr.get_client_list(RRClass.CH) + self.__mgr.reconfigure({"classes": {"IN": []}}) + self.check_client_list_content(clist) + + def test_get_clients_map(self): + # This is basically a trivial getter, so it should be sufficient + # to check we can call it as we expect. + self.__mgr.reconfigure(DEFAULT_CONFIG) + clients_map = self.__mgr.get_clients_map() + self.assertEqual(2, len(clients_map)) # should contain 'IN' and 'CH' + + # Check the retrieved map is usable even after further reconfig(). + self.__mgr.reconfigure({"classes": {"IN": []}}) + self.check_client_list_content(clients_map[RRClass.CH]) + if __name__ == "__main__": isc.log.init("bind10") isc.log.resetUnitTestRootLogger() From 0fa66c2662c0f8eac63ed44668f90d8b687859c4 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Mon, 10 Jun 2013 15:18:56 -0700 Subject: [PATCH 06/35] [2854] update doc for DataSrcClientsMgr on use_cache ctor parameter it will now be used by memmgr, so there's at least one exception. --- src/lib/python/isc/server_common/datasrc_clients_mgr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/python/isc/server_common/datasrc_clients_mgr.py b/src/lib/python/isc/server_common/datasrc_clients_mgr.py index 946f8d974a..aefefdfab6 100644 --- a/src/lib/python/isc/server_common/datasrc_clients_mgr.py +++ b/src/lib/python/isc/server_common/datasrc_clients_mgr.py @@ -40,10 +40,11 @@ class DataSrcClientsMgr: def __init__(self, use_cache=False): """Constructor. - In the initial implementation, user applications of this class are - generally expected to NOT use in-memory cache; use_cache would be - set to True only for tests. In future, some applications such as - outbound zone transfer may want to set it to True. + In the initial implementation, most user applications of this class + are generally expected to NOT use in-memory cache; the only expected + exception is the memory (cache) manager, which, by definition, + needs to deal with in-memory data. In future, some more applications + such as outbound zone transfer may want to set it to True. Parameter: use_cache (bool): If set to True, enable in-memory cache on From 425062cfead674420744cee41784e4d894f9ac2d Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Mon, 10 Jun 2013 16:20:08 -0700 Subject: [PATCH 07/35] [2854] added tentative support for datasrc config generation IDs. --- .../isc/server_common/datasrc_clients_mgr.py | 14 ++++++++++++-- .../tests/datasrc_clients_mgr_test.py | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/lib/python/isc/server_common/datasrc_clients_mgr.py b/src/lib/python/isc/server_common/datasrc_clients_mgr.py index aefefdfab6..6b6b8ce0df 100644 --- a/src/lib/python/isc/server_common/datasrc_clients_mgr.py +++ b/src/lib/python/isc/server_common/datasrc_clients_mgr.py @@ -62,8 +62,14 @@ class DataSrcClientsMgr: self.__clients_map = {} self.__map_lock = threading.Lock() + # The generation ID of the configuration corresponding to + # current __clinets_map. Until we support the concept of generations + # in the configuration framework, we tentatively maintain it within + # this class. + self.__gen_id = 0 + def get_clients_map(self): - """Returns a dict from RR class to ConfigurableClientList. + """Returns a dict from RR class to ConfigurableClientList with gen ID. It corresponds to the generation of data source configuration at the time of the call. It can be safely called while reconfigure() is @@ -79,7 +85,7 @@ class DataSrcClientsMgr: """ with self.__map_lock: - return self.__clients_map + return (self.__gen_id, self.__clients_map) def get_client_list(self, rrclass): """Return the configured ConfigurableClientList for the RR class. @@ -149,6 +155,10 @@ class DataSrcClientsMgr: new_map[rrclass] = new_client_list with self.__map_lock: self.__clients_map = new_map + + # NOTE: when we support the concept of generations this should + # be retrieved from the configuration + self.__gen_id += 1 except Exception as ex: # Catch all types of exceptions as a whole: there won't be much # granularity for exceptions raised from the C++ module anyway. diff --git a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py index 9883256d56..d045971d8f 100644 --- a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py +++ b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py @@ -113,14 +113,22 @@ class DataSrcClientsMgrTest(unittest.TestCase): def test_get_clients_map(self): # This is basically a trivial getter, so it should be sufficient # to check we can call it as we expect. + + # Initially map iss empty, the generation ID is 0. + self.assertEqual((0, {}), self.__mgr.get_clients_map()) + self.__mgr.reconfigure(DEFAULT_CONFIG) - clients_map = self.__mgr.get_clients_map() + genid, clients_map = self.__mgr.get_clients_map() + self.assertEqual(1, genid) self.assertEqual(2, len(clients_map)) # should contain 'IN' and 'CH' # Check the retrieved map is usable even after further reconfig(). self.__mgr.reconfigure({"classes": {"IN": []}}) self.check_client_list_content(clients_map[RRClass.CH]) + # generation ID should be incremented again + self.assertEqual(2, self.__mgr.get_clients_map()[0]) + if __name__ == "__main__": isc.log.init("bind10") isc.log.resetUnitTestRootLogger() From 054d97ceb5c12455b1272e00a860e111f67b6ce9 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Wed, 12 Jun 2013 10:42:20 -0700 Subject: [PATCH 08/35] [2854] corrected DataSrcClientsMgr.reconfigure() so it'll work for the default. 'new_config' can be empty for the initial default and won't work as expected. --- .../isc/server_common/datasrc_clients_mgr.py | 14 ++++++++--- .../tests/datasrc_clients_mgr_test.py | 25 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/lib/python/isc/server_common/datasrc_clients_mgr.py b/src/lib/python/isc/server_common/datasrc_clients_mgr.py index 6b6b8ce0df..4f565dfffc 100644 --- a/src/lib/python/isc/server_common/datasrc_clients_mgr.py +++ b/src/lib/python/isc/server_common/datasrc_clients_mgr.py @@ -117,7 +117,7 @@ class DataSrcClientsMgr: client_list = self.__clients_map.get(rrclass) return client_list - def reconfigure(self, config): + def reconfigure(self, new_config, config_data): """(Re)configure the set of client lists. This method takes a new set of data source configuration, builds @@ -142,12 +142,20 @@ class DataSrcClientsMgr: at the same time. Parameter: - config (dict): configuration data for the data_sources module. + new_config (dict): configuration data for the data_sources module + (actually unused in this method). + config_data (isc.config.ConfigData): the latest full config data + for the data_sources module. Usually the second parameter of + the (remote) configuration update callback for the module. """ try: new_map = {} - for rrclass_cfg, class_cfg in config.get('classes').items(): + # We only refer to config_data, not new_config (diff from the + # previous). the latter may be empty for the initial default + # configuration while the former works for all cases. + for rrclass_cfg, class_cfg in \ + config_data.get_value('classes')[0].items(): rrclass = isc.dns.RRClass(rrclass_cfg) new_client_list = isc.datasrc.ConfigurableClientList(rrclass) new_client_list.configure(json.dumps(class_cfg), diff --git a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py index d045971d8f..e717d6554e 100644 --- a/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py +++ b/src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py @@ -32,6 +32,8 @@ class DataSrcClientsMgrTest(unittest.TestCase): # We construct the manager with enabling in-memory cache for easier # tests. There should be no risk of inter-thread issues in the tests. self.__mgr = DataSrcClientsMgr(use_cache=True) + self.__datasrc_cfg = isc.config.ConfigData( + isc.config.module_spec_from_file(DATASRC_SPECFILE)) def test_init(self): """Check some initial state. @@ -52,31 +54,33 @@ class DataSrcClientsMgrTest(unittest.TestCase): # There should be at least in-memory only data for the static # bind/CH zone. (We don't assume the existence of SQLite3 datasrc, # so it'll still work if and when we make the default DB-independent). - self.__mgr.reconfigure(DEFAULT_CONFIG) + self.__mgr.reconfigure({}, self.__datasrc_cfg) clist = self.__mgr.get_client_list(RRClass.CH) self.assertIsNotNone(clist) self.assertTrue(clist.find(Name('bind'), True, False)[2]) # Reconfigure it with a simple new config: the list for CH will be # gone, and and an empty list for IN will be installed. - self.__mgr.reconfigure({"classes": {"IN": []}}) + self.__datasrc_cfg.set_local_config({"classes": {"IN": []}}) + self.__mgr.reconfigure({}, self.__datasrc_cfg) self.assertIsNone(self.__mgr.get_client_list(RRClass.CH)) self.assertIsNotNone(self.__mgr.get_client_list(RRClass.IN)) def test_reconfigure_error(self): """Check reconfigure failure preserves the old config.""" # Configure it with the default - self.__mgr.reconfigure(DEFAULT_CONFIG) + self.__mgr.reconfigure({}, self.__datasrc_cfg) self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH)) # Then try invalid configuration - self.assertRaises(ConfigError, self.__mgr.reconfigure, 42) + self.assertRaises(ConfigError, self.__mgr.reconfigure, {}, 42) self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH)) # Another type of invalid configuration: exception would come from # the C++ wrapper. + self.__datasrc_cfg.set_local_config({"classes": {"IN": 42}}) self.assertRaises(ConfigError, - self.__mgr.reconfigure, {"classes": {"IN": 42}}) + self.__mgr.reconfigure, {}, self.__datasrc_cfg) self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH)) def check_client_list_content(self, clist): @@ -105,9 +109,11 @@ class DataSrcClientsMgrTest(unittest.TestCase): def test_reconfig_while_using_old(self): """Check datasrc client and finder can work even after list is gone.""" - self.__mgr.reconfigure(DEFAULT_CONFIG) + self.__mgr.reconfigure({}, self.__datasrc_cfg) clist = self.__mgr.get_client_list(RRClass.CH) - self.__mgr.reconfigure({"classes": {"IN": []}}) + + self.__datasrc_cfg.set_local_config({"classes": {"IN": []}}) + self.__mgr.reconfigure({}, self.__datasrc_cfg) self.check_client_list_content(clist) def test_get_clients_map(self): @@ -117,13 +123,14 @@ class DataSrcClientsMgrTest(unittest.TestCase): # Initially map iss empty, the generation ID is 0. self.assertEqual((0, {}), self.__mgr.get_clients_map()) - self.__mgr.reconfigure(DEFAULT_CONFIG) + self.__mgr.reconfigure({}, self.__datasrc_cfg) genid, clients_map = self.__mgr.get_clients_map() self.assertEqual(1, genid) self.assertEqual(2, len(clients_map)) # should contain 'IN' and 'CH' # Check the retrieved map is usable even after further reconfig(). - self.__mgr.reconfigure({"classes": {"IN": []}}) + self.__datasrc_cfg.set_local_config({"classes": {"IN": []}}) + self.__mgr.reconfigure({}, self.__datasrc_cfg) self.check_client_list_content(clients_map[RRClass.CH]) # generation ID should be incremented again From 554c01b75c2e1a16bebd00a2d1476b0c85f70ef3 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Wed, 12 Jun 2013 10:44:18 -0700 Subject: [PATCH 09/35] [2854] made trivial updates to xfrin for the correction to reconfigure() --- src/bin/xfrin/tests/xfrin_test.py | 8 ++++---- src/bin/xfrin/xfrin.py.in | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/xfrin/tests/xfrin_test.py b/src/bin/xfrin/tests/xfrin_test.py index bfcbdcb584..c431f4c600 100644 --- a/src/bin/xfrin/tests/xfrin_test.py +++ b/src/bin/xfrin/tests/xfrin_test.py @@ -256,11 +256,11 @@ class MockDataSrcClientsMgr(): def get_client_list(self, rrclass): return self.found_datasrc_client_list - def reconfigure(self, arg1): + def reconfigure(self, arg1, arg2): # the only current test simply needs to know this is called with - # the expected argument and exceptions are handled. if we need more + # the expected arguments and exceptions are handled. if we need more # variations in tests, this mock method should be extended. - self.reconfigure_param.append(arg1) + self.reconfigure_param.append((arg1, arg2)) raise isc.server_common.datasrc_clients_mgr.ConfigError( 'reconfigure failure') @@ -3038,7 +3038,7 @@ class TestXfrin(unittest.TestCase): # we just check it's called as expected, and the only possible # exception doesn't cause disruption. self.xfr._datasrc_config_handler(True, False) - self.assertEqual([True], + self.assertEqual([(True, False)], self.xfr._datasrc_clients_mgr.reconfigure_param) def raise_interrupt(): diff --git a/src/bin/xfrin/xfrin.py.in b/src/bin/xfrin/xfrin.py.in index bcf96afe48..6ff28050ed 100755 --- a/src/bin/xfrin/xfrin.py.in +++ b/src/bin/xfrin/xfrin.py.in @@ -1459,7 +1459,7 @@ class Xfrin: """ try: - self._datasrc_clients_mgr.reconfigure(new_config) + self._datasrc_clients_mgr.reconfigure(new_config, config_data) except isc.server_common.datasrc_clients_mgr.ConfigError as ex: logger.error(XFRIN_DATASRC_CONFIG_ERROR, ex) From 2987b196ad617e5cc8f759717edae50866162888 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Tue, 11 Jun 2013 16:35:05 -0700 Subject: [PATCH 10/35] [2854] add new helper python pkg/module for isc.memmgr.datasrc_info --- configure.ac | 2 + src/lib/python/isc/Makefile.am | 2 +- src/lib/python/isc/memmgr/Makefile.am | 10 + src/lib/python/isc/memmgr/__init__.py | 0 src/lib/python/isc/memmgr/datasrc_info.py | 218 ++++++++++++++++++ src/lib/python/isc/memmgr/tests/Makefile.am | 34 +++ .../isc/memmgr/tests/datasrc_info_tests.py | 185 +++++++++++++++ 7 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 src/lib/python/isc/memmgr/Makefile.am create mode 100644 src/lib/python/isc/memmgr/__init__.py create mode 100644 src/lib/python/isc/memmgr/datasrc_info.py create mode 100644 src/lib/python/isc/memmgr/tests/Makefile.am create mode 100644 src/lib/python/isc/memmgr/tests/datasrc_info_tests.py diff --git a/configure.ac b/configure.ac index 559178ba6d..f0a5fb14c2 100644 --- a/configure.ac +++ b/configure.ac @@ -1264,6 +1264,8 @@ AC_CONFIG_FILES([Makefile src/lib/python/isc/bind10/tests/Makefile src/lib/python/isc/ddns/Makefile src/lib/python/isc/ddns/tests/Makefile + src/lib/python/isc/memmgr/Makefile + src/lib/python/isc/memmgr/tests/Makefile src/lib/python/isc/xfrin/Makefile src/lib/python/isc/xfrin/tests/Makefile src/lib/python/isc/server_common/Makefile diff --git a/src/lib/python/isc/Makefile.am b/src/lib/python/isc/Makefile.am index 712843e4d9..7b7d768f4b 100644 --- a/src/lib/python/isc/Makefile.am +++ b/src/lib/python/isc/Makefile.am @@ -1,5 +1,5 @@ SUBDIRS = datasrc util cc config dns log net notify testutils acl bind10 -SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics +SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics memmgr python_PYTHON = __init__.py diff --git a/src/lib/python/isc/memmgr/Makefile.am b/src/lib/python/isc/memmgr/Makefile.am new file mode 100644 index 0000000000..efb4742719 --- /dev/null +++ b/src/lib/python/isc/memmgr/Makefile.am @@ -0,0 +1,10 @@ +SUBDIRS = . tests + +python_PYTHON = __init__.py datasrc_info.py + +pythondir = $(pyexecdir)/isc/memmgr + +CLEANDIRS = __pycache__ + +clean-local: + rm -rf $(CLEANDIRS) diff --git a/src/lib/python/isc/memmgr/__init__.py b/src/lib/python/isc/memmgr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/lib/python/isc/memmgr/datasrc_info.py b/src/lib/python/isc/memmgr/datasrc_info.py new file mode 100644 index 0000000000..ecb7fce66b --- /dev/null +++ b/src/lib/python/isc/memmgr/datasrc_info.py @@ -0,0 +1,218 @@ +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import os +import json + +class SegmentInfoError(Exception): + """An exception raised for general errors in the SegmentInfo class.""" + pass + +class SegmentInfo: + """A base class to maintain information about memory segments. + + An instance of this class corresponds to the memory segment used + for in-memory cache of a specific single data source. It manages + information to set/reset the latest effective segment (such as + path to a memory mapped file) and sets of other modules using the + segment. + + Since there can be several different types of memory segments, + the top level class provides abstract interfaces independent from + segment-type specific details. Such details are expected to be + delegated to subclasses corresponding to specific types of segments. + + The implementation is still incomplete. It will have more attributes + such as a set of current readers, methods for adding or deleting + the readers. These will probably be implemented in this base class + as they will be independent from segment-type specific details. + + """ + # Common constants of user type: reader or writer + READER = 0 + WRITER = 1 + + def create(type, genid, rrclass, datasrc_name, mgr_config): + """Factory of specific SegmentInfo subclass instance based on the + segment type. + + This is specifically for the memmgr, and segments that are not of + its interest will be ignored. This method returns None in these + cases. At least 'local' type segments will be ignored this way. + + If an unknown type of segment is specified, this method throws an + SegmentInfoError exception. The assumption is that this method + is called after the corresponding data source configuration has been + validated, at which point such unknown segments should have been + rejected. + + Parameters: + type (str or None): The type of memory segment; None if the segment + isn't used. + genid (int): The generation ID of the corresponding data source + configuration. + rrclass (isc.dns.RRClass): The RR class of the data source. + datasrc_name (str): The name of the data source. + mgr_config (dict): memmgr configuration related to memory segment + information. The content of the dict is type specific; each + subclass is expected to know which key is necessary and the + semantics of its value. + + """ + if type == 'mapped': + return MappedSegmentInfo(genid, rrclass, datasrc_name, mgr_config) + elif type is None or type == 'local': + return None + raise SegmentInfoError('unknown segment type to create info: ' + type) + + def get_reset_param(self, user_type): + """Return parameters to reset the zone table memory segment. + + It returns a json expression in string that contains parameters for + the specified type of user to reset a zone table segment with + isc.datasrc.ConfigurableClientList.reset_memory_segment(). + It can also be passed to the user module as part of command + parameters. + + Each subclass must implement this method. + + Parameter: + user_type (READER or WRITER): specifies the type of user to reset + the segment. + + """ + raise SegmentInfoError('get_reset_param is not implemented') + + def switch_versions(self): + """Switch internal information for the reader segment and writer + segment. + + This method is expected to be called when the writer on one version + of memory segment completes updates and the memmgr is going to + have readers switch to the updated version. Details of the + information to be switched would depend on the segment type, and + are delegated to the specific subclass. + + Each subclass must implement this method. + + """ + raise SegmentInfoError('switch_versions is not implemented') + +class MappedSegmentInfo(SegmentInfo): + """SegmentInfo implementation of 'mapped' type memory segments. + + It maintains paths to mapped files both readers and the writer. + + While objets of this class are expected to be shared by multiple + threads, it assumes operations are serialized through message passing, + so access to this class itself is not protected by any explicit + synchronization mechanism. + + """ + def __init__(self, genid, rrclass, datasrc_name, mgr_config): + super().__init__() + + # Something like "/var/bind10/zone-IN-1-sqlite3-mapped" + self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \ + 'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \ + '-mapped' + + # Current versions (suffix of the mapped files) for readers and the + # writer. In this initial implementation we assume that all possible + # readers are waiting for a new version (not using pre-existing one), + # and the writer is expected to build a new segment as version "0". + self.__reader_ver = None # => 0 => 1 => 0 => 1 ... + self.__writer_ver = 0 # => 1 => 0 => 1 => 0 ... + + def get_reset_param(self, user_type): + ver = self.__reader_ver if user_type == self.READER else \ + self.__writer_ver + if ver is None: + return None + mapped_file = self.__mapped_file_base + '.' + str(ver) + return json.dumps({'mapped-file': mapped_file}) + + def switch_versions(self): + # Swith the versions as noted in the constructor. + self.__writer_ver ^= 1 + + if self.__reader_ver is None: + self.__reader_ver = 0 + else: + self.__reader_ver ^= 1 + + # Versions should be different + assert(self.__reader_ver != self.__writer_ver) + +class DataSrcInfo: + """A container for datasrc.ConfigurableClientLists and associated + in-memory segment information corresponding to a given geration of + configuration. + + This class maintains all datasrc.ConfigurableClientLists in a form + of dict from RR classes corresponding to a given single generation + of data source configuration, along with sets of memory segment + information that needs to be used by memmgr. + + Once constructed, mappings do not change (different generation of + configuration will result in another DataSrcInfo objects). Status + of SegmentInfo objects stored in this class object may change over time. + + Attributes: these are all constant and read only. For dict objects, + mapping shouldn't be modified either. + gen_id (int): The corresponding configuration generation ID. + clients_map (dict, isc.dns.RRClass=>isc.datasrc.ConfigurableClientList): + The configured client lists for all RR classes of the generation. + segment_info_map (dict, (isc.dns.RRClass, str)=>SegmentInfo): + SegmentInfo objects managed in the DataSrcInfo objects. Can be + retrieved by (RRClass, ). + + """ + def __init__(self, genid, clients_map, mgr_config): + """Constructor. + + As long as given parameters are of valid type and have been + validated, this constructor shouldn't raise an exception. + + Parameters: + genid (int): see gen_id attribute + clients_map (dict): see clients_map attribute + mgr_config (dict, str=>key-dependent-value): A copy of the current + memmgr configuration, in case it's needed to construct a specific + type of SegmentInfo. The specific SegmentInfo class is expected + to know the key-value mappings that it needs. + + """ + self.__gen_id = genid + self.__clients_map = clients_map + self.__segment_info_map = {} + for (rrclass, client_list) in clients_map.items(): + for (name, sgmt_type, _) in client_list.get_status(): + sgmt_info = SegmentInfo.create(sgmt_type, genid, rrclass, name, + mgr_config) + if sgmt_info is not None: + self.__segment_info_map[(rrclass, name)] = sgmt_info + + @property + def gen_id(self): + return self.__gen_id + + @property + def clients_map(self): + return self.__clients_map + + @property + def segment_info_map(self): + return self.__segment_info_map diff --git a/src/lib/python/isc/memmgr/tests/Makefile.am b/src/lib/python/isc/memmgr/tests/Makefile.am new file mode 100644 index 0000000000..1f78d9f62d --- /dev/null +++ b/src/lib/python/isc/memmgr/tests/Makefile.am @@ -0,0 +1,34 @@ +PYCOVERAGE_RUN = @PYCOVERAGE_RUN@ +PYTESTS = datasrc_info_tests.py +EXTRA_DIST = $(PYTESTS) + +# If necessary (rare cases), explicitly specify paths to dynamic libraries +# required by loadable python modules. +if SET_ENV_LIBRARY_PATH +LIBRARY_PATH_PLACEHOLDER = $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/cryptolink/.libs:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/cc/.libs:$(abs_top_builddir)/src/lib/config/.libs:$(abs_top_builddir)/src/lib/log/.libs:$(abs_top_builddir)/src/lib/util/.libs:$(abs_top_builddir)/src/lib/util/threads/.libs:$(abs_top_builddir)/src/lib/acl/.libs:$(abs_top_builddir)/src/lib/exceptions/.libs:$(abs_top_builddir)/src/lib/datasrc/.libs:$$$(ENV_LIBRARY_PATH) +endif + +# Some tests require backend shared memory support +if USE_SHARED_MEMORY +HAVE_SHARED_MEMORY=yes +else +HAVE_SHARED_MEMORY=no +endif + +# test using command-line arguments, so use check-local target instead of TESTS +# B10_FROM_BUILD is necessary to load data source backend from the build tree. +check-local: +if ENABLE_PYTHON_COVERAGE + touch $(abs_top_srcdir)/.coverage + rm -f .coverage + ${LN_S} $(abs_top_srcdir)/.coverage .coverage +endif + for pytest in $(PYTESTS) ; do \ + echo Running test: $$pytest ; \ + $(LIBRARY_PATH_PLACEHOLDER) \ + TESTDATA_PATH=$(builddir) \ + B10_FROM_BUILD=$(abs_top_builddir) \ + HAVE_SHARED_MEMORY=$(HAVE_SHARED_MEMORY) \ + PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/lib/dns/python/.libs \ + $(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \ + done diff --git a/src/lib/python/isc/memmgr/tests/datasrc_info_tests.py b/src/lib/python/isc/memmgr/tests/datasrc_info_tests.py new file mode 100644 index 0000000000..ba1322c6c1 --- /dev/null +++ b/src/lib/python/isc/memmgr/tests/datasrc_info_tests.py @@ -0,0 +1,185 @@ +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import json +import os +import unittest + +from isc.dns import * +import isc.config +import isc.datasrc +import isc.log +from isc.server_common.datasrc_clients_mgr import DataSrcClientsMgr +from isc.memmgr.datasrc_info import * + +class TestSegmentInfo(unittest.TestCase): + def setUp(self): + self.__mapped_file_dir = os.environ['TESTDATA_PATH'] + self.__sgmt_info = SegmentInfo.create('mapped', 0, RRClass.IN, + 'sqlite3', + {'mapped_file_dir': + self.__mapped_file_dir}) + + def __check_sgmt_reset_param(self, user_type, expected_ver): + """Common check on the return value of get_reset_param() for + MappedSegmentInfo. + + Unless it's expected to return None, it should be a map that + maps "mapped-file" to the expected version of mapped-file. + + """ + if expected_ver is None: + self.assertIsNone(self.__sgmt_info.get_reset_param(user_type)) + return + param = json.loads(self.__sgmt_info.get_reset_param(user_type)) + self.assertEqual(self.__mapped_file_dir + + '/zone-IN-0-sqlite3-mapped.' + str(expected_ver), + param['mapped-file']) + + def test_initial_params(self): + self.__check_sgmt_reset_param(SegmentInfo.WRITER, 0) + self.__check_sgmt_reset_param(SegmentInfo.READER, None) + + def test_swtich_versions(self): + self.__sgmt_info.switch_versions() + self.__check_sgmt_reset_param(SegmentInfo.WRITER, 1) + self.__check_sgmt_reset_param(SegmentInfo.READER, 0) + + self.__sgmt_info.switch_versions() + self.__check_sgmt_reset_param(SegmentInfo.WRITER, 0) + self.__check_sgmt_reset_param(SegmentInfo.READER, 1) + + def test_init_others(self): + # For local type of segment, information isn't needed and won't be + # created. + self.assertIsNone(SegmentInfo.create('local', 0, RRClass.IN, + 'sqlite3', {})) + + # Unknown type of segment will result in an exception. + self.assertRaises(SegmentInfoError, SegmentInfo.create, 'unknown', 0, + RRClass.IN, 'sqlite3', {}) + + def test_missing_methods(self): + # Bad subclass of SegmentInfo that doesn't implement mandatory methods. + class TestSegmentInfo(SegmentInfo): + pass + + self.assertRaises(SegmentInfoError, + TestSegmentInfo().get_reset_param, + SegmentInfo.WRITER) + self.assertRaises(SegmentInfoError, TestSegmentInfo().switch_versions) + +class MockClientList: + """A mock ConfigurableClientList class. + + Just providing minimal shortcut interfaces needed for DataSrcInfo class. + + """ + def __init__(self, status_list): + self.__status_list = status_list + + def get_status(self): + return self.__status_list + +class TestDataSrcInfo(unittest.TestCase): + def setUp(self): + self.__mapped_file_dir = os.environ['TESTDATA_PATH'] + self.__mgr_config = {'mapped_file_dir': self.__mapped_file_dir} + self.__sqlite3_dbfile = os.environ['TESTDATA_PATH'] + '/' + 'zone.db' + self.__clients_map = { + # mixture of 'local' and 'mapped' and 'unused' (type =None) + # segments + RRClass.IN: MockClientList([('datasrc1', 'local', None), + ('datasrc2', 'mapped', None), + ('datasrc3', None, None)]), + RRClass.CH: MockClientList([('datasrc2', 'mapped', None), + ('datasrc1', 'local', None)]) } + + def tearDown(self): + if os.path.exists(self.__sqlite3_dbfile): + os.unlink(self.__sqlite3_dbfile) + + def __check_sgmt_reset_param(self, sgmt_info, writer_file): + # Check if the initial state of (mapped) segment info object has + # expected values. + self.assertIsNone(sgmt_info.get_reset_param(SegmentInfo.READER)) + param = json.loads(sgmt_info.get_reset_param(SegmentInfo.WRITER)) + self.assertEqual(writer_file, param['mapped-file']) + + def test_init(self): + """Check basic scenarios of constructing DataSrcInfo.""" + + # This checks that all data sources of all RR classes are covered, + # "local" segments are ignored, info objects for "mapped" segments + # are created and stored in segment_info_map. + datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config) + self.assertEqual(42, datasrc_info.gen_id) + self.assertEqual(self.__clients_map, datasrc_info.clients_map) + self.assertEqual(2, len(datasrc_info.segment_info_map)) + sgmt_info = datasrc_info.segment_info_map[(RRClass.IN, 'datasrc2')] + self.__check_sgmt_reset_param(sgmt_info, self.__mapped_file_dir + + '/zone-IN-42-datasrc2-mapped.0') + sgmt_info = datasrc_info.segment_info_map[(RRClass.CH, 'datasrc2')] + self.__check_sgmt_reset_param(sgmt_info, self.__mapped_file_dir + + '/zone-CH-42-datasrc2-mapped.0') + + # A case where clist.get_status() returns an empty list; shouldn't + # cause disruption + self.__clients_map = { RRClass.IN: MockClientList([])} + datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config) + self.assertEqual(42, datasrc_info.gen_id) + self.assertEqual(0, len(datasrc_info.segment_info_map)) + + # A case where clients_map is empty; shouldn't cause disruption + self.__clients_map = {} + datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config) + self.assertEqual(42, datasrc_info.gen_id) + self.assertEqual(0, len(datasrc_info.segment_info_map)) + + def test_production(self): + """Check the behavior closer to a production environment. + + Instead of using a mock classes, just for confirming we didn't miss + something. + + """ + # This test uses real "mmaped" segment and doesn't work without + # shared memory support + if os.environ['HAVE_SHARED_MEMORY'] != 'yes': + return + + datasrc_config = { + "classes": { + "IN": [{"type": "sqlite3", "cache-enable": True, + "cache-type": "mapped", "cache-zones": [], + "params": {"database_file": self.__sqlite3_dbfile}}] + } + } + cmgr = DataSrcClientsMgr(use_cache=True) + cmgr.reconfigure(datasrc_config) + + genid, clients_map = cmgr.get_clients_map() + datasrc_info = DataSrcInfo(genid, clients_map, self.__mgr_config) + + self.assertEqual(1, datasrc_info.gen_id) + self.assertEqual(clients_map, datasrc_info.clients_map) + self.assertEqual(1, len(datasrc_info.segment_info_map)) + sgmt_info = datasrc_info.segment_info_map[(RRClass.IN, 'sqlite3')] + self.assertIsNone(sgmt_info.get_reset_param(SegmentInfo.READER)) + +if __name__ == "__main__": + isc.log.init("bind10-test") + isc.log.resetUnitTestRootLogger() + unittest.main() From 2ca7ef10889eccf7ebe70f1662fed83c33f62c19 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Thu, 6 Jun 2013 17:15:27 -0700 Subject: [PATCH 11/35] [2854] added a minimal framework for the memmgr daemon. it's basically empty yet. --- configure.ac | 3 + src/bin/memmgr/.gitignore | 3 + src/bin/memmgr/Makefile.am | 49 +++++++++++++++ src/bin/memmgr/b10-memmgr.xml | 95 +++++++++++++++++++++++++++++ src/bin/memmgr/memmgr.py.in | 22 +++++++ src/bin/memmgr/memmgr.spec | 23 +++++++ src/bin/memmgr/memmgr_messages.mes | 17 ++++++ src/bin/memmgr/tests/Makefile.am | 30 +++++++++ src/bin/memmgr/tests/memmgr_test.py | 21 +++++++ 9 files changed, 263 insertions(+) create mode 100644 src/bin/memmgr/.gitignore create mode 100644 src/bin/memmgr/Makefile.am create mode 100644 src/bin/memmgr/b10-memmgr.xml create mode 100755 src/bin/memmgr/memmgr.py.in create mode 100644 src/bin/memmgr/memmgr.spec create mode 100644 src/bin/memmgr/memmgr_messages.mes create mode 100644 src/bin/memmgr/tests/Makefile.am create mode 100755 src/bin/memmgr/tests/memmgr_test.py diff --git a/configure.ac b/configure.ac index f0a5fb14c2..ac1f756e74 100644 --- a/configure.ac +++ b/configure.ac @@ -1192,6 +1192,8 @@ AC_CONFIG_FILES([Makefile src/bin/loadzone/Makefile src/bin/loadzone/tests/Makefile src/bin/loadzone/tests/correct/Makefile + src/bin/memmgr/Makefile + src/bin/memmgr/tests/Makefile src/bin/msgq/Makefile src/bin/msgq/tests/Makefile src/bin/auth/Makefile @@ -1379,6 +1381,7 @@ AC_OUTPUT([doc/version.ent src/bin/loadzone/loadzone.py src/bin/usermgr/run_b10-cmdctl-usermgr.sh src/bin/usermgr/b10-cmdctl-usermgr.py + src/bin/memmgr/memmgr.py src/bin/msgq/msgq.py src/bin/msgq/run_msgq.sh src/bin/auth/auth.spec.pre diff --git a/src/bin/memmgr/.gitignore b/src/bin/memmgr/.gitignore new file mode 100644 index 0000000000..9b12294ee7 --- /dev/null +++ b/src/bin/memmgr/.gitignore @@ -0,0 +1,3 @@ +/b10-memmgr +/memmgr.py +/b10-memmgr.8 diff --git a/src/bin/memmgr/Makefile.am b/src/bin/memmgr/Makefile.am new file mode 100644 index 0000000000..8fc1cdcf8d --- /dev/null +++ b/src/bin/memmgr/Makefile.am @@ -0,0 +1,49 @@ +SUBDIRS = . tests + +pkglibexecdir = $(libexecdir)/@PACKAGE@ + +pkglibexec_SCRIPTS = b10-memmgr + +b10_memmgrdir = $(pkgdatadir) +b10_memmgr_DATA = memmgr.spec + +nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py +pylogmessagedir = $(pyexecdir)/isc/log_messages/ + +CLEANFILES = b10-memmgr memmgr.pyc +CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py +CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.pyc + +EXTRA_DIST = memmgr_messages.mes memmgr.spec + +man_MANS = b10-memmgr.8 +DISTCLEANFILES = $(man_MANS) +EXTRA_DIST += $(man_MANS) b10-memmgr.xml + +if GENERATE_DOCS + +b10-memmgr.8: b10-memmgr.xml + @XSLTPROC@ --novalid --xinclude --nonet -o $@ http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $(srcdir)/b10-memmgr.xml + +else + +$(man_MANS): + @echo Man generation disabled. Creating dummy $@. Configure with --enable-generate-docs to enable it. + @echo Man generation disabled. Remove this file, configure with --enable-generate-docs, and rebuild BIND 10 > $@ + +endif + +# Define rule to build logging source files from message file +$(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py : memmgr_messages.mes + $(top_builddir)/src/lib/log/compiler/message \ + -d $(PYTHON_LOGMSGPKG_DIR)/work -p $(srcdir)/memmgr_messages.mes + +# this is done here since configure.ac AC_OUTPUT doesn't expand exec_prefix +b10-memmgr: memmgr.py $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py + $(SED) -e "s|@@PYTHONPATH@@|@pyexecdir@|" memmgr.py >$@ + chmod a+x $@ + +CLEANDIRS = __pycache__ + +clean-local: + rm -rf $(CLEANDIRS) diff --git a/src/bin/memmgr/b10-memmgr.xml b/src/bin/memmgr/b10-memmgr.xml new file mode 100644 index 0000000000..2f3ae67d22 --- /dev/null +++ b/src/bin/memmgr/b10-memmgr.xml @@ -0,0 +1,95 @@ +]> + + + + + + June 4, 2012 + + + + b10-memmgr + 8 + BIND10 + + + + b10-memmgr + BIND 10 memory manager daemon + + + + + 2013 + Internet Systems Consortium, Inc. ("ISC") + + + + + + b10-memmgr + + + + + DESCRIPTION + + + + ARGUMENTS + + The arguments are as follows: + + + + + + , + + + + + Print the command line arguments and exit. + + + + + + CONFIGURATION AND COMMANDS + + + + + SEE ALSO + + + bind108 + , + BIND 10 Guide. + + + + + HISTORY + + diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in new file mode 100755 index 0000000000..5f21334671 --- /dev/null +++ b/src/bin/memmgr/memmgr.py.in @@ -0,0 +1,22 @@ +#!@PYTHON@ + +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +import sys; sys.path.append('@@PYTHONPATH@@') + +if '__main__' == __name__: + pass diff --git a/src/bin/memmgr/memmgr.spec b/src/bin/memmgr/memmgr.spec new file mode 100644 index 0000000000..247b012386 --- /dev/null +++ b/src/bin/memmgr/memmgr.spec @@ -0,0 +1,23 @@ +{ + "module_spec": { + "module_name": "Memmgr", + "config_data": [ + { + } + ], + "commands": [ + { + "command_name": "shutdown", + "command_description": "Shut down Memmgr", + "command_args": [ + { + "item_name": "pid", + "item_type": "integer", + "item_optional": true + } + ] + } + ] + } +} + diff --git a/src/bin/memmgr/memmgr_messages.mes b/src/bin/memmgr/memmgr_messages.mes new file mode 100644 index 0000000000..a59bcc35e2 --- /dev/null +++ b/src/bin/memmgr/memmgr_messages.mes @@ -0,0 +1,17 @@ +# Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC") +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +# When you add a message to this file, it is a good idea to run +# /tools/reorder_message_file.py to make sure the +# messages are in the correct order. diff --git a/src/bin/memmgr/tests/Makefile.am b/src/bin/memmgr/tests/Makefile.am new file mode 100644 index 0000000000..347ff87e76 --- /dev/null +++ b/src/bin/memmgr/tests/Makefile.am @@ -0,0 +1,30 @@ +PYCOVERAGE_RUN=@PYCOVERAGE_RUN@ +PYTESTS = memmgr_test.py +EXTRA_DIST = $(PYTESTS) + +# If necessary (rare cases), explicitly specify paths to dynamic libraries +# required by loadable python modules. +LIBRARY_PATH_PLACEHOLDER = +if SET_ENV_LIBRARY_PATH +LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/cryptolink/.libs:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/cc/.libs:$(abs_top_builddir)/src/lib/config/.libs:$(abs_top_builddir)/src/lib/log/.libs:$(abs_top_builddir)/src/lib/util/.libs:$(abs_top_builddir)/src/lib/util/threads/.libs:$(abs_top_builddir)/src/lib/exceptions/.libs:$(abs_top_builddir)/src/lib/util/io/.libs:$(abs_top_builddir)/src/lib/datasrc/.libs:$(abs_top_builddir)/src/lib/acl/.libs:$$$(ENV_LIBRARY_PATH) +endif + +# test using command-line arguments, so use check-local target instead of TESTS +# We set B10_FROM_BUILD below, so that the test can refer to the in-source +# spec file. +check-local: +if ENABLE_PYTHON_COVERAGE + touch $(abs_top_srcdir)/.coverage + rm -f .coverage + ${LN_S} $(abs_top_srcdir)/.coverage .coverage +endif + for pytest in $(PYTESTS) ; do \ + echo Running test: $$pytest ; \ + B10_FROM_SOURCE=$(abs_top_srcdir) \ + B10_FROM_BUILD=$(abs_top_builddir) \ + $(LIBRARY_PATH_PLACEHOLDER) \ + PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/bin/memmgr:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/util/io/.libs \ + TESTDATASRCDIR=$(abs_srcdir)/testdata/ \ + TESTDATA_PATH=$(abs_top_srcdir)/src/lib/testutils/testdata \ + $(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \ + done diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py new file mode 100755 index 0000000000..3699fcf26e --- /dev/null +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -0,0 +1,21 @@ +# Copyright (C) 2013 Internet Systems Consortium. +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM +# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import isc.log +import unittest + +if __name__== "__main__": + isc.log.resetUnitTestRootLogger() + unittest.main() From 81847206409e7cb4bb69bda8a22f7fc54a7a6b2a Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Thu, 6 Jun 2013 19:05:49 -0700 Subject: [PATCH 12/35] [2854] memmgr log setup --- src/lib/python/isc/log_messages/Makefile.am | 2 ++ src/lib/python/isc/log_messages/memmgr_messages.py | 1 + 2 files changed, 3 insertions(+) create mode 100644 src/lib/python/isc/log_messages/memmgr_messages.py diff --git a/src/lib/python/isc/log_messages/Makefile.am b/src/lib/python/isc/log_messages/Makefile.am index c8b9c7a975..caf1e3b772 100644 --- a/src/lib/python/isc/log_messages/Makefile.am +++ b/src/lib/python/isc/log_messages/Makefile.am @@ -4,6 +4,7 @@ EXTRA_DIST = __init__.py EXTRA_DIST += init_messages.py EXTRA_DIST += cmdctl_messages.py EXTRA_DIST += ddns_messages.py +EXTRA_DIST += memmgr_messages.py EXTRA_DIST += stats_messages.py EXTRA_DIST += stats_httpd_messages.py EXTRA_DIST += xfrin_messages.py @@ -24,6 +25,7 @@ CLEANFILES = __init__.pyc CLEANFILES += init_messages.pyc CLEANFILES += cmdctl_messages.pyc CLEANFILES += ddns_messages.pyc +CLEANFILES += memmgr_messages.pyc CLEANFILES += stats_messages.pyc CLEANFILES += stats_httpd_messages.pyc CLEANFILES += xfrin_messages.pyc diff --git a/src/lib/python/isc/log_messages/memmgr_messages.py b/src/lib/python/isc/log_messages/memmgr_messages.py new file mode 100644 index 0000000000..8c59cc9092 --- /dev/null +++ b/src/lib/python/isc/log_messages/memmgr_messages.py @@ -0,0 +1 @@ +from work.memmgr_messages import * From 11de0085d24cdb5dfcc5a158cd30699cfde90159 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Thu, 6 Jun 2013 19:19:04 -0700 Subject: [PATCH 13/35] [2854] added some minimal behavior for memgr. --- src/bin/memmgr/memmgr.py.in | 20 ++++++++++++++++++-- src/bin/memmgr/memmgr.spec | 3 --- src/bin/memmgr/memmgr_messages.mes | 2 ++ src/bin/memmgr/tests/memmgr_test.py | 12 +++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in index 5f21334671..c59e7f36a9 100755 --- a/src/bin/memmgr/memmgr.py.in +++ b/src/bin/memmgr/memmgr.py.in @@ -15,8 +15,24 @@ # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import os +import sys +import signal -import sys; sys.path.append('@@PYTHONPATH@@') +sys.path.append('@@PYTHONPATH@@') +import isc.log +from isc.log_messages.memmgr_messages import * +from isc.server_common.bind10_server import BIND10Server + +MODULE_NAME = 'memmgr' + +isc.log.init('b10-memmgr', buffer=True) +logger = isc.log.Logger(MODULE_NAME) + +class Memmgr(BIND10Server): + def _config_handler(self, new_config): + pass if '__main__' == __name__: - pass + mgr = Memmgr() + sys.exit(mgr.run(MODULE_NAME)) diff --git a/src/bin/memmgr/memmgr.spec b/src/bin/memmgr/memmgr.spec index 247b012386..09a1971d76 100644 --- a/src/bin/memmgr/memmgr.spec +++ b/src/bin/memmgr/memmgr.spec @@ -2,8 +2,6 @@ "module_spec": { "module_name": "Memmgr", "config_data": [ - { - } ], "commands": [ { @@ -20,4 +18,3 @@ ] } } - diff --git a/src/bin/memmgr/memmgr_messages.mes b/src/bin/memmgr/memmgr_messages.mes index a59bcc35e2..df0872d261 100644 --- a/src/bin/memmgr/memmgr_messages.mes +++ b/src/bin/memmgr/memmgr_messages.mes @@ -15,3 +15,5 @@ # When you add a message to this file, it is a good idea to run # /tools/reorder_message_file.py to make sure the # messages are in the correct order. + +% MEMMGR_CONFIG_UPDATE received new configuration: %1 diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py index 3699fcf26e..82f5906dd4 100755 --- a/src/bin/memmgr/tests/memmgr_test.py +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -13,9 +13,19 @@ # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -import isc.log import unittest +import isc.log +import isc.config +import memmgr + +class MockMemmgr(memmgr.Memmgr): + pass + +class TestMemmgr(unittest.TestCase): + def setUp(self): + self.__mgr = MockMemmgr() + if __name__== "__main__": isc.log.resetUnitTestRootLogger() unittest.main() From ab3f32eee4e5eb1d696ac88392497e52b994fbbb Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Fri, 7 Jun 2013 18:07:36 -0700 Subject: [PATCH 14/35] [2854] add path to memmgr to the run_bind10 script --- src/bin/bind10/run_bind10.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/bind10/run_bind10.sh.in b/src/bin/bind10/run_bind10.sh.in index a22f300ecc..09c97084b6 100755 --- a/src/bin/bind10/run_bind10.sh.in +++ b/src/bin/bind10/run_bind10.sh.in @@ -20,7 +20,7 @@ export PYTHON_EXEC BIND10_PATH=@abs_top_builddir@/src/bin/bind10 -PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/resolver:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/stats:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:@abs_top_builddir@/src/bin/ddns:@abs_top_builddir@/src/bin/dhcp6:@abs_top_builddir@/src/bin/sockcreator:$PATH +PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/resolver:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/stats:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:@abs_top_builddir@/src/bin/ddns:@abs_top_builddir@/src/bin/dhcp6:@abs_top_builddir@/src/bin/sockcreator:@abs_top_builddir@/src/bin/memmgr:$PATH export PATH PYTHONPATH=@abs_top_builddir@/src/lib/python/isc/log_messages:@abs_top_builddir@/src/lib/python/isc/cc:@abs_top_builddir@/src/lib/python:@abs_top_builddir@/src/lib/dns/python/.libs:@abs_top_builddir@/src/lib/xfr/.libs:@abs_top_builddir@/src/lib/log/.libs:@abs_top_builddir@/src/lib/util/io/.libs:@abs_top_builddir@/src/lib/python/isc/config:@abs_top_builddir@/src/lib/python/isc/acl/.libs:@abs_top_builddir@/src/lib/python/isc/datasrc/.libs From 983f11cfdd11153b234ff7626c5ff1dccb1baeb2 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Mon, 10 Jun 2013 10:58:41 -0700 Subject: [PATCH 15/35] [2854] handle memmgr configuration; it's just mapped_file_dir for now. --- configure.ac | 1 + src/bin/memmgr/.gitignore | 1 + src/bin/memmgr/Makefile.am | 5 +- src/bin/memmgr/memmgr.py.in | 64 +++++++++++++++ .../{memmgr.spec => memmgr.spec.pre.in} | 5 ++ src/bin/memmgr/memmgr_messages.mes | 9 +- src/bin/memmgr/tests/memmgr_test.py | 82 ++++++++++++++++++- 7 files changed, 164 insertions(+), 3 deletions(-) rename src/bin/memmgr/{memmgr.spec => memmgr.spec.pre.in} (67%) diff --git a/configure.ac b/configure.ac index ac1f756e74..285d09e74c 100644 --- a/configure.ac +++ b/configure.ac @@ -1382,6 +1382,7 @@ AC_OUTPUT([doc/version.ent src/bin/usermgr/run_b10-cmdctl-usermgr.sh src/bin/usermgr/b10-cmdctl-usermgr.py src/bin/memmgr/memmgr.py + src/bin/memmgr/memmgr.spec.pre src/bin/msgq/msgq.py src/bin/msgq/run_msgq.sh src/bin/auth/auth.spec.pre diff --git a/src/bin/memmgr/.gitignore b/src/bin/memmgr/.gitignore index 9b12294ee7..3743d584db 100644 --- a/src/bin/memmgr/.gitignore +++ b/src/bin/memmgr/.gitignore @@ -1,3 +1,4 @@ /b10-memmgr /memmgr.py +/memmgr.spec /b10-memmgr.8 diff --git a/src/bin/memmgr/Makefile.am b/src/bin/memmgr/Makefile.am index 8fc1cdcf8d..94a380a0ea 100644 --- a/src/bin/memmgr/Makefile.am +++ b/src/bin/memmgr/Makefile.am @@ -14,7 +14,7 @@ CLEANFILES = b10-memmgr memmgr.pyc CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.pyc -EXTRA_DIST = memmgr_messages.mes memmgr.spec +EXTRA_DIST = memmgr_messages.mes man_MANS = b10-memmgr.8 DISTCLEANFILES = $(man_MANS) @@ -38,6 +38,9 @@ $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py : memmgr_messages.mes $(top_builddir)/src/lib/log/compiler/message \ -d $(PYTHON_LOGMSGPKG_DIR)/work -p $(srcdir)/memmgr_messages.mes +memmgr.spec: memmgr.spec.pre + $(SED) -e "s|@@LOCALSTATEDIR@@|$(localstatedir)|" memmgr.spec.pre > $@ + # this is done here since configure.ac AC_OUTPUT doesn't expand exec_prefix b10-memmgr: memmgr.py $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py $(SED) -e "s|@@PYTHONPATH@@|@pyexecdir@|" memmgr.py >$@ diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in index c59e7f36a9..99a5c1a89b 100755 --- a/src/bin/memmgr/memmgr.py.in +++ b/src/bin/memmgr/memmgr.py.in @@ -21,6 +21,7 @@ import signal sys.path.append('@@PYTHONPATH@@') import isc.log +#from isc.log import DBGLVL_TRACE_BASIC from isc.log_messages.memmgr_messages import * from isc.server_common.bind10_server import BIND10Server @@ -29,8 +30,71 @@ MODULE_NAME = 'memmgr' isc.log.init('b10-memmgr', buffer=True) logger = isc.log.Logger(MODULE_NAME) +class ConfigError(Exception): + """An exception class raised for configuration errors of Memmgr.""" + pass + class Memmgr(BIND10Server): + def __init__(self): + # configurable parameter: initially this is the only param, so + # we only maintain as a single attribute. As the class is extended + # and more configurable, consider introducing a compound type or + # class. + # This is defined as "protected" so tests can inspect it; others + # shouldn't use it directly. + self._mapped_file_dir = None + def _config_handler(self, new_config): + """Configuration handler, called via BIND10Server. + + This method must be exception free. We assume minimum validity + about the parameter, though: it should be a valid dict, and conform + to the type specification of the spec file. + + """ + logger.debug(logger.DBGLVL_TRACE_BASIC, MEMMGR_CONFIG_UPDATE) + + # Default answer: + answer = isc.config.create_answer(0) + + # If this is the first time, initialize the local attributes with the + # latest full config data, which consist of the defaults with + # possibly overridden by user config. Otherwise, just apply the latest + # diff. + if self._mapped_file_dir is None: + new_config = self.mod_ccsession.get_full_config() + try: + self.__update_config(new_config) + except Exception as ex: + logger.error(MEMMGR_CONFIG_FAIL, ex) + answer = isc.config.create_answer( + 1, 'Memmgr failed to apply configuration updates: ' + str(ex)) + + return answer + + def __update_config(self, new_config): + """Apply config changes to local attributes. + + This is a subroutine of _config_handler. It's supposed to provide + strong exception guarantee: either all changes successfully apply + or, if any error is found, none applies. In the latter case the + entire original configuration should be kept. + + Errors are to be reported as an exception. + + """ + new_mapped_file_dir = new_config.get('mapped_file_dir') + if new_mapped_file_dir is not None: + if not os.path.isdir(new_mapped_file_dir): + raise ConfigError('mapped_file_dir is not a directory: ' + + new_mapped_file_dir) + if not os.access(new_mapped_file_dir, os.W_OK): + raise ConfigError('mapped_file_dir is not writable: ' + + new_mapped_file_dir) + self._mapped_file_dir = new_mapped_file_dir + + def _setup_module(self): + """Module specific initialization for BIND10Server.""" pass if '__main__' == __name__: diff --git a/src/bin/memmgr/memmgr.spec b/src/bin/memmgr/memmgr.spec.pre.in similarity index 67% rename from src/bin/memmgr/memmgr.spec rename to src/bin/memmgr/memmgr.spec.pre.in index 09a1971d76..6729690512 100644 --- a/src/bin/memmgr/memmgr.spec +++ b/src/bin/memmgr/memmgr.spec.pre.in @@ -2,6 +2,11 @@ "module_spec": { "module_name": "Memmgr", "config_data": [ + { "item_name": "mapped_file_dir", + "item_type": "string", + "item_optional": true, + "item_default": "@@LOCALSTATEDIR@@/@PACKAGE@/mapped_files" + } ], "commands": [ { diff --git a/src/bin/memmgr/memmgr_messages.mes b/src/bin/memmgr/memmgr_messages.mes index df0872d261..30ee52ed0b 100644 --- a/src/bin/memmgr/memmgr_messages.mes +++ b/src/bin/memmgr/memmgr_messages.mes @@ -16,4 +16,11 @@ # /tools/reorder_message_file.py to make sure the # messages are in the correct order. -% MEMMGR_CONFIG_UPDATE received new configuration: %1 +% MEMMGR_CONFIG_FAIL failed to apply configuration updates: %1 +The memmgr daemon tried to apply configuration updates but found an error. +The cause of the error is included in the message. None of the received +updates applied, and the daemon keeps running with the previous configuration. + +% MEMMGR_CONFIG_UPDATE received new configuration +A debug message. The memmgr daemon receives configuratiopn updates +and is now applying them to its running configurations. diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py index 82f5906dd4..eb7a9d882b 100755 --- a/src/bin/memmgr/tests/memmgr_test.py +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -14,17 +14,97 @@ # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import unittest +import os +import re import isc.log import isc.config +from isc.config import parse_answer import memmgr +from isc.testutils.ccsession_mock import MockModuleCCSession + +class MyCCSession(MockModuleCCSession, isc.config.ConfigData): + def __init__(self, specfile, config_handler, command_handler): + super().__init__() + specfile = os.environ['B10_FROM_BUILD'] + '/src/bin/memmgr/memmgr.spec' + module_spec = isc.config.module_spec_from_file(specfile) + isc.config.ConfigData.__init__(self, module_spec) + + def start(self): + pass class MockMemmgr(memmgr.Memmgr): - pass + def _setup_ccsession(self): + orig_cls = isc.config.ModuleCCSession + isc.config.ModuleCCSession = MyCCSession + try: + super()._setup_ccsession() + except Exception: + raise + finally: + isc.config.ModuleCCSession = orig_cls class TestMemmgr(unittest.TestCase): def setUp(self): + # Some tests use this directory. Make sure it doesn't pre-exist. + self.__test_mapped_file_dir = \ + os.environ['B10_FROM_BUILD'] + \ + '/src/bin/memmgr/tests/test_mapped_files' + if os.path.isdir(self.__test_mapped_file_dir): + os.rmdir(self.__test_mapped_file_dir) + self.__mgr = MockMemmgr() + # Fake some 'os' module functions for easier tests + self.__orig_os_access = os.access + self.__orig_isdir = os.path.isdir + + def tearDown(self): + # Restore faked values + os.access = self.__orig_os_access + os.path.isdir = self.__orig_isdir + + # If at test created a mapped-files directory, delete it. + if os.path.isdir(self.__test_mapped_file_dir): + os.rmdir(self.__test_mapped_file_dir) + + def test_init(self): + self.assertIsNone(self.__mgr._mapped_file_dir) + + def test_configure(self): + self.__mgr._setup_ccsession() + + # Pretend specified directories exist and writable + os.path.isdir = lambda x: True + os.access = lambda x, y: True + + # At the initial configuration, if mapped_file_dir isn't specified, + # the default value will be set. + self.assertEqual((0, None), + parse_answer(self.__mgr._config_handler({}))) + self.assertEqual('mapped_files', + self.__mgr._mapped_file_dir.split('/')[-1]) + + # Update the configuration. + user_cfg = {'mapped_file_dir': '/some/path/dir'} + self.assertEqual((0, None), + parse_answer(self.__mgr._config_handler(user_cfg))) + self.assertEqual('/some/path/dir', self.__mgr._mapped_file_dir) + + # Bad update: diretory doesn't exist (we assume it really doesn't + # exist in the tested environment). Update won't be made. + os.path.isdir = self.__orig_isdir # use real library + user_cfg = {'mapped_file_dir': '/another/path/dir'} + answer = parse_answer(self.__mgr._config_handler(user_cfg)) + self.assertEqual(1, answer[0]) + self.assertIsNotNone(re.search('not a directory', answer[1])) + + # Bad update: directory exists but is not readable. + os.mkdir(self.__test_mapped_file_dir, 0o500) # drop writable bit + os.access = self.__orig_os_access + user_cfg = {'mapped_file_dir': self.__test_mapped_file_dir} + answer = parse_answer(self.__mgr._config_handler(user_cfg)) + self.assertEqual(1, answer[0]) + self.assertIsNotNone(re.search('not writable', answer[1])) if __name__== "__main__": isc.log.resetUnitTestRootLogger() From e65eaed0d786f8939852b6368de827f7294d396c Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Mon, 10 Jun 2013 14:56:23 -0700 Subject: [PATCH 16/35] [2854] make the default mapped_files directory by make install --- src/bin/memmgr/Makefile.am | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bin/memmgr/Makefile.am b/src/bin/memmgr/Makefile.am index 94a380a0ea..a1fb2097c8 100644 --- a/src/bin/memmgr/Makefile.am +++ b/src/bin/memmgr/Makefile.am @@ -48,5 +48,14 @@ b10-memmgr: memmgr.py $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py CLEANDIRS = __pycache__ +# install the default directory for memory-mapped files. Note that the +# path must be identical to the default value in memmgr.spec. We'll make +# it readable only for the owner to minimize the risk of accidents. +install-data-local: + $(mkinstalldirs) $(DESTDIR)@localstatedir@/@PACKAGE@/mapped_files + +install-data-hook: + -chmod 700 $(DESTDIR)@localstatedir@/@PACKAGE@/mapped_files + clean-local: rm -rf $(CLEANDIRS) From 007b1f5cd7b48bccf501aa624325388e360aa828 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Tue, 11 Jun 2013 16:42:43 -0700 Subject: [PATCH 17/35] [2854] update makefile so memmgr will be built and tested --- src/bin/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/Makefile.am b/src/bin/Makefile.am index bdef8303c5..bf6182772c 100644 --- a/src/bin/Makefile.am +++ b/src/bin/Makefile.am @@ -1,5 +1,5 @@ SUBDIRS = bind10 bindctl cfgmgr ddns loadzone msgq cmdctl auth xfrin \ xfrout usermgr zonemgr stats tests resolver sockcreator dhcp4 dhcp6 d2\ - dbutil sysinfo + dbutil sysinfo memmgr check-recursive: all-recursive From a66549d4455e23d4486d9287157790ccd49a942c Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Tue, 11 Jun 2013 22:35:57 -0700 Subject: [PATCH 18/35] [2854] complete _setup_module() with adding datasrc config. --- src/bin/memmgr/memmgr.py.in | 17 +++++++++++++++-- src/bin/memmgr/memmgr_messages.mes | 7 +++++++ src/bin/memmgr/tests/memmgr_test.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in index 99a5c1a89b..51392ea1f2 100755 --- a/src/bin/memmgr/memmgr.py.in +++ b/src/bin/memmgr/memmgr.py.in @@ -22,8 +22,9 @@ import signal sys.path.append('@@PYTHONPATH@@') import isc.log #from isc.log import DBGLVL_TRACE_BASIC +from isc.config import ModuleSpecError, ModuleCCSessionError from isc.log_messages.memmgr_messages import * -from isc.server_common.bind10_server import BIND10Server +from isc.server_common.bind10_server import BIND10Server, BIND10ServerFatal MODULE_NAME = 'memmgr' @@ -95,7 +96,19 @@ class Memmgr(BIND10Server): def _setup_module(self): """Module specific initialization for BIND10Server.""" - pass + try: + # memmgr isn't usable if data source is not configured, and + # as long as cfgmgr is ready there's no timing issue. So we + # immediately shut it down if it's missing. See ddns.py.in + # about exceptions to catch. + self.mod_ccsession.add_remote_config_by_name( + 'data_sources', self._datasrc_config_handler) + except (ModuleSpecError, ModuleCCSessionError) as ex: + logger.error(MEMMGR_NO_DATASRC_CONF, ex) + raise BIND10ServerFatal('failed to setup memmgr module') + + def _datasrc_config_handler(self, new_config, config_data): + return if '__main__' == __name__: mgr = Memmgr() diff --git a/src/bin/memmgr/memmgr_messages.mes b/src/bin/memmgr/memmgr_messages.mes index 30ee52ed0b..0c8fb27ee7 100644 --- a/src/bin/memmgr/memmgr_messages.mes +++ b/src/bin/memmgr/memmgr_messages.mes @@ -24,3 +24,10 @@ updates applied, and the daemon keeps running with the previous configuration. % MEMMGR_CONFIG_UPDATE received new configuration A debug message. The memmgr daemon receives configuratiopn updates and is now applying them to its running configurations. + +% MEMMGR_NO_DATASRC_CONF failed to add data source configuration: %1 +The memmgr daemon tried to incorporate data source configuration +on its startup but failed to do so. The most likely cause of this +is that data sources are not simply configured. If so, they must be. +The memmgr daemon cannot do any meaningful work without data sources, +so it immediately terminates itself. diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py index eb7a9d882b..b979b67a7d 100755 --- a/src/bin/memmgr/tests/memmgr_test.py +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -29,10 +29,17 @@ class MyCCSession(MockModuleCCSession, isc.config.ConfigData): specfile = os.environ['B10_FROM_BUILD'] + '/src/bin/memmgr/memmgr.spec' module_spec = isc.config.module_spec_from_file(specfile) isc.config.ConfigData.__init__(self, module_spec) + self.add_remote_params = [] # for inspection + self.add_remote_exception = None # to raise exception from the method def start(self): pass + def add_remote_config_by_name(self, mod_name, handler): + if self.add_remote_exception is not None: + raise self.add_remote_exception + self.add_remote_params.append((mod_name, handler)) + class MockMemmgr(memmgr.Memmgr): def _setup_ccsession(self): orig_cls = isc.config.ModuleCCSession @@ -106,6 +113,27 @@ class TestMemmgr(unittest.TestCase): self.assertEqual(1, answer[0]) self.assertIsNotNone(re.search('not writable', answer[1])) + def test_setup_module(self): + # _setup_module should add data_sources remote module with + # expected parameters. + self.__mgr._setup_ccsession() + self.assertEqual([], self.__mgr.mod_ccsession.add_remote_params) + self.__mgr._setup_module() + self.assertEqual([('data_sources', + self.__mgr._datasrc_config_handler)], + self.__mgr.mod_ccsession.add_remote_params) + + # If data source isn't configured it's considered fatal. + self.__mgr.mod_ccsession.add_remote_exception = \ + isc.config.ModuleCCSessionError('faked exception') + self.assertRaises(isc.server_common.bind10_server.BIND10ServerFatal, + self.__mgr._setup_module) + + self.__mgr.mod_ccsession.add_remote_exception = \ + isc.config.ModuleSpecError('faked exception') + self.assertRaises(isc.server_common.bind10_server.BIND10ServerFatal, + self.__mgr._setup_module) + if __name__== "__main__": isc.log.resetUnitTestRootLogger() unittest.main() From 1d9ea23f74983eb952733a9cacd5cdc6d99db5d4 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Tue, 11 Jun 2013 23:41:48 -0700 Subject: [PATCH 19/35] [2854] complete Memmgr._datasrc_config_handler(). --- src/bin/memmgr/memmgr.py.in | 31 ++++++++++++- src/bin/memmgr/memmgr_messages.mes | 11 +++++ src/bin/memmgr/tests/memmgr_test.py | 71 +++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in index 51392ea1f2..f2c7c8cc9c 100755 --- a/src/bin/memmgr/memmgr.py.in +++ b/src/bin/memmgr/memmgr.py.in @@ -25,12 +25,18 @@ import isc.log from isc.config import ModuleSpecError, ModuleCCSessionError from isc.log_messages.memmgr_messages import * from isc.server_common.bind10_server import BIND10Server, BIND10ServerFatal +from isc.server_common.datasrc_clients_mgr \ + import DataSrcClientsMgr, ConfigError +from isc.memmgr.datasrc_info import DataSrcInfo +import isc.util.process MODULE_NAME = 'memmgr' isc.log.init('b10-memmgr', buffer=True) logger = isc.log.Logger(MODULE_NAME) +isc.util.process.rename() + class ConfigError(Exception): """An exception class raised for configuration errors of Memmgr.""" pass @@ -45,6 +51,14 @@ class Memmgr(BIND10Server): # shouldn't use it directly. self._mapped_file_dir = None + # The manager to keep track of data source configuration. Allow + # tests to inspect/tweak it. + self._datasrc_clients_mgr = DataSrcClientsMgr(use_cache=True) + + # List of DataSrcInfo. Used as a queue to maintain info for all + # active configuration generations. Allow tests to inspec it. + self._datasrc_info_list = [] + def _config_handler(self, new_config): """Configuration handler, called via BIND10Server. @@ -108,7 +122,22 @@ class Memmgr(BIND10Server): raise BIND10ServerFatal('failed to setup memmgr module') def _datasrc_config_handler(self, new_config, config_data): - return + """Callback of data_sources configuration update. + + This method must be exception free, so we catch all expected + exceptions internally; unexpected ones should mean a programming + error and will terminate the program. + + """ + try: + self._datasrc_clients_mgr.reconfigure(new_config, config_data) + genid, clients_map = self._datasrc_clients_mgr.get_clients_map() + datasrc_info = DataSrcInfo(genid, clients_map, + {'mapped_file_dir': + self._mapped_file_dir}) + self._datasrc_info_list.append(datasrc_info) + except isc.server_common.datasrc_clients_mgr.ConfigError as ex: + logger.error(MEMMGR_DATASRC_CONFIG_ERROR, ex) if '__main__' == __name__: mgr = Memmgr() diff --git a/src/bin/memmgr/memmgr_messages.mes b/src/bin/memmgr/memmgr_messages.mes index 0c8fb27ee7..4648fa2937 100644 --- a/src/bin/memmgr/memmgr_messages.mes +++ b/src/bin/memmgr/memmgr_messages.mes @@ -25,6 +25,17 @@ updates applied, and the daemon keeps running with the previous configuration. A debug message. The memmgr daemon receives configuratiopn updates and is now applying them to its running configurations. +% MEMMGR_DATASRC_CONFIG_ERROR failed to update data source configuration: %1 +Configuration for the global data sources is updated, but the update +cannot be applied to memmgr. The memmgr module will still keep running +with the previous configuration, but the cause of the failure and +other log messages must be carefully examined because if only memmgr +rejects the new configuration then the entire BIND 10 system will have +inconsistent state among different modules. If other modules accept +the update but memmgr produces this error, it's quite likely that the +system isn't working as expected, and is probably better to be shut down +to figure out and fix the cause. + % MEMMGR_NO_DATASRC_CONF failed to add data source configuration: %1 The memmgr daemon tried to incorporate data source configuration on its startup but failed to do so. The most likely cause of this diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py index b979b67a7d..2d2fc6023f 100755 --- a/src/bin/memmgr/tests/memmgr_test.py +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -18,6 +18,7 @@ import os import re import isc.log +from isc.dns import RRClass import isc.config from isc.config import parse_answer import memmgr @@ -51,6 +52,15 @@ class MockMemmgr(memmgr.Memmgr): finally: isc.config.ModuleCCSession = orig_cls +# Defined for easier tests with DataSrcClientsMgr.reconfigure(), which +# only needs get_value() method +class MockConfigData: + def __init__(self, data): + self.__data = data + + def get_value(self, identifier): + return self.__data[identifier], False + class TestMemmgr(unittest.TestCase): def setUp(self): # Some tests use this directory. Make sure it doesn't pre-exist. @@ -75,7 +85,20 @@ class TestMemmgr(unittest.TestCase): os.rmdir(self.__test_mapped_file_dir) def test_init(self): + """Check some initial conditions""" self.assertIsNone(self.__mgr._mapped_file_dir) + self.assertEqual([], self.__mgr._datasrc_info_list) + + # Try to configure a data source clients with the manager. This + # should confirm the manager object is instantiated enabling in-memory + # cache. + cfg_data = MockConfigData( + {"classes": {"IN": [{"type": "MasterFiles", + "cache-enable": True, "params": {}}]}}) + self.__mgr._datasrc_clients_mgr.reconfigure({}, cfg_data) + clist = \ + self.__mgr._datasrc_clients_mgr.get_client_list(RRClass.IN) + self.assertEqual(1, len(clist.get_status())) def test_configure(self): self.__mgr._setup_ccsession() @@ -134,6 +157,54 @@ class TestMemmgr(unittest.TestCase): self.assertRaises(isc.server_common.bind10_server.BIND10ServerFatal, self.__mgr._setup_module) + def test_datasrc_config_handler(self): + self.__mgr._mapped_file_dir = '/some/path' + + # A simple (boring) case with real class implementations. This + # confirms the methods are called as expected. + cfg_data = MockConfigData( + {"classes": {"IN": [{"type": "MasterFiles", + "cache-enable": True, "params": {}}]}}) + self.__mgr._datasrc_config_handler({}, cfg_data) + self.assertEqual(1, len(self.__mgr._datasrc_info_list)) + self.assertEqual(1, self.__mgr._datasrc_info_list[0].gen_id) + + # Below we're using a mock DataSrcClientMgr for easier tests + class MockDataSrcClientMgr: + def __init__(self, status_list, raise_on_reconfig=False): + self.__status_list = status_list + self.__raise_on_reconfig = raise_on_reconfig + + def reconfigure(self, new_config, config_data): + if self.__raise_on_reconfig: + raise isc.server_common.datasrc_clients_mgr.ConfigError( + 'test error') + # otherwise do nothing + + def get_clients_map(self): + return 42, {RRClass.IN: self} + + def get_status(self): # mocking get_clients_map()[1].get_status() + return self.__status_list + + # This confirms memmgr's config is passed and handled correctly. + # From memmgr's point of view it should be enough we have an object + # in segment_info_map. Note also that the new DataSrcInfo is appended + # to the list + self.__mgr._datasrc_clients_mgr = \ + MockDataSrcClientMgr([('sqlite3', 'mapped', None)]) + self.__mgr._datasrc_config_handler(None, None) # params don't matter + self.assertEqual(2, len(self.__mgr._datasrc_info_list)) + self.assertIsNotNone( + self.__mgr._datasrc_info_list[1].segment_info_map[ + (RRClass.IN, 'sqlite3')]) + + # Emulate the case reconfigure() fails. Exception isn't propagated, + # but the list doesn't change. + self.__mgr._datasrc_clients_mgr = MockDataSrcClientMgr(None, True) + self.__mgr._datasrc_config_handler(None, None) + self.assertEqual(2, len(self.__mgr._datasrc_info_list)) + if __name__== "__main__": isc.log.resetUnitTestRootLogger() unittest.main() From ebe5c03858d0e41ae7e3a8477399d0f54712569a Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Tue, 11 Jun 2013 23:50:26 -0700 Subject: [PATCH 20/35] [2854] made local config params generic --- src/bin/memmgr/memmgr.py.in | 27 +++++++++++++++++---------- src/bin/memmgr/tests/memmgr_test.py | 10 ++++++---- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/bin/memmgr/memmgr.py.in b/src/bin/memmgr/memmgr.py.in index f2c7c8cc9c..75406fc642 100755 --- a/src/bin/memmgr/memmgr.py.in +++ b/src/bin/memmgr/memmgr.py.in @@ -15,6 +15,7 @@ # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import copy import os import sys import signal @@ -43,13 +44,11 @@ class ConfigError(Exception): class Memmgr(BIND10Server): def __init__(self): - # configurable parameter: initially this is the only param, so - # we only maintain as a single attribute. As the class is extended - # and more configurable, consider introducing a compound type or - # class. + # Running configurable parameters: on initial configuration this will + # be a dict: str=>config_value. # This is defined as "protected" so tests can inspect it; others # shouldn't use it directly. - self._mapped_file_dir = None + self._config_params = None # The manager to keep track of data source configuration. Allow # tests to inspect/tweak it. @@ -76,7 +75,7 @@ class Memmgr(BIND10Server): # latest full config data, which consist of the defaults with # possibly overridden by user config. Otherwise, just apply the latest # diff. - if self._mapped_file_dir is None: + if self._config_params is None: new_config = self.mod_ccsession.get_full_config() try: self.__update_config(new_config) @@ -98,6 +97,13 @@ class Memmgr(BIND10Server): Errors are to be reported as an exception. """ + # If this is the first time, build everything from the scratch. + # Otherwise, make a full local copy and update it. + if self._config_params is None: + new_config_params = {} + else: + new_config_params = copy.deepcopy(self._config_params) + new_mapped_file_dir = new_config.get('mapped_file_dir') if new_mapped_file_dir is not None: if not os.path.isdir(new_mapped_file_dir): @@ -106,7 +112,10 @@ class Memmgr(BIND10Server): if not os.access(new_mapped_file_dir, os.W_OK): raise ConfigError('mapped_file_dir is not writable: ' + new_mapped_file_dir) - self._mapped_file_dir = new_mapped_file_dir + new_config_params['mapped_file_dir'] = new_mapped_file_dir + + # All copy, switch to the new configuration. + self._config_params = new_config_params def _setup_module(self): """Module specific initialization for BIND10Server.""" @@ -132,9 +141,7 @@ class Memmgr(BIND10Server): try: self._datasrc_clients_mgr.reconfigure(new_config, config_data) genid, clients_map = self._datasrc_clients_mgr.get_clients_map() - datasrc_info = DataSrcInfo(genid, clients_map, - {'mapped_file_dir': - self._mapped_file_dir}) + datasrc_info = DataSrcInfo(genid, clients_map, self._config_params) self._datasrc_info_list.append(datasrc_info) except isc.server_common.datasrc_clients_mgr.ConfigError as ex: logger.error(MEMMGR_DATASRC_CONFIG_ERROR, ex) diff --git a/src/bin/memmgr/tests/memmgr_test.py b/src/bin/memmgr/tests/memmgr_test.py index 2d2fc6023f..9f5ef9a7e4 100755 --- a/src/bin/memmgr/tests/memmgr_test.py +++ b/src/bin/memmgr/tests/memmgr_test.py @@ -86,7 +86,7 @@ class TestMemmgr(unittest.TestCase): def test_init(self): """Check some initial conditions""" - self.assertIsNone(self.__mgr._mapped_file_dir) + self.assertIsNone(self.__mgr._config_params) self.assertEqual([], self.__mgr._datasrc_info_list) # Try to configure a data source clients with the manager. This @@ -112,13 +112,15 @@ class TestMemmgr(unittest.TestCase): self.assertEqual((0, None), parse_answer(self.__mgr._config_handler({}))) self.assertEqual('mapped_files', - self.__mgr._mapped_file_dir.split('/')[-1]) + self.__mgr._config_params['mapped_file_dir']. + split('/')[-1]) # Update the configuration. user_cfg = {'mapped_file_dir': '/some/path/dir'} self.assertEqual((0, None), parse_answer(self.__mgr._config_handler(user_cfg))) - self.assertEqual('/some/path/dir', self.__mgr._mapped_file_dir) + self.assertEqual('/some/path/dir', + self.__mgr._config_params['mapped_file_dir']) # Bad update: diretory doesn't exist (we assume it really doesn't # exist in the tested environment). Update won't be made. @@ -158,7 +160,7 @@ class TestMemmgr(unittest.TestCase): self.__mgr._setup_module) def test_datasrc_config_handler(self): - self.__mgr._mapped_file_dir = '/some/path' + self.__mgr._config_params = {'mapped_file_dir': '/some/path'} # A simple (boring) case with real class implementations. This # confirms the methods are called as expected. From a43f9f2860f156daa1a8caf95d6c0ff1a95323a9 Mon Sep 17 00:00:00 2001 From: JINMEI Tatuya Date: Wed, 12 Jun 2013 00:08:15 -0700 Subject: [PATCH 21/35] [2854] initial man page --- src/bin/memmgr/b10-memmgr.xml | 46 +++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/bin/memmgr/b10-memmgr.xml b/src/bin/memmgr/b10-memmgr.xml index 2f3ae67d22..5813e832c4 100644 --- a/src/bin/memmgr/b10-memmgr.xml +++ b/src/bin/memmgr/b10-memmgr.xml @@ -20,7 +20,7 @@ - June 4, 2012 + June 11, 2012 @@ -49,29 +49,39 @@ DESCRIPTION + The b10-memmgr daemon manages sharable + memory segments storing in-memory DNS zone data, and + communicates with other BIND 10 modules about the segment + information so the entire system will use these segments + in a consistent manner. + ARGUMENTS - The arguments are as follows: - - - - - - , - - - - - Print the command line arguments and exit. - - - + The b10-memmgr daemon does not take + any command line arguments. + + CONFIGURATION AND COMMANDS + + The configurable settings are: + + + mapped_file_dir + A path to store files to be mapped to memory. This must be + writable to the b10-memmgr daemon. + + + + The module commands are: + + + shutdown exits b10-memmgr. + @@ -87,6 +97,10 @@ HISTORY + + The b10-memmgr daemon was first implemented + in 2013 for the ISC BIND 10 project. +