[#1818] Added HttpClient thread pool deferred start

HttpClient's thread pool can now be started
post-construction via new start() method

src/lib/config/cmd_http_listener.*
    Clean up.

src/lib/config/tests/cmd_http_listener_unittests.cc
    Expanded testing.

src/lib/http/client.*
    HttpClient::HttpClient - added defer_thread_start parameter
    HttpClient::start() - new method to start thread-pool

src/lib/http/tests/mt_client_unittests.cc
    TEST_F(MtHttpClientTest, deferredStart) - new test
    TEST_F(MtHttpClientTest, restartAfterStop)  - new test

src/lib/config/cmd_http_listener.cc

@@ -66,8 +66,8 @@ CmdHttpListener::start() {
                                               HttpListener::RequestTimeout(TIMEOUT_AGENT_RECEIVE_COMMAND),
                                               HttpListener::IdleTimeout(TIMEOUT_AGENT_IDLE_CONNECTION_TIMEOUT)));
 
-        // Create the thread pool.
+        // Create the thread pooli with immediate start.
-        threads_.reset(new HttpThreadPool(io_service_, thread_pool_size_, false));
+        threads_.reset(new HttpThreadPool(io_service_, thread_pool_size_));
 
         // Instruct the HTTP listener to actually open socket, install
         // callback and start listening.
@@ -134,8 +134,8 @@ CmdHttpListener::getRunState() const {
 
 bool
 CmdHttpListener::isListening() const {
-    // If we have a listener we're listening.
+    return (threads_ && (threads_->getRunState() == HttpThreadPool::RunState::PAUSED 
-    return (http_listener_ != 0);
+                         || threads_->getRunState() == HttpThreadPool::RunState::RUN));
 }
 
 } // namespace isc::config

src/lib/config/cmd_http_listener.h

@@ -50,6 +50,9 @@ public:
     /// @brief Stops the listener's thread pool.
     void stop();
 
+    /// @brief Fetches the run state of the thread pool.
+    ///
+    /// @return Run state of the pool.
     http::HttpThreadPool::RunState getRunState() const;
 
     /// @brief Checks if we are listening to the HTTP requests.
@@ -60,28 +63,28 @@ public:
     /// @brief Fetches the IP address on which to listen.
     ///
     /// @return IOAddress containing the address on which to listen.
-    isc::asiolink::IOAddress& getAddress() {
+    isc::asiolink::IOAddress getAddress() const {
         return (address_);
     }
 
     /// @brief Fetches the port number on which to listen.
     ///
     /// @return uint16_t containing the port number on which to listen.
-    uint16_t getPort() {
+    uint16_t getPort() const {
         return (port_);
     }
 
     /// @brief Fetches the maximum size of the thread pool.
     ///
     /// @return uint16_t containing the maximum size of the thread pool.
-    uint16_t getThreadPoolSize() {
+    uint16_t getThreadPoolSize() const {
         return (thread_pool_size_);
     }
 
     /// @brief Fetches the number of threads in the pool.
     ///
     /// @return uint16_t containing the number of running threads.
-    uint16_t getThreadCount() {
+    uint16_t getThreadCount() const {
         if (!threads_) {
             return (0);
         }
@@ -89,6 +92,10 @@ public:
         return (threads_->getThreadCount());
     }
 
+    asiolink::IOServicePtr getIOService() const {
+        return(io_service_);
+    }
+
 private:
     /// @brief IP address on which to listen.
     isc::asiolink::IOAddress address_;

src/lib/config/tests/cmd_http_listener_unittests.cc

@@ -555,8 +555,8 @@ public:
     size_t pause_cnt_;
 };
 
-/// Verifies the construction, starting, stopping, and destruction
+/// Verifies the construction, starting, stopping, pausing, resuming,
-/// of CmdHttpListener.
+/// and destruction of CmdHttpListener.
 TEST_F(CmdHttpListenerTest, basics) {
     // Make sure multi-threading is off.
     MultiThreadingMgr::instance().setMode(false);
@@ -572,9 +572,13 @@ TEST_F(CmdHttpListenerTest, basics) {
     EXPECT_EQ(listener_->getPort(), port);
     EXPECT_EQ(listener_->getThreadPoolSize(), 1);
 
-    // It should not be listening and have no threads.
+    // It should not have an IOService, should not be listening
+    // should have no threads.
+    ASSERT_FALSE(listener_->getIOService());
     EXPECT_FALSE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 0);
+    ASSERT_THROW_MSG(listener_->getRunState(), InvalidOperation,
+                     "CmdHttpListener::getRunState - no thread pool!");
 
     // Verify that we cannot start it when multi-threading is disabled.
     ASSERT_FALSE(MultiThreadingMgr::instance().getMode());
@@ -593,6 +597,9 @@ TEST_F(CmdHttpListenerTest, basics) {
     ASSERT_NO_THROW_LOG(listener_->start());
     ASSERT_TRUE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 1);
+    ASSERT_TRUE(listener_->getIOService());
+    EXPECT_FALSE(listener_->getIOService()->stopped());
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::RUN);
 
     // Trying to start it again should fail.
     ASSERT_THROW_MSG(listener_->start(), InvalidOperation,
@@ -602,6 +609,8 @@ TEST_F(CmdHttpListenerTest, basics) {
     ASSERT_NO_THROW_LOG(listener_->stop());
     ASSERT_FALSE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 0);
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::STOPPED);
+    ASSERT_FALSE(listener_->getIOService());
 
     // Make sure we can call stop again without problems.
     ASSERT_NO_THROW_LOG(listener_->stop());
@@ -610,6 +619,9 @@ TEST_F(CmdHttpListenerTest, basics) {
     ASSERT_NO_THROW_LOG(listener_->start());
     ASSERT_TRUE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 1);
+    ASSERT_TRUE(listener_->getIOService());
+    EXPECT_FALSE(listener_->getIOService()->stopped());
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::RUN);
 
     // Destroying it should also stop it.
     // If the test timeouts we know it didn't!
@@ -622,15 +634,34 @@ TEST_F(CmdHttpListenerTest, basics) {
     EXPECT_EQ(listener_->getPort(), port);
     EXPECT_EQ(listener_->getThreadPoolSize(), 4);
     ASSERT_TRUE(listener_->isListening());
+    ASSERT_TRUE(listener_->getIOService());
+    EXPECT_FALSE(listener_->getIOService()->stopped());
+
+    // Verify we can pause it.  We should still be listening, threads intact,
+    // IOservice stopped, state set to PAUSED.
+    ASSERT_NO_THROW_LOG(listener_->pause());
+    ASSERT_TRUE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 4);
+    ASSERT_TRUE(listener_->getIOService());
+    EXPECT_TRUE(listener_->getIOService()->stopped());
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::PAUSED);
+
+    // Verify we can resume it.
+    ASSERT_NO_THROW_LOG(listener_->resume());
+    ASSERT_TRUE(listener_->isListening());
+    EXPECT_EQ(listener_->getThreadCount(), 4);
+    ASSERT_TRUE(listener_->getIOService());
+    EXPECT_FALSE(listener_->getIOService()->stopped());
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::RUN);
 
     // Stop it and verify we're no longer listening.
     ASSERT_NO_THROW_LOG(listener_->stop());
     ASSERT_FALSE(listener_->isListening());
     EXPECT_EQ(listener_->getThreadCount(), 0);
+    ASSERT_FALSE(listener_->getIOService());
+    EXPECT_EQ(listener_->getRunState(),  HttpThreadPool::RunState::STOPPED);
 }
 
-
 // This test verifies that an HTTP connection can be established and used to
 // transmit an HTTP request and receive the response.
 TEST_F(CmdHttpListenerTest, basicListenAndRespond) {

src/lib/http/client.cc

@@ -1740,11 +1740,13 @@ public:
     /// threaded mode.  (Currently ignored in multi-threaded mode)
     /// @param thread_pool_size maximum number of concurrent threads
     /// Internally this also sets the maximum number concurrent connections
-    /// @param defer_thread_start if true, then the thread pool will be
-    /// created but started  Applicable only when thread-pool-size is
-    /// greater than zero. 
-    /// will not be startedfalse, then 
     /// per URL.
+    /// @param defer_thread_start When true, creation of the pool threads is
+    /// deferred until a subsequent call to @ref start(). In this case the
+    /// pool's operational state post-construction is STOPPED.  Otherwise,
+    /// the thread pool threads will be created and started, with the post-
+    /// construction state being RUN.  Applicable only when thread-pool size
+    /// is greater than zero.
     HttpClientImpl(IOService& io_service, size_t thread_pool_size = 0,
                    bool defer_thread_start = false)
         : thread_pool_size_(thread_pool_size), threads_() {
@@ -1752,7 +1754,7 @@ public:
             // Create our own private IOService.
             thread_io_service_.reset(new IOService());
 
-            // Create the thread pool. 
+            // Create the thread pool.
             threads_.reset(new HttpThreadPool(thread_io_service_, thread_pool_size_,
                                               defer_thread_start));
 
@@ -1853,7 +1855,8 @@ private:
     HttpThreadPoolPtr threads_;
 };
 
-HttpClient::HttpClient(IOService& io_service, size_t thread_pool_size) {
+HttpClient::HttpClient(IOService& io_service, size_t thread_pool_size,
+                       bool defer_thread_start/* = false*/) {
     if (thread_pool_size > 0) {
         if (!MultiThreadingMgr::instance().getMode()) {
             isc_throw(InvalidOperation,
@@ -1862,7 +1865,8 @@ HttpClient::HttpClient(IOService& io_service, size_t thread_pool_size) {
         }
     }
 
-    impl_.reset(new HttpClientImpl(io_service, thread_pool_size));
+    impl_.reset(new HttpClientImpl(io_service, thread_pool_size,
+                                   defer_thread_start));
 }
 
 HttpClient::~HttpClient() {
@@ -1909,6 +1913,11 @@ HttpClient::closeIfOutOfBand(int socket_fd)  {
     return (impl_->conn_pool_->closeIfOutOfBand(socket_fd));
 }
 
+void
+HttpClient::start() {
+    impl_->start();
+}
+
 void
 HttpClient::stop() {
     impl_->stop();

src/lib/http/client.h

@@ -136,10 +136,20 @@ public:
     ///
     /// @param io_service IO service to be used by the HTTP client.
     /// @param thread_pool_size maximum number of threads in the thread pool.
+    /// @param defer_thread_start if true, then the thread pool will be
+    /// created but started  Applicable only when thread-pool-size is
+    /// greater than zero.
     /// A value greater than zero enables multi-threaded mode and sets the
     /// maximum number of concurrent connections per URL.  A value of zero
     /// (default) enables single-threaded mode with one connection per URL.
-    explicit HttpClient(asiolink::IOService& io_service, size_t thread_pool_size = 0);
+    /// @param defer_thread_start When true, creation of the pool threads is
+    /// deferred until a subsequent call to @ref start(). In this case the
+    /// pool's operational state post-construction is STOPPED.  Otherwise,
+    /// the thread pool threads will be created and started, with the post-
+    /// construction state being RUN.  Applicable only when thread-pool size
+    /// is greater than zero.
+    explicit HttpClient(asiolink::IOService& io_service, size_t thread_pool_size = 0,
+                        bool defer_thread_start = false);
 
     /// @brief Destructor.
     ~HttpClient();
@@ -242,6 +252,9 @@ public:
                           const CloseHandler& close_callback =
                           CloseHandler());
 
+    /// @brief Starts client's thread pool, if mult-threaded.
+    void start();
+
     /// @brief Halts client-side IO activity.
     ///
     /// Closes all connections, discards any queued requests, and in
@@ -249,7 +262,6 @@ public:
     /// IOService.
     void stop();
 
-
     /// @brief Closes a connection if it has an out-of-band socket event
     ///
     /// If the  client owns a connection using the given socket and that

src/lib/http/tests/mt_client_unittests.cc

@@ -858,6 +858,7 @@ TEST_F(MtHttpClientTest, basics) {
     ASSERT_TRUE(client->getThreadIOService());
     ASSERT_EQ(client->getThreadPoolSize(), 3);
     ASSERT_EQ(client->getThreadCount(), 3);
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::RUN);
 
     // Verify stop doesn't throw.
     ASSERT_NO_THROW_LOG(client->stop());
@@ -881,6 +882,79 @@ TEST_F(MtHttpClientTest, basics) {
     ASSERT_NO_THROW_LOG(client.reset());
 }
 
+// Verifies we can construct with deferred start.
+TEST_F(MtHttpClientTest, deferredStart) {
+    MultiThreadingMgr::instance().setMode(true);
+    HttpClientPtr client;
+    size_t thread_pool_size = 3;
+
+    // Create MT client with deferred start.
+    ASSERT_NO_THROW_LOG(client.reset(new HttpClient(io_service_, thread_pool_size, true)));
+    ASSERT_TRUE(client);
+
+    // Client should be STOPPED, with no threads.
+    ASSERT_TRUE(client->getThreadIOService());
+    ASSERT_EQ(client->getThreadPoolSize(), thread_pool_size);
+    ASSERT_EQ(client->getThreadCount(), 0);
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::STOPPED);
+
+    // We should be able to start it.
+    ASSERT_NO_THROW(client->start());
+
+    // Verify we have threads and run state is RUN.
+    ASSERT_EQ(client->getThreadCount(), 3);
+    ASSERT_TRUE(client->getThreadIOService());
+    ASSERT_FALSE(client->getThreadIOService()->stopped());
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::RUN);
+
+    // Cannot start it twice.
+    ASSERT_THROW_MSG(client->start(), InvalidOperation,
+                     "HttpThreadPool::start already started!");
+
+    // Verify we didn't break it.
+    ASSERT_EQ(client->getThreadCount(), 3);
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::RUN);
+
+    // Make sure destruction doesn't throw.
+    ASSERT_NO_THROW_LOG(client.reset());
+}
+
+// Verifies we can restart after stop.
+TEST_F(MtHttpClientTest, restartAfterStop) {
+    MultiThreadingMgr::instance().setMode(true);
+    HttpClientPtr client;
+    size_t thread_pool_size = 3;
+
+    // Create MT client with instant start.
+    ASSERT_NO_THROW_LOG(client.reset(new HttpClient(io_service_, thread_pool_size)));
+    ASSERT_TRUE(client);
+
+    // Verify we're started.
+    ASSERT_EQ(client->getThreadCount(), 3);
+    ASSERT_TRUE(client->getThreadIOService());
+    ASSERT_FALSE(client->getThreadIOService()->stopped());
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::RUN);
+
+    // Stop should succeed.
+    ASSERT_NO_THROW_LOG(client->stop());
+
+    // Verify we're stopped.
+    ASSERT_EQ(client->getThreadCount(), 0);
+    ASSERT_TRUE(client->getThreadIOService());
+    ASSERT_TRUE(client->getThreadIOService()->stopped());
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::STOPPED);
+
+    // Starting again should succeed.
+    ASSERT_NO_THROW_LOG(client->start());
+    ASSERT_EQ(client->getThreadCount(), 3);
+    ASSERT_TRUE(client->getThreadIOService());
+    ASSERT_FALSE(client->getThreadIOService()->stopped());
+    ASSERT_EQ(client->getRunState(), HttpThreadPool::RunState::RUN);
+
+    // Make sure destruction doesn't throw.
+    ASSERT_NO_THROW_LOG(client.reset());
+}
+
 // Now we'll run some permutations of the number of client threads,
 // requests, and listeners.