From 39df330f90b089f6e9242b048566c7f37168cab0 Mon Sep 17 00:00:00 2001
From: Bruce Toll <4109762+tollb@users.noreply.github.com>
Date: Wed, 1 Feb 2023 09:26:16 -0500
Subject: [PATCH 4/5] Track closing tcp connections to avoid race

Use two lists to represent active and closed (but still pending in Asio)
connections. Defer freeing listeners until Asio acknowledges the close.

This avoids a race where Asio can attempt to access the listener after
it has been deleted. The use of lists avoids potential for reallocation
of listeners while in use by Asio.
---
 src/http/Server.C | 25 +++++++++++++++----------
 src/http/Server.h |  5 ++++-
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/http/Server.C b/src/http/Server.C
index b26a4447..d57a17da 100644
--- a/src/http/Server.C
+++ b/src/http/Server.C
@@ -501,13 +501,13 @@ void Server::startAccept()
    * Ssl connection, this performance impact is negligible (and both
    * need to access the ConnectionManager mutex in any case).
    */
-  for (std::size_t i = 0; i < tcp_listeners_.size(); ++i) {
-    asio::ip::tcp::acceptor &acceptor = tcp_listeners_[i].acceptor;
-    TcpConnectionPtr &new_connection = tcp_listeners_[i].new_connection;
+  for (auto &&tcp_listener: tcp_listeners_) {
+    asio::ip::tcp::acceptor &acceptor = tcp_listener.acceptor;
+    TcpConnectionPtr &new_connection = tcp_listener.new_connection;
     acceptor.async_accept(new_connection->socket(),
                           accept_strand_.wrap(
                             std::bind(&Server::handleTcpAccept, this,
-                                        &tcp_listeners_[i],
+                                        &tcp_listener,
                                         std::placeholders::_1)));
   }
 
@@ -594,9 +594,9 @@ void Server::resume()
 
 void Server::handleResume()
 {
-  for (std::size_t i = 0; i < tcp_listeners_.size(); ++i)
-    tcp_listeners_[i].acceptor.close();
-  tcp_listeners_.clear();
+  for (auto &&tcp_listener: tcp_listeners_)
+    tcp_listener.acceptor.close();
+  tcp_listeners_closed_.splice(tcp_listeners_closed_.end(), tcp_listeners_);
 
 #ifdef HTTP_WITH_SSL
   for (std::size_t i = 0; i < ssl_listeners_.size(); ++i)
@@ -616,6 +616,11 @@ void Server::handleTcpAccept(TcpListener *listener, const Wt::AsioWrapper::error
   } else if (!listener->acceptor.is_open()) {
     // server shutdown
     LOG_DEBUG("handleTcpAccept: async_accept error (acceptor closed, probably server shutdown): " << e.message());
+
+    auto tcp_listeners_closed_before = tcp_listeners_closed_.size();
+    tcp_listeners_closed_.remove_if([listener] (TcpListener &t) { return &t == listener; });
+    auto tcp_listeners_closed_count = tcp_listeners_closed_before - tcp_listeners_closed_.size();
+    LOG_INFO("handleTcpAccept: async_accept: acceptor closed, removed " << tcp_listeners_closed_count << " matching connections");
     return;
   } else {
     LOG_ERROR("handleTcpAccept: async_accept error: " << e.message());
@@ -662,9 +667,9 @@ void Server::handleStop()
   // The server is stopped by cancelling all outstanding asynchronous
   // operations. Once all operations have finished the io_service::run() call
   // will exit.
-  for (std::size_t i = 0; i < tcp_listeners_.size(); ++i)
-    tcp_listeners_[i].acceptor.close();
-  tcp_listeners_.clear();
+  for (auto &&tcp_listener: tcp_listeners_)
+    tcp_listener.acceptor.close();
+  tcp_listeners_closed_.splice(tcp_listeners_closed_.end(), tcp_listeners_);
 
 #ifdef HTTP_WITH_SSL
   for (std::size_t i = 0; i < ssl_listeners_.size(); ++i)
diff --git a/src/http/Server.h b/src/http/Server.h
index 36758d4d..a50fe9cc 100644
--- a/src/http/Server.h
+++ b/src/http/Server.h
@@ -134,7 +134,10 @@ private:
   Wt::AsioWrapper::strand accept_strand_;
 
   /// Acceptors used to listen for incoming http connections.
-  std::vector<TcpListener> tcp_listeners_;
+  std::list<TcpListener> tcp_listeners_;
+
+  /// Acceptors that have been closed and are waiting asio acknowledgement of close
+  std::list<TcpListener> tcp_listeners_closed_;
 
 #ifdef HTTP_WITH_SSL
   struct SslListener {
-- 
2.38.1

