fix: don't let one unreachable server stall the whole listener

src/TDSProxy/TDSConnection.cs

@@ -492,7 +492,21 @@ public TDSConnection(TDSProxyService service,
 
 			_insideEP = insideEndPoint;
 			_insideClient = new TcpClient(_insideEP.AddressFamily) {NoDelay = false};
-			_insideClient.Connect(insideEndPoint);
+
+			try
+			{
+				ConnectWithTimeout(_insideClient, insideEndPoint);
+			}
+			catch
+			{
+				// The far leg never came up. Undo what this constructor has already done: the
+				// Stopping subscription above is what keeps the instance alive, so without this
+				// the failed attempt lingers - still holding the client's socket - until the
+				// service stops. An hour of an unreachable server strands an hour of attempts.
+				AbandonBeforeConnected();
+				throw;
+			}
+
 			EnableKeepAlive(_insideClient.Client);
 			_insideStream = _insideClient.GetStream();
 			_insideActiveStream = _insideStream; // Start with plain stream, may upgrade to SSL later
@@ -501,6 +515,53 @@ public TDSConnection(TDSProxyService service,
 			_processingTask = ProcessConnection();
 		}
 
+		/// <summary>
+		/// How long to wait for the far server to answer a connection attempt. Left to the OS this
+		/// is a fixed sequence of SYN retries - over two minutes on Linux - which a blackholed route
+		/// runs through in full: longer than any client waits, and a parked thread throughout.
+		/// </summary>
+		static readonly TimeSpan ConnectTimeout = TimeSpan.FromSeconds(10);
+
+		static void ConnectWithTimeout(TcpClient client, IPEndPoint endPoint)
+		{
+			using (var cts = new CancellationTokenSource(ConnectTimeout))
+			{
+				try
+				{
+					client.ConnectAsync(endPoint.Address, endPoint.Port, cts.Token)
+					      .AsTask()
+					      .GetAwaiter()
+					      .GetResult();
+				}
+				catch (OperationCanceledException) when (cts.IsCancellationRequested)
+				{
+					throw new TimeoutException(
+						$"Timed out after {ConnectTimeout.TotalSeconds:0} seconds connecting to {endPoint}.");
+				}
+			}
+		}
+
+		/// <summary>
+		/// Roll back the part of construction that ran before the far leg was up, so an attempt that
+		/// never became a connection does not count as active and does not keep itself alive. The
+		/// outside client belongs to the caller, which closes it.
+		/// </summary>
+		void AbandonBeforeConnected()
+		{
+			_state = StateEnum.Closed;
+			Interlocked.Decrement(ref ActiveConnectionCount);
+			_service.Stopping -= service_Stopping;
+
+			try
+			{
+				_insideClient.Close();
+			}
+			catch (Exception e)
+			{
+				log.Error($"Error closing the inside client for connection from {_outsideEP}", e);
+			}
+		}
+
 		/// <summary>
 		/// Turn on TCP keep-alive so a peer that disappears without FIN or RST is noticed.
 		/// A link that is cut rather than closed leaves a blocking read waiting forever:
@@ -560,7 +621,7 @@ void IDisposable.Dispose()
 
 				try
 				{
-					_insideStream.Close();
+					_insideStream?.Close();
 				}
 				catch (Exception e)
 				{

src/TDSProxy/TDSListener.cs

@@ -203,9 +203,11 @@ private SslProtocols ParseSslProtocols(string protocols)
 
 		private void AcceptConnection(IAsyncResult result)
 		{
+			TcpClient readClient;
+
 			try
 			{
-				TcpClient readClient = ((TcpListener)result.AsyncState).EndAcceptTcpClient(result);
+				readClient = ((TcpListener)result.AsyncState).EndAcceptTcpClient(result);
 
 				log.InfoFormat("Accepted connection from {0} on {1}, will forward to {2}", readClient.Client.RemoteEndPoint, readClient.Client.LocalEndPoint, ForwardTo);
 
@@ -215,25 +217,56 @@ private void AcceptConnection(IAsyncResult result)
 					readClient.Close();
 					return;
 				}
-
-				new TDSConnection(_service, this, readClient, ForwardTo);
 			}
-			catch (ObjectDisposedException) { /* We're shutting down, ignore */ }
+			catch (ObjectDisposedException)
+			{
+				/* We're shutting down, ignore */
+				return;
+			}
 			catch (Exception e)
 			{
-				log.Fatal("Error in AcceptConnection.", e);
+				log.Fatal("Error accepting connection.", e);
+				return;
+			}
+			finally
+			{
+				ResumeAccepting();
 			}
 
-			if (!_stopped)
+			// Setting up the connection dials the far server, and that dial blocks. It has to happen
+			// after the listener is accepting again: a server that swallows SYNs takes the connect
+			// timeout to fail, and until this method returned, that was equally how long every other
+			// client sat in the backlog waiting to be accepted.
+			try
 			{
+				new TDSConnection(_service, this, readClient, ForwardTo);
+			}
+			catch (Exception e)
+			{
+				log.Fatal("Error in AcceptConnection.", e);
 				try
 				{
-					_tcpListener?.BeginAcceptTcpClient(AcceptConnection, _tcpListener);
+					readClient.Close();
+				}
+				catch (Exception closeError)
+				{
+					log.Error("Error closing a connection that could not be set up.", closeError);
 				}
-				catch (ObjectDisposedException) { /* We're shutting down, ignore */ }
 			}
 		}
 
+		void ResumeAccepting()
+		{
+			if (_stopped)
+				return;
+
+			try
+			{
+				_tcpListener?.BeginAcceptTcpClient(AcceptConnection, _tcpListener);
+			}
+			catch (ObjectDisposedException) { /* We're shutting down, ignore */ }
+		}
+
 		public void Dispose()
 		{
 			if (!_stopped)