From b4113b926e117e2e04dddf4a5bdb2126a832047c Mon Sep 17 00:00:00 2001 From: Lefteris Zafiris Date: Wed, 28 Jul 2021 13:20:30 +0300 Subject: [PATCH 1/3] Set the TCP keepalive socket options Configure the TCP keepalive timeout and interval in a way that matches our server side behaviour. opsbugs #2100 --- customerio/client_base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/customerio/client_base.py b/customerio/client_base.py index 388ad14..396b8e6 100644 --- a/customerio/client_base.py +++ b/customerio/client_base.py @@ -4,10 +4,12 @@ from __future__ import division from datetime import datetime, timezone import math +import socket from requests import Session from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry +from urllib3.connection import HTTPConnection class CustomerIOException(Exception): @@ -18,6 +20,14 @@ def __init__(self, retries=3, timeout=10, backoff_factor=0.02): self.timeout = timeout self.retries = retries + # Set the TCP keepalive settings to the values dicated by our server-side configuration. + HTTPConnection.default_socket_options = ( HTTPConnection.default_socket_options + [ + (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + (socket.SOL_TCP, socket.TCP_KEEPIDLE, 550), + (socket.SOL_TCP, socket.TCP_KEEPINTVL, 60) + ] + ) + self.http = Session() # Retry request a number of times before raising an exception # also define backoff_factor to delay each retry From 6f672acfc3be9c43e08b0111e103c199f98a8a33 Mon Sep 17 00:00:00 2001 From: Lefteris Zafiris Date: Wed, 28 Jul 2021 18:23:07 +0300 Subject: [PATCH 2/3] Set TCP keepalive timeout to 300 sec While testing we found out that the TCP timeout on GCP loadbalancers is 360 sec. The 600 sec value refers to connections from the LB to backends. --- customerio/client_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/customerio/client_base.py b/customerio/client_base.py index 396b8e6..ecb2370 100644 --- a/customerio/client_base.py +++ b/customerio/client_base.py @@ -23,7 +23,7 @@ def __init__(self, retries=3, timeout=10, backoff_factor=0.02): # Set the TCP keepalive settings to the values dicated by our server-side configuration. HTTPConnection.default_socket_options = ( HTTPConnection.default_socket_options + [ (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), - (socket.SOL_TCP, socket.TCP_KEEPIDLE, 550), + (socket.SOL_TCP, socket.TCP_KEEPIDLE, 300), (socket.SOL_TCP, socket.TCP_KEEPINTVL, 60) ] ) From 9b21b12c24d3b6914211bb440480397494913584 Mon Sep 17 00:00:00 2001 From: Stephen Young Date: Tue, 5 May 2026 23:49:39 -0400 Subject: [PATCH 3/3] Configure TCP keepalive through HTTP adapter --- customerio/client_base.py | 32 +++++++++++++++++++------------- tests/test_customerio.py | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/customerio/client_base.py b/customerio/client_base.py index 46ae3e5..562bd38 100644 --- a/customerio/client_base.py +++ b/customerio/client_base.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from requests import Session -from requests.adapters import HTTPAdapter +from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter from urllib3.connection import HTTPConnection from urllib3.util.retry import Retry @@ -26,22 +26,28 @@ def _tcp_keepalive_socket_options(): tcp_protocol = getattr(socket, "SOL_TCP", socket.IPPROTO_TCP) tcp_keepidle = getattr(socket, "TCP_KEEPIDLE", getattr(socket, "TCP_KEEPALIVE", None)) - options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] + options = list(HTTPConnection.default_socket_options) + keepalive_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] if tcp_keepidle is not None: - options.append((tcp_protocol, tcp_keepidle, TCP_KEEPALIVE_IDLE_TIMEOUT)) + keepalive_options.append((tcp_protocol, tcp_keepidle, TCP_KEEPALIVE_IDLE_TIMEOUT)) if hasattr(socket, "TCP_KEEPINTVL"): - options.append((tcp_protocol, socket.TCP_KEEPINTVL, TCP_KEEPALIVE_INTERVAL)) + keepalive_options.append((tcp_protocol, socket.TCP_KEEPINTVL, TCP_KEEPALIVE_INTERVAL)) + + for option in keepalive_options: + if option not in options: + options.append(option) return options -def _set_tcp_keepalive_socket_options(): - socket_options = list(HTTPConnection.default_socket_options) - for option in _tcp_keepalive_socket_options(): - if option not in socket_options: - socket_options.append(option) +class TCPKeepAliveHTTPAdapter(HTTPAdapter): + def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + pool_kwargs.setdefault("socket_options", _tcp_keepalive_socket_options()) + super().init_poolmanager(connections, maxsize, block=block, **pool_kwargs) - HTTPConnection.default_socket_options = socket_options + def proxy_manager_for(self, proxy, **proxy_kwargs): + proxy_kwargs.setdefault("socket_options", _tcp_keepalive_socket_options()) + return super().proxy_manager_for(proxy, **proxy_kwargs) class ClientBase: @@ -52,8 +58,6 @@ def __init__(self, retries=3, timeout=10, backoff_factor=0.02, use_connection_po self.use_connection_pooling = use_connection_pooling self._current_session = None - _set_tcp_keepalive_socket_options() - @property def http(self): if self._current_session is None: @@ -131,7 +135,9 @@ def _build_session(self): session.mount( "https://", - HTTPAdapter(max_retries=Retry(total=self.retries, backoff_factor=self.backoff_factor)), + TCPKeepAliveHTTPAdapter( + max_retries=Retry(total=self.retries, backoff_factor=self.backoff_factor) + ), ) return session diff --git a/tests/test_customerio.py b/tests/test_customerio.py index 37dd4b0..5ae2d65 100644 --- a/tests/test_customerio.py +++ b/tests/test_customerio.py @@ -1,12 +1,15 @@ import json +import socket import unittest from datetime import datetime from functools import partial import urllib3 from requests.auth import _basic_auth_str +from urllib3.connection import HTTPConnection from customerio import CustomerIO, CustomerIOException, Regions +from customerio.client_base import TCP_KEEPALIVE_IDLE_TIMEOUT, TCP_KEEPALIVE_INTERVAL from customerio.constants import CIOID, EMAIL, ID from tests.server import HTTPSTestCase @@ -64,6 +67,26 @@ def test_client_setup(self): with self.assertRaises(CustomerIOException): CustomerIO(site_id="site_id", api_key="api_key", region="au") + def test_keepalive_socket_options_are_configured_on_adapter(self): + default_socket_options = list(HTTPConnection.default_socket_options) + client = CustomerIO(site_id="site_id", api_key="api_key") + socket_options = client.http.adapters["https://"].poolmanager.connection_pool_kw[ + "socket_options" + ] + tcp_protocol = getattr(socket, "SOL_TCP", socket.IPPROTO_TCP) + tcp_keepidle = getattr(socket, "TCP_KEEPIDLE", getattr(socket, "TCP_KEEPALIVE", None)) + + for option in default_socket_options: + self.assertIn(option, socket_options) + self.assertIn((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), socket_options) + if tcp_keepidle is not None: + self.assertIn((tcp_protocol, tcp_keepidle, TCP_KEEPALIVE_IDLE_TIMEOUT), socket_options) + if hasattr(socket, "TCP_KEEPINTVL"): + self.assertIn( + (tcp_protocol, socket.TCP_KEEPINTVL, TCP_KEEPALIVE_INTERVAL), socket_options + ) + self.assertEqual(HTTPConnection.default_socket_options, default_socket_options) + def test_client_connection_handling(self): retries = self.cio.retries # should not raise exception as i should be less than retries and