Preserve real client IPs through Caddy and external load balancers

.env

@@ -11,6 +11,15 @@ USE_PROXY_AUTH=0
 PROXY_AUTH_USER=
 PROXY_AUTH_PASS=
 PUBLIC_ADDRESS=https://localhost:8443
+# Internal Docker IPv4 network and a free, non-gateway address reserved for Caddy.
+# Existing installations: follow the README network migration before updating.
+APP_INTERNAL_SUBNET=172.30.250.0/24
+# Dynamic container allocation must stay inside the subnet and exclude Caddy.
+APP_INTERNAL_DYNAMIC_RANGE=172.30.250.128/25
+PUBLIC_PROXY_IP=172.30.250.2
+# Optional external LB IPs/CIDRs, SPACE-separated (not comma-separated).
+# Empty trusts no external proxies. Use only the actual LB peers/chains.
+EXTERNAL_TRUSTED_PROXIES=
 ERAMBA_MCP_IMAGE_TAG=latest
 OAUTH2_INTROSPECTION_CLIENT_ID=eramba-mcp-introspect
 OAUTH2_INTROSPECTION_CLIENT_SECRET=Replace-this-with-a-unique-secret

.github/workflows/Docker.yml

@@ -24,8 +24,9 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
 
-      - name: Check MCP simple-install contract
-        run: npx --yes bats@1.12.0 tests/mcp-simple-install.bats
+      - name: Check simple-install and trusted-proxy contracts
+        timeout-minutes: 5
+        run: npx --yes bats@1.12.0 tests/*.bats
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3

Caddyfile

@@ -1,6 +1,37 @@
+{
+	servers {
+		trusted_proxies static {$EXTERNAL_TRUSTED_PROXIES}
+		trusted_proxies_strict
+		client_ip_headers X-Forwarded-For
+	}
+}
+
 :443 {
 	tls /certs/mycert.crt /certs/mycert.key
 
+	# Unsupported aliases can collide with standard headers after PHP normalizes
+	# hyphens/dots to underscores. Never forward them, even from a trusted LB.
+	request_header -X_Forwarded*
+	request_header -X.Forwarded*
+	request_header -X-Forwarded_*
+	request_header -X-Forwarded.*
+	request_header -X_Real*
+	request_header -X.Real*
+	request_header -X-Real_*
+	request_header -X-Real.*
+	request_header -Client_IP
+	request_header -Client.IP
+
+	# Caddy sets trusted_proxy from the original connection peer, not client_ip.
+	# It handles X-Forwarded-For/Proto/Host itself; remove other metadata that an
+	# untrusted caller must not introduce before Apache sees trusted Caddy.
+	@untrusted vars trusted_proxy false
+	request_header @untrusted -X-Forwarded-Port
+	request_header @untrusted -X-Forwarded-Prefix
+	request_header @untrusted -Forwarded
+	request_header @untrusted -X-Real-IP
+	request_header @untrusted -Client-IP
+
 	# Keep Inspector local-only. Proxying the UI and its local bridge through the
 	# same public origin as Eramba/MCP makes OAuth and app rendering brittle.
 	@mcp path /mcp /openapi /openapi.json /.well-known/oauth-protected-resource /.well-known/oauth-protected-resource/mcp /.well-known/oauth-authorization-server /.well-known/openid-configuration
@@ -10,6 +41,7 @@
 
 	handle {
 		reverse_proxy {$ERAMBA_UPSTREAM:https://eramba:443} {
+			header_up X-Forwarded-For {client_ip}
 			transport http {
 				tls_insecure_skip_verify
 			}

README.md

@@ -5,3 +5,43 @@ The bundled files in `apache/ssl/` are a branded local development certificate i
 Eramba and its MCP endpoint share the `PUBLIC_ADDRESS` origin. With the default configuration the application is available at `https://localhost:8443` and MCP at `https://localhost:8443/mcp`; issuer, resource, metadata, and OpenAPI URLs are derived automatically. Do not add separate public MCP URL variables.
 
 Before starting a real deployment, replace `OAUTH2_INTROSPECTION_CLIENT_SECRET` in `.env` with a unique random secret. `ERAMBA_MCP_IMAGE_TAG` defaults to the latest MCP image and can be pinned when required.
+
+## Client IPs and trusted reverse proxies
+
+**Image dependency:** this configuration requires an Eramba image that implements the `TRUSTED_PROXIES` Apache startup configuration. Publish and verify that image for the installed edition before rolling out these installer changes. Existing images without this support do not make `TRUST_PROXY=1` safe. The Compose files still use `latest`; a tag named `latest` alone is not evidence that it contains the change. Pin a verified compatible release or image digest using your deployment's Compose override. There is no new release version assigned in this repository.
+
+The request path is client → optional external load balancer → bundled Caddy → Eramba. Caddy resolves `X-Forwarded-For` from right to left and sends one client address to Eramba. Apache trusts only Caddy's fixed IPv4 address, normalizes the request's client IP, and supplies that same address to CakePHP and Laravel. No application configuration or PHP changes are required.
+
+Configure these installer values in `.env`:
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `APP_INTERNAL_SUBNET` | `172.30.250.0/24` | IPv4 CIDR for the existing `app_internal` network. Choose a non-overlapping network. |
+| `APP_INTERNAL_DYNAMIC_RANGE` | `172.30.250.128/25` | Subrange for automatically allocated container addresses, inside `APP_INTERNAL_SUBNET` and excluding `PUBLIC_PROXY_IP`. |
+| `PUBLIC_PROXY_IP` | `172.30.250.2` | Fixed Caddy IPv4 address inside the subnet but outside the dynamic range; it must not be the network, broadcast, gateway, or another container's address. |
+| `EXTERNAL_TRUSTED_PROXIES` | empty | Space-separated external load balancer IPs/CIDRs; both IPv4 and IPv6 are supported. Empty means no external proxy is trusted. |
+
+Compose derives the application's `TRUSTED_PROXIES` as `${PUBLIC_PROXY_IP}/32` and sets `TRUST_PROXY=1`; do not put external load balancer networks in the application's list. `TRUSTED_PROXIES` is the image's comma-separated interface, whereas `EXTERNAL_TRUSTED_PROXIES` is Caddy's **space-separated** interface. The Enterprise overlay inherits the same network and trust settings. The `USE_PROXY`/`PROXY_*` settings describe an outbound proxy and are unrelated.
+
+For example, if only the load balancer at `192.0.2.10` connects to Caddy, set `EXTERNAL_TRUSTED_PROXIES=192.0.2.10/32`. For multiple trusted hops, list the actual controlled proxy addresses or narrowly scoped CIDRs, separated by spaces. Configure the external edge to overwrite client-supplied forwarding metadata, or to append its observed client to `X-Forwarded-For`; each trusted hop must sanitize the other forwarded headers it supplies. Do not use a wildcard, all-address ranges, or broad private networks merely to make an address appear correctly.
+
+Caddy ignores incoming `X-Forwarded-For`, `X-Forwarded-Proto`, and `X-Forwarded-Host` from untrusted peers. It also removes their `X-Forwarded-Port`, `X-Forwarded-Prefix`, `Forwarded`, `X-Real-IP`, and `Client-IP`. Legitimate metadata from configured external proxies is preserved. Underscore and dot aliases of these headers are always removed, including on MCP paths. This trust decision uses the original connection peer, independently of the resolved client IP.
+
+Docker checks that the fixed IPv4 address and dynamic range belong to the network subnet and rejects invalid or conflicting allocations when creating the network/container. Keep Caddy outside the dynamic range so earlier-starting services such as MySQL cannot take its address; do not reserve it with `aux_addresses`, which also prevents Caddy from claiming it. `docker compose config --quiet` checks Compose interpolation and structure only; it does not prove that a subnet is available or that the static address is outside the dynamic range. Review existing Docker networks, LAN/VPN routes, and other installations before choosing a subnet. Caddy refuses malformed external IP/CIDR entries at startup.
+
+### Updating an existing installation
+
+This update does **not** migrate or renumber an existing Docker network automatically. Adding explicit IPAM/static addressing requires a planned stack restart. Keep a backup of the previous Compose files, Caddyfile, `.env`, and image digests, and use your normal database/data backup procedure first.
+
+1. Before replacing the running configuration, identify the current `app_internal` network and its allocated addresses. `docker network ls --filter label=com.docker.compose.network=app_internal` may show several projects; select the network belonging to this installation, then inspect it with `docker network inspect <network-name>`. Record its subnet and gateway and check every attached container. If unrelated containers use it, coordinate their removal instead of forcing a disconnect or deleting the network.
+2. Prefer preserving the current subnet: set `APP_INTERNAL_SUBNET` to its actual CIDR, choose an unused, non-gateway `PUBLIC_PROXY_IP` within it, and set `APP_INTERNAL_DYNAMIC_RANGE` to a subrange excluding that address with enough room for the other services. For example, subnet `10.70.8.0/24`, dynamic range `10.70.8.128/25`, and Caddy `10.70.8.25` keep those allocations separate. The new default `.2` may already belong to MySQL or another container in an existing installation. Other containers receive addresses from the new dynamic range when recreated, even if the overall subnet is preserved. Review rules depending on individual addresses. If a different subnet is necessary, also check for collisions and update external firewall/network rules that depend on the old network. Do not copy the defaults over a running deployment without this check.
+3. Confirm that the selected Eramba image contains the Apache trusted-proxy startup support, including `/usr/local/bin/configure-trusted-proxies.php` and its entrypoint integration. Pull the verified image before the maintenance window. Keep the same Compose project name, working directory, overlays, and volume configuration throughout the update, so the application and database keep their existing named volumes.
+4. Render the updated configuration without displaying secrets: `docker compose -f docker-compose.simple-install.yml config --quiet`. For Enterprise, include `-f docker-compose.simple-install.enterprise.yml` in this and every subsequent Compose command.
+5. During the maintenance window, run `docker compose -f docker-compose.simple-install.yml down` **without `-v`**. This removes the project's containers/networks but preserves the named application and database volumes. If Docker reports remaining network endpoints, resolve their ownership before proceeding; do not prune volumes or force-remove a shared network.
+6. Run `docker compose -f docker-compose.simple-install.yml up -d` with the same project identity and overlays. Inspect service status and Caddy/Apache startup logs, then verify application access, HTTPS URLs, MCP access if used, and the client IP from a real request through the intended ingress. Confirm direct forged forwarding headers do not alter that client IP or generated URL. Test the configured external load balancer separately if one is used.
+
+To roll back the network/configuration change, stop the stack with `down` **without `-v`**, restore the saved files and prior image references, and recreate it with the same project identity and volumes. The previous implicit network configuration may select a different subnet when recreated; use the recorded subnet in a temporary IPAM override if exact restoration is needed. Rolling back installer files does not roll back application database migrations; follow the Eramba release's rollback procedure if the image update changed the database. Never delete the data volumes to resolve a networking error.
+
+### Configuration tests
+
+Run `npx --yes bats@1.12.0 tests/*.bats` with Docker, Compose, `jq`, and `curl` available. Compose checks use dummy credentials. Caddy checks use the existing `caddy:2.8-alpine` version in temporary containers with random localhost ports and remove those containers afterward; they do not start MySQL or an Eramba application stack. The allocation test also creates and removes a temporary internal network using the default installer subnet, which must be unused on the test host, and starts two dummy dynamic services before claiming Caddy's static address.

docker-compose.simple-install.yml

@@ -44,6 +44,8 @@ services:
       DB_PASSWORD: ${DB_PASSWORD}
       CACHE_URL: ${CACHE_URL}
       PUBLIC_ADDRESS: ${PUBLIC_ADDRESS}
+      TRUST_PROXY: "1"
+      TRUSTED_PROXIES: ${PUBLIC_PROXY_IP:-172.30.250.2}/32
       USE_PROXY: ${USE_PROXY}
       PROXY_HOST: ${PROXY_HOST}
       PROXY_PORT: ${PROXY_PORT}
@@ -175,12 +177,14 @@ services:
     environment:
       MCP_UPSTREAM: mcp_server:3000
       ERAMBA_UPSTREAM: https://eramba:443
+      EXTERNAL_TRUSTED_PROXIES: ${EXTERNAL_TRUSTED_PROXIES:-}
     depends_on:
       - eramba
       - mcp_server
     networks:
-      - app_internal
-      - host_access
+      app_internal:
+        ipv4_address: ${PUBLIC_PROXY_IP:-172.30.250.2}
+      host_access:
 volumes:
   app:
   data:
@@ -191,5 +195,9 @@ volumes:
 networks:
   app_internal:
     internal: true
+    ipam:
+      config:
+        - subnet: ${APP_INTERNAL_SUBNET:-172.30.250.0/24}
+          ip_range: ${APP_INTERNAL_DYNAMIC_RANGE:-172.30.250.128/25}
   triggers_net:
   host_access:

tests/fixtures/Caddyfile.proxy-test

@@ -0,0 +1,35 @@
+import /etc/caddy/installer.Caddyfile
+
+(aliases) {
+	@aliases query alias-test=1
+	respond @aliases "__FORWARDING_ALIASES__"
+}
+
+https://:8080 {
+	tls /certs/mycert.crt /certs/mycert.key
+	import aliases
+	respond /metadata "{http.request.header.X-Forwarded-Proto}|{http.request.header.X-Forwarded-Host}|{http.request.header.X-Forwarded-Port}|{http.request.header.X-Forwarded-Prefix}|{http.request.header.Forwarded}|{http.request.header.X-Real-IP}|{http.request.header.Client-IP}"
+	respond "{http.request.header.X-Forwarded-For}"
+}
+
+http://:8082 {
+	import aliases
+	respond "mcp"
+}
+
+# Model an external LB that appends its observed client to an incoming chain.
+http://:8081 {
+	reverse_proxy https://127.0.0.1:443 {
+		header_up X-Forwarded-For "{http.request.header.X-Forwarded-For}, 10.40.5.6"
+		header_up X-Forwarded-Proto https
+		header_up X-Forwarded-Host public.example:8443
+		header_up X-Forwarded-Port 8443
+		header_up X-Forwarded-Prefix /grc
+		header_up Forwarded "for=10.40.5.6;proto=https"
+		header_up X-Real-IP 10.40.5.6
+		header_up Client-IP 10.40.5.6
+		transport http {
+			tls_insecure_skip_verify
+		}
+	}
+}

tests/fixtures/compose.env

@@ -0,0 +1,19 @@
+# Deliberately fake values: configuration tests must not read deployment secrets.
+DB_HOST=mysql
+DB_DATABASE=test
+DB_USERNAME=test
+DB_PASSWORD=test-only
+CACHE_URL=Redis://?server=redis&port=6379
+MYSQL_ROOT_PASSWORD=test-only
+USE_PROXY=0
+PROXY_HOST=
+PROXY_PORT=
+USE_PROXY_AUTH=0
+PROXY_AUTH_USER=
+PROXY_AUTH_PASS=
+PUBLIC_ADDRESS=https://localhost:8443
+OAUTH2_INTROSPECTION_CLIENT_ID=test-only
+OAUTH2_INTROSPECTION_CLIENT_SECRET=test-only
+DOCKER_DEPLOYMENT=1
+LDAPTLS_REQCERT=never
+TRIGGER_RUNNER_SHARED_SECRET=test-only

tests/mcp-simple-install.bats

@@ -2,11 +2,15 @@
 
 setup() {
   REPOSITORY_ROOT="$(cd "${BATS_TEST_DIRNAME}/.." && pwd)"
+  render_compose
+}
 
+render_compose() {
   run env -i PATH="$PATH" HOME="$HOME" docker compose \
     --project-directory "$REPOSITORY_ROOT" \
-    --env-file "$REPOSITORY_ROOT/.env" \
+    --env-file "$REPOSITORY_ROOT/tests/fixtures/compose.env" \
     -f "$REPOSITORY_ROOT/docker-compose.simple-install.yml" \
+    "$@" \
     config --format json
 
   [ "$status" -eq 0 ]
@@ -40,3 +44,32 @@ setup() {
   [ "$(jq -r '[.services.public_proxy.volumes[].target] | index("/certs/mycert.crt") != null' <<<"$COMPOSE_CONFIG")" = "true" ]
   [ "$(jq -r '[.services.public_proxy.volumes[].target] | index("/certs/mycert.key") != null' <<<"$COMPOSE_CONFIG")" = "true" ]
 }
+
+@test "application trust follows the fixed Caddy address on the internal network" {
+  [ "$(jq -r '.services.eramba.environment.TRUST_PROXY' <<<"$COMPOSE_CONFIG")" = "1" ]
+  [ "$(jq -r '.services.eramba.environment.TRUSTED_PROXIES' <<<"$COMPOSE_CONFIG")" = "172.30.250.2/32" ]
+  [ "$(jq -r '.services.public_proxy.networks.app_internal.ipv4_address' <<<"$COMPOSE_CONFIG")" = "172.30.250.2" ]
+  [ "$(jq -r '.networks.app_internal.ipam.config[0].subnet' <<<"$COMPOSE_CONFIG")" = "172.30.250.0/24" ]
+  [ "$(jq -r '.networks.app_internal.ipam.config[0].ip_range' <<<"$COMPOSE_CONFIG")" = "172.30.250.128/25" ]
+  [ "$(jq -r '.networks.app_internal.internal' <<<"$COMPOSE_CONFIG")" = "true" ]
+  [ "$(jq -r '.services.public_proxy.environment.EXTERNAL_TRUSTED_PROXIES' <<<"$COMPOSE_CONFIG")" = "" ]
+}
+
+@test "custom proxy address and external trust survive the enterprise overlay" {
+  cat >"$BATS_TEST_TMPDIR/proxies.env" <<'ENV'
+APP_INTERNAL_SUBNET=10.70.8.0/24
+APP_INTERNAL_DYNAMIC_RANGE=10.70.8.128/25
+PUBLIC_PROXY_IP=10.70.8.25
+EXTERNAL_TRUSTED_PROXIES=192.0.2.10/32 2001:db8:1::/64
+ENV
+  render_compose --env-file "$BATS_TEST_TMPDIR/proxies.env" \
+    -f "$REPOSITORY_ROOT/docker-compose.simple-install.enterprise.yml"
+
+  [ "$(jq -r '.services.eramba.image' <<<"$COMPOSE_CONFIG")" = "ghcr.io/eramba/eramba-enterprise:latest" ]
+  [ "$(jq -r '.services.eramba.environment.TRUSTED_PROXIES' <<<"$COMPOSE_CONFIG")" = "10.70.8.25/32" ]
+  [ "$(jq -r '.services.public_proxy.networks.app_internal.ipv4_address' <<<"$COMPOSE_CONFIG")" = "10.70.8.25" ]
+  [ "$(jq -r '.networks.app_internal.ipam.config[0].subnet' <<<"$COMPOSE_CONFIG")" = "10.70.8.0/24" ]
+  [ "$(jq -r '.networks.app_internal.ipam.config[0].ip_range' <<<"$COMPOSE_CONFIG")" = "10.70.8.128/25" ]
+  [ "$(jq -r '.services.public_proxy.environment.EXTERNAL_TRUSTED_PROXIES' <<<"$COMPOSE_CONFIG")" = "192.0.2.10/32 2001:db8:1::/64" ]
+  [ "$(jq -r '.services.eramba.environment.TRUST_PROXY' <<<"$COMPOSE_CONFIG")" = "1" ]
+}

tests/proxy-network.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bats
+
+setup_file() {
+  docker image inspect caddy:2.8-alpine >/dev/null 2>&1 || docker pull caddy:2.8-alpine >/dev/null
+}
+
+setup() {
+  REPOSITORY_ROOT="$(cd "${BATS_TEST_DIRNAME}/.." && pwd)"
+  TEST_NETWORK=""
+  TEST_CONTAINERS=()
+}
+
+teardown() {
+  for container in "${TEST_CONTAINERS[@]}"; do
+    docker rm -f "$container" >/dev/null
+  done
+  if [ -n "$TEST_NETWORK" ]; then
+    docker network rm "$TEST_NETWORK" >/dev/null
+  fi
+}
+
+@test "dynamic services can start before Caddy without taking its reserved IPv4 address" {
+  run env -i PATH="$PATH" HOME="$HOME" docker compose \
+    --project-directory "$REPOSITORY_ROOT" \
+    --env-file "$REPOSITORY_ROOT/tests/fixtures/compose.env" \
+    -f "$REPOSITORY_ROOT/docker-compose.simple-install.yml" config --format json
+  [ "$status" -eq 0 ]
+  config="$output"
+  subnet="$(jq -r '.networks.app_internal.ipam.config[0].subnet' <<<"$config")"
+  dynamic_range="$(jq -r '.networks.app_internal.ipam.config[0].ip_range' <<<"$config")"
+  proxy_ip="$(jq -r '.services.public_proxy.networks.app_internal.ipv4_address' <<<"$config")"
+
+  run docker network create --internal --subnet "$subnet" --ip-range "$dynamic_range" "eramba-proxy-allocation-$$"
+  [ "$status" -eq 0 ]
+  TEST_NETWORK="$output"
+
+  # Model services that Compose starts before public_proxy. A pool covering the
+  # whole subnet would allocate .2 here and prevent Caddy from starting.
+  for service in mysql eramba; do
+    run docker create --network "$TEST_NETWORK" --entrypoint sh caddy:2.8-alpine -c 'sleep 300'
+    [ "$status" -eq 0 ]
+    TEST_CONTAINERS+=("$output")
+    run docker start "${TEST_CONTAINERS[${#TEST_CONTAINERS[@]}-1]}"
+    [ "$status" -eq 0 ]
+  done
+
+  run docker create --network "$TEST_NETWORK" --ip "$proxy_ip" --entrypoint sh caddy:2.8-alpine -c 'sleep 300'
+  [ "$status" -eq 0 ]
+  TEST_CONTAINERS+=("$output")
+  run docker start "${TEST_CONTAINERS[2]}"
+  [ "$status" -eq 0 ]
+  run docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${TEST_CONTAINERS[2]}"
+  [ "$status" -eq 0 ]
+  [ "$output" = "$proxy_ip" ]
+}