Skip to content

Commit 6d28587

Browse files
committed
fix(certificates): protect local TLS certificate lifecycle
Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>
1 parent 29af5fc commit 6d28587

7 files changed

Lines changed: 231 additions & 34 deletions

File tree

.github/workflows/common.yml

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,24 @@ jobs:
252252
NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
253253
run: |
254254
docker compose build
255+
docker compose run --rm cert-gen
256+
docker compose run --rm --entrypoint sh cert-gen -c '
257+
test -z "$(find /certs/ca -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)"
258+
test "$(stat -c %a /certs/ca/ca.crt)" = 644
259+
for certificate in /certs/node*/node.crt; do
260+
test "$(stat -c %a "$certificate")" = 644
261+
done
262+
for private_key in /certs/node*/node.key; do
263+
test "$(stat -c %a "$private_key")" = 600
264+
done
265+
'
266+
docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.before
267+
docker compose run --rm cert-gen
268+
docker compose run --rm --entrypoint sh cert-gen -c 'sha256sum /certs/ca/ca.crt /certs/node*/node.crt /certs/node*/node.key' > /tmp/certificate-fingerprints.after
269+
diff --unified /tmp/certificate-fingerprints.before /tmp/certificate-fingerprints.after
255270
docker compose up --detach
256271
printf 'Waiting for cluster'
257-
timeout 60 bash -c -- 'until curl --output /dev/null --silent --insecure --fail https://localhost:2113/-/readiness; do printf '.'; sleep 2; done'
272+
for port in 2111 2112 2113; do
273+
timeout 60 bash -c -- "until curl --output /dev/null --silent --fail --cacert certs/ca/ca.crt https://localhost:$port/-/readiness; do printf '.'; sleep 2; done"
274+
done
258275
docker compose down

docker-compose.yml

Lines changed: 23 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,12 @@
11
services:
2-
volumes-provisioner:
3-
image: hasnat/volumes-provisioner
4-
environment:
5-
PROVISION_DIRECTORIES: "1000:1000:0755:/tmp/certs"
6-
volumes:
7-
- "./certs:/tmp/certs"
8-
network_mode: none
9-
2+
# Local development only. Production certificates must come from the deployment platform's PKI.
103
cert-gen:
11-
image: eventstore/es-gencert-cli:1.0.2
12-
entrypoint: bash
13-
user: "1000:1000"
14-
command: >
15-
-c "mkdir -p ./certs && cd /certs
16-
&& es-gencert-cli create-ca
17-
&& es-gencert-cli create-node -out ./node1 -ip-addresses 127.0.0.1,172.30.240.11 -dns-names localhost
18-
&& es-gencert-cli create-node -out ./node2 -ip-addresses 127.0.0.1,172.30.240.12 -dns-names localhost
19-
&& es-gencert-cli create-node -out ./node3 -ip-addresses 127.0.0.1,172.30.240.13 -dns-names localhost
20-
&& find . -type f -print0 | xargs -0 chmod 666"
4+
build:
5+
context: ./
6+
dockerfile: scripts/certificates/Dockerfile
217
volumes:
228
- "./certs:/certs"
23-
depends_on:
24-
- volumes-provisioner
9+
network_mode: none
2510

2611
esdb-node1:
2712
build:
@@ -33,8 +18,8 @@ services:
3318
environment:
3419
- EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113
3520
- EVENTSTORE_REPLICATION_IP=172.30.240.11
36-
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node1/node.crt
37-
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node1/node.key
21+
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
22+
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
3823
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
3924
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2111
4025
ports:
@@ -43,10 +28,12 @@ services:
4328
clusternetwork:
4429
ipv4_address: 172.30.240.11
4530
volumes:
46-
- ./certs:/etc/eventstore/certs
31+
- ./certs/ca:/etc/eventstore/certs/ca:ro
32+
- ./certs/node1:/etc/eventstore/certs/node:ro
4733
restart: unless-stopped
4834
depends_on:
49-
- cert-gen
35+
cert-gen:
36+
condition: service_completed_successfully
5037

5138
esdb-node2:
5239
build:
@@ -58,8 +45,8 @@ services:
5845
environment:
5946
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113
6047
- EVENTSTORE_REPLICATION_IP=172.30.240.12
61-
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node2/node.crt
62-
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node2/node.key
48+
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
49+
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
6350
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
6451
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2112
6552
ports:
@@ -68,10 +55,12 @@ services:
6855
clusternetwork:
6956
ipv4_address: 172.30.240.12
7057
volumes:
71-
- ./certs:/etc/eventstore/certs
58+
- ./certs/ca:/etc/eventstore/certs/ca:ro
59+
- ./certs/node2:/etc/eventstore/certs/node:ro
7260
restart: unless-stopped
7361
depends_on:
74-
- cert-gen
62+
cert-gen:
63+
condition: service_completed_successfully
7564

7665
esdb-node3:
7766
build:
@@ -83,8 +72,8 @@ services:
8372
environment:
8473
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113
8574
- EVENTSTORE_REPLICATION_IP=172.30.240.13
86-
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node3/node.crt
87-
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node3/node.key
75+
- EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt
76+
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key
8877
- EVENTSTORE_ADVERTISE_HOST_TO_CLIENT_AS=127.0.0.1
8978
- EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2113
9079
ports:
@@ -93,10 +82,12 @@ services:
9382
clusternetwork:
9483
ipv4_address: 172.30.240.13
9584
volumes:
96-
- ./certs:/etc/eventstore/certs
85+
- ./certs/ca:/etc/eventstore/certs/ca:ro
86+
- ./certs/node3:/etc/eventstore/certs/node:ro
9787
restart: unless-stopped
9888
depends_on:
99-
- cert-gen
89+
cert-gen:
90+
condition: service_completed_successfully
10091

10192
networks:
10293
clusternetwork:

docs/installation.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,9 @@ container.
6767

6868
Before running a durable node or cluster:
6969

70-
- Provide node certificates explicitly.
70+
- Choose an approved PKI or issuer and follow the [production certificate management](security.md#production-certificate-management) guidance.
71+
- Issue a separate node certificate with the required SAN, common-name, and key-usage policy for each node.
72+
- Mount node certificates, private keys, and trusted roots read-only, and automate renewal plus explicit reload or rolling restart.
7173
- Decide whether clients use TLS and configure the connection strings
7274
accordingly.
7375
- Configure authentication methods in [Security](security.md).

docs/operations.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -507,6 +507,10 @@ As of version 23.10.0, it is possible to do a rolling update with new certificat
507507

508508
The next step is to replace the outdated certificates with the newly generated certificates.
509509

510+
Replace each node certificate and its private key as one versioned unit. Do not expose a new certificate with an old key, or a new key with an old certificate. Kubernetes Secret and CSI projections can take time to update, so confirm that all mounted files contain the expected version before reloading the node. Certificate files mounted with `subPath` do not receive Secret updates.
511+
512+
When rotating the CA, establish overlapping trust before changing node certificates. Add both the old and new trusted roots to every node and client. Reload or restart every process, or verify that a client dynamically reloads its trust bundle, before issuing or activating any node certificate from the new CA. Then issue and activate node certificates from the new CA one node at a time. Remove the old root only after all node and client traffic has been verified against the new chain.
513+
510514
If you are using symlinks, then you can update the symlink to point it to the new certificates.
511515

512516
#### Linux OS
@@ -531,6 +535,8 @@ You can reload the certificate configuration without restarting the node by call
531535
`Operations.ReloadConfig` method with the credentials of an `admin` or `ops` user. You can also
532536
reload the configuration from the _Operations_ page of the Admin UI.
533537

538+
Automated certificate renewal is not complete until the running node loads the new files. If your certificate controller cannot call `Operations.ReloadConfig` safely, perform a rolling restart one node at a time. Verify successful certificate-loading logs, readiness, and cluster membership before continuing to the next node.
539+
534540
#### Linux OS
535541

536542
Linux users can also send the SIGHUP signal to the TrogonEventStore to reload the certificates.

docs/security.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,8 @@ If multiple matching root certificates are found, then the root certificate with
323323

324324
Use your platform certificate tooling, public CA, or private CA process to create certificates that match your deployment. The built-in development mode can generate a certificate for one secure node on localhost, but it is not a cluster PKI bootstrap mechanism.
325325

326+
The repository Docker Compose configuration uses a small OpenSSL-based bootstrap helper for disposable local clusters. It creates a private CA for the persisted local certificate set, but does not retain the CA private key in the directory mounted into the nodes. It is not a production PKI.
327+
326328
For a multi-node deployment, provision:
327329

328330
- A trusted CA certificate available to every node and client.
@@ -337,6 +339,23 @@ compatibility.
337339

338340
Keep CA private keys outside the node and client environments. Install only the CA certificate, node certificate, and node private key required by each machine.
339341

342+
#### Production certificate management
343+
344+
TrogonEventStore does not integrate directly with a certificate authority or secret store. It loads the node certificate, private key, intermediate chain, and trusted roots from the configured files. Use the certificate lifecycle system already operated by your platform to issue and deliver those files.
345+
346+
For Kubernetes, issue a separate certificate for every StatefulSet pod or stable node identity. Use [cert-manager](https://cert-manager.io/) with an approved issuer, or a Secrets Store CSI provider, and mount the certificate, private key, and trusted roots as read-only volumes. Do not mount certificate files using `subPath`, because Kubernetes does not propagate Secret updates to those mounts. Limit access to the TrogonEventStore service account.
347+
348+
Common provider patterns include:
349+
350+
- **Vault PKI:** use a constrained Vault PKI role through cert-manager's Vault issuer or an authenticated Vault Agent. Permit only the DNS names, IP addresses, usages, and lifetimes required by the cluster.
351+
- **AWS:** use [AWS Private CA Connector for Kubernetes](https://docs.aws.amazon.com/privateca/latest/userguide/PcaKubernetes.html) with cert-manager. Request both Server Authentication and Client Authentication usages for node certificates.
352+
- **Azure:** use [Azure Key Vault with the Secrets Store CSI driver](https://learn.microsoft.com/azure/aks/csi-secrets-store-driver). Retrieve a certificate as a secret when the node needs its private key; Key Vault key or certificate objects alone do not provide the complete private-key material to the file-based loader.
353+
- **Google Cloud:** issue node certificates from Certificate Authority Service and deliver the PEM files through a Kubernetes Secret or [Secret Manager CSI](https://cloud.google.com/secret-manager/docs/secret-manager-managed-csi-component).
354+
355+
Managed load-balancer certificates do not replace node-to-node certificate authentication. The current file-based loader also cannot use a non-exportable private key held only by an HSM or cloud key service.
356+
357+
Certificate renewal updates the mounted files but does not by itself activate them in the running process. After the files change, reload each node using `Operations.ReloadConfig`, the Admin UI, or `SIGHUP` on Linux. If the platform cannot invoke reload safely, restart nodes one at a time. See [Certificate update upon expiry](operations.md#certificate-update-upon-expiry).
358+
340359
::: warning
341360
Keep certificate private keys readable only by the account running TrogonEventStore. Restrictive permissions
342361
reduce the risk of another local account reading the key.

scripts/certificates/Dockerfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
2+
3+
RUN apk add --no-cache openssl=3.5.7-r0
4+
5+
COPY --chmod=755 scripts/certificates/generate-cluster-certificates.sh /usr/local/bin/generate-cluster-certificates
6+
7+
ENTRYPOINT ["/usr/local/bin/generate-cluster-certificates"]
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#!/usr/bin/env sh
2+
3+
set -eu
4+
5+
output_directory="${CERT_OUTPUT_DIRECTORY:-/certs}"
6+
ca_days="${CERT_CA_DAYS:-3650}"
7+
node_days="${CERT_NODE_DAYS:-825}"
8+
node_common_name="${CERT_NODE_COMMON_NAME:-trogondb-node}"
9+
output_owner="${CERT_OUTPUT_OWNER:-10000:10000}"
10+
11+
expected_files="
12+
$output_directory/ca/ca.crt
13+
$output_directory/node1/node.crt
14+
$output_directory/node1/node.key
15+
$output_directory/node2/node.crt
16+
$output_directory/node2/node.key
17+
$output_directory/node3/node.crt
18+
$output_directory/node3/node.key
19+
"
20+
21+
validate_node() {
22+
node_name="$1"
23+
node_ip="$2"
24+
node_directory="$output_directory/$node_name"
25+
26+
openssl verify -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
27+
openssl verify -purpose sslserver -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
28+
openssl verify -purpose sslclient -CAfile "$output_directory/ca/ca.crt" "$node_directory/node.crt" >/dev/null
29+
openssl x509 -in "$node_directory/node.crt" -noout -checkend 0 >/dev/null
30+
openssl x509 -in "$node_directory/node.crt" -noout -checkhost localhost >/dev/null
31+
openssl x509 -in "$node_directory/node.crt" -noout -checkhost "esdb-$node_name" >/dev/null
32+
openssl x509 -in "$node_directory/node.crt" -noout -checkip 127.0.0.1 >/dev/null
33+
openssl x509 -in "$node_directory/node.crt" -noout -checkip "$node_ip" >/dev/null
34+
test "$(openssl x509 -in "$node_directory/node.crt" -noout -subject -nameopt RFC2253)" = "subject=CN=$node_common_name"
35+
36+
certificate_public_key="$(mktemp)"
37+
private_public_key="$(mktemp)"
38+
openssl x509 -in "$node_directory/node.crt" -pubkey -noout >"$certificate_public_key"
39+
openssl pkey -in "$node_directory/node.key" -pubout >"$private_public_key" 2>/dev/null
40+
cmp "$certificate_public_key" "$private_public_key" >/dev/null
41+
rm -f "$certificate_public_key" "$private_public_key"
42+
}
43+
44+
normalize_output_permissions() {
45+
chown -R "$output_owner" "$output_directory"
46+
chmod 755 "$output_directory" "$output_directory/ca"
47+
chmod 700 "$output_directory"/node*
48+
chmod 600 "$output_directory"/node*/node.key
49+
chmod 644 "$output_directory/ca/ca.crt" "$output_directory"/node*/node.crt
50+
}
51+
52+
validate_existing_certificates() {
53+
openssl verify -CAfile "$output_directory/ca/ca.crt" "$output_directory/ca/ca.crt" >/dev/null
54+
validate_node node1 172.30.240.11
55+
validate_node node2 172.30.240.12
56+
validate_node node3 172.30.240.13
57+
}
58+
59+
existing_files=0
60+
missing_files=0
61+
62+
for output_path in "$output_directory" "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3"; do
63+
if [ -L "$output_path" ] || { [ -e "$output_path" ] && [ ! -d "$output_path" ]; }; then
64+
echo "Certificate output path '$output_path' must be a real directory, not a link or another file type." >&2
65+
exit 1
66+
fi
67+
done
68+
69+
if [ -d "$output_directory/ca" ]; then
70+
unexpected_ca_entry="$(find "$output_directory/ca" -mindepth 1 -maxdepth 1 ! -name ca.crt -print -quit)"
71+
if [ -n "$unexpected_ca_entry" ]; then
72+
echo "Unexpected content exists in '$output_directory/ca'. Remove the local certificate directory and regenerate it so only the public ca.crt is exposed to nodes." >&2
73+
exit 1
74+
fi
75+
fi
76+
77+
for expected_file in $expected_files; do
78+
if [ -L "$expected_file" ] || { [ -e "$expected_file" ] && [ ! -f "$expected_file" ]; }; then
79+
echo "Certificate output '$expected_file' must be a regular file, not a link or another file type." >&2
80+
exit 1
81+
elif [ -f "$expected_file" ]; then
82+
existing_files=$((existing_files + 1))
83+
else
84+
missing_files=$((missing_files + 1))
85+
fi
86+
done
87+
88+
if [ "$existing_files" -gt 0 ]; then
89+
if [ "$missing_files" -gt 0 ]; then
90+
echo "Certificate output is incomplete. Remove '$output_directory' before regenerating it." >&2
91+
exit 1
92+
fi
93+
94+
normalize_output_permissions
95+
validate_existing_certificates
96+
echo "Using the existing validated cluster certificates in '$output_directory'."
97+
exit 0
98+
fi
99+
100+
umask 077
101+
mkdir -p "$output_directory/ca" "$output_directory/node1" "$output_directory/node2" "$output_directory/node3"
102+
private_directory="$(mktemp -d)"
103+
trap 'rm -rf "$private_directory"' EXIT
104+
ca_key="$private_directory/ca.key"
105+
106+
openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:3072 -out "$ca_key" 2>/dev/null
107+
openssl req -x509 -new -sha256 \
108+
-key "$ca_key" \
109+
-out "$output_directory/ca/ca.crt" \
110+
-days "$ca_days" \
111+
-subj "/CN=TrogonEventStore Development CA" \
112+
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
113+
-addext "keyUsage=critical,keyCertSign,cRLSign" \
114+
-addext "subjectKeyIdentifier=hash"
115+
116+
generate_node() {
117+
node_name="$1"
118+
node_ip="$2"
119+
serial_number="$3"
120+
node_directory="$output_directory/$node_name"
121+
extension_file="$private_directory/$node_name.extensions"
122+
request_file="$private_directory/$node_name.csr"
123+
124+
cat >"$extension_file" <<EOF
125+
[node]
126+
basicConstraints=critical,CA:FALSE
127+
keyUsage=critical,digitalSignature,keyEncipherment
128+
extendedKeyUsage=serverAuth,clientAuth
129+
subjectKeyIdentifier=hash
130+
authorityKeyIdentifier=keyid,issuer
131+
subjectAltName=DNS:localhost,DNS:esdb-$node_name,IP:127.0.0.1,IP:$node_ip
132+
EOF
133+
134+
openssl req -new -newkey rsa:3072 -nodes -sha256 \
135+
-keyout "$node_directory/node.key" \
136+
-out "$request_file" \
137+
-subj "/CN=$node_common_name" 2>/dev/null
138+
openssl x509 -req -sha256 \
139+
-in "$request_file" \
140+
-CA "$output_directory/ca/ca.crt" \
141+
-CAkey "$ca_key" \
142+
-set_serial "$serial_number" \
143+
-days "$node_days" \
144+
-extfile "$extension_file" \
145+
-extensions node \
146+
-out "$node_directory/node.crt"
147+
}
148+
149+
generate_node node1 172.30.240.11 1001
150+
generate_node node2 172.30.240.12 1002
151+
generate_node node3 172.30.240.13 1003
152+
153+
normalize_output_permissions
154+
validate_existing_certificates
155+
echo "Generated and validated cluster certificates in '$output_directory'."

0 commit comments

Comments
 (0)