From 61930b06e476a9ede03a47ff8ad9e21968156b6b Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sun, 30 Aug 2026 13:38:14 +0100 Subject: [PATCH] fix(secrets): recover Tailscale OAuth through ESO --- README.md | 6 +-- .../apps/bitwarden-sm-operator/app.yaml | 7 ---- .../apps/external-secrets/application.yaml | 37 +++++++++++++++++++ .../manifests/certificate.yaml | 19 ++++++++++ .../manifests/clustersecretstore.yaml | 27 ++++++++++++++ .../external-secrets/manifests/issuer.yaml | 7 ++++ .../manifests/kustomization.yaml | 6 +++ .../oci-lab/apps/external-secrets/values.yaml | 5 +++ .../apps/oci-pivot-secrets/application.yaml | 11 +++--- .../manifests/bitwardensecret.yaml | 15 -------- .../manifests/external-secret.yaml | 20 ++++++++++ .../manifests/kustomization.yaml | 2 +- .../manifests/clustersecretstore.yaml | 2 +- .../manifests/kustomization.yaml | 1 + .../manifests/tailscale-operator-oauth.yaml | 26 +++++++++++++ docs/deploy/runbook.md | 18 +++++++-- .../oci-lab/oci-pivot-controller.yaml | 2 +- 17 files changed, 175 insertions(+), 36 deletions(-) delete mode 100644 clusters/oci-lab/apps/bitwarden-sm-operator/app.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/application.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/manifests/certificate.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/manifests/clustersecretstore.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/manifests/issuer.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/manifests/kustomization.yaml create mode 100644 clusters/oci-lab/apps/external-secrets/values.yaml delete mode 100644 clusters/oci-lab/apps/oci-pivot-secrets/manifests/bitwardensecret.yaml create mode 100644 clusters/oci-lab/apps/oci-pivot-secrets/manifests/external-secret.yaml create mode 100644 clusters/unraid-lab/apps/external-secrets/manifests/tailscale-operator-oauth.yaml diff --git a/README.md b/README.md index 6f32401..46c6861 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ on more than one): | [harbor](clusters/unraid-lab/apps/harbor/) | goharbor/harbor | container registry backing the Image Factory | | [arc-controller](clusters/unraid-lab/apps/arc-controller/) | actions/gha-runner-scale-set-controller | GitHub Actions Runner Controller | | [arc-runners](clusters/unraid-lab/apps/arc-runners/) | ARC runner scale set (ApplicationSet) | self-hosted GH Actions runners, Harbor↔GHCR mirror + homelab jobs | -| [bitwarden-sm-operator](clusters/unraid-lab/apps/bitwarden-sm-operator/) | charts.bitwarden.com/sm-operator | Bitwarden Secrets Manager operator (also on `oci-lab`) | +| [external-secrets](clusters/unraid-lab/apps/external-secrets/) | vendored external-secrets | ESO plus Bitwarden SDK server | | [radar-pg](clusters/unraid-lab/apps/radar-pg/) | syscode-labs/radar-postgre | radar's Postgres backend | | [argocd-ingress](clusters/unraid-lab/apps/argocd-ingress/) | raw manifest | tailscale ingress for the Argo CD UI | | [hubble-proxyclass](clusters/unraid-lab/apps/hubble-proxyclass/) | raw manifest | kernel-network ProxyClass for the hubble-ui tailscale ingress proxy | @@ -43,9 +43,9 @@ on more than one): | App | Chart / source | Role | | --- | --- | --- | -| [bitwarden-sm-operator](clusters/oci-lab/apps/bitwarden-sm-operator/) | charts.bitwarden.com/sm-operator | Bitwarden Secrets Manager operator (also on `unraid-lab`) | +| [external-secrets](clusters/oci-lab/apps/external-secrets/) | vendored external-secrets | ESO plus Bitwarden SDK server | | [oci-pivot-controller](clusters/oci-lab/apps/oci-pivot-controller/) | ghcr.io/syscode-labs/charts | OCI free-tier node lifecycle controller | -| [oci-pivot-secrets](clusters/oci-lab/apps/oci-pivot-secrets/) | raw manifest | `BitwardenSecret` feeding oci-pivot-controller's OCI credentials | +| [oci-pivot-secrets](clusters/oci-lab/apps/oci-pivot-secrets/) | raw manifest | `ExternalSecret` feeding oci-pivot-controller's OCI credentials | ## Adding or changing an app diff --git a/clusters/oci-lab/apps/bitwarden-sm-operator/app.yaml b/clusters/oci-lab/apps/bitwarden-sm-operator/app.yaml deleted file mode 100644 index ae9c34d..0000000 --- a/clusters/oci-lab/apps/bitwarden-sm-operator/app.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# oci-lab-only chart-app. Bitwarden's own first-party Secrets Manager -# operator — no sdk-server companion, no TLS setup needed (unlike External -# Secrets Operator's generic Bitwarden provider, which this replaced). -chart: sm-operator -repoURL: https://charts.bitwarden.com/ -version: "2.0.3" -namespace: sm-operator-system diff --git a/clusters/oci-lab/apps/external-secrets/application.yaml b/clusters/oci-lab/apps/external-secrets/application.yaml new file mode 100644 index 0000000..3ac6f99 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/application.yaml @@ -0,0 +1,37 @@ +# ESO is an OCI raw-lane application because the vendored chart, its values, +# and the Bitwarden store resources must be applied together. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: oci-external-secrets + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-2" + argocd.argoproj.io/compare-options: ServerSideDiff=true +spec: + project: default + sources: + - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git + targetRevision: HEAD + path: vendor/external-secrets + helm: + releaseName: external-secrets + valueFiles: + - $values/clusters/oci-lab/apps/external-secrets/values.yaml + - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git + targetRevision: HEAD + ref: values + - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git + targetRevision: HEAD + path: clusters/oci-lab/apps/external-secrets/manifests + destination: + name: oci-lab + namespace: external-secrets + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - SkipDryRunOnMissingResource=true diff --git a/clusters/oci-lab/apps/external-secrets/manifests/certificate.yaml b/clusters/oci-lab/apps/external-secrets/manifests/certificate.yaml new file mode 100644 index 0000000..509e790 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/manifests/certificate.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: bitwarden-sdk-server + namespace: external-secrets +spec: + secretName: bitwarden-tls-certs + dnsNames: + - bitwarden-sdk-server.external-secrets.svc.cluster.local + - localhost + ipAddresses: + - 127.0.0.1 + issuerRef: + name: eso-selfsigned + kind: Issuer + privateKey: + algorithm: RSA + encoding: PKCS8 + size: 2048 diff --git a/clusters/oci-lab/apps/external-secrets/manifests/clustersecretstore.yaml b/clusters/oci-lab/apps/external-secrets/manifests/clustersecretstore.yaml new file mode 100644 index 0000000..fea1a81 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/manifests/clustersecretstore.yaml @@ -0,0 +1,27 @@ +# This reuses the existing OCI Bitwarden machine-account token secret. The +# token is secret zero and stays out of Git; ESO replaces only its consumer. +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: bitwarden-oci +spec: + conditions: + - namespaces: ["oci-pivot-system"] + provider: + bitwardensecretsmanager: + apiURL: https://api.bitwarden.com + identityURL: https://identity.bitwarden.com + auth: + secretRef: + credentials: + name: bw-auth-token + key: token + namespace: oci-pivot-system + bitwardenServerSDKURL: https://bitwarden-sdk-server.external-secrets.svc.cluster.local:9998 + caProvider: + type: Secret + name: bitwarden-tls-certs + key: ca.crt + namespace: external-secrets + organizationID: 28d10c96-b6fb-4e33-b486-b1f700017837 + projectID: 306ed959-eb18-49e0-a9cf-b4a10008446b diff --git a/clusters/oci-lab/apps/external-secrets/manifests/issuer.yaml b/clusters/oci-lab/apps/external-secrets/manifests/issuer.yaml new file mode 100644 index 0000000..b3025d3 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/manifests/issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: eso-selfsigned + namespace: external-secrets +spec: + selfSigned: {} diff --git a/clusters/oci-lab/apps/external-secrets/manifests/kustomization.yaml b/clusters/oci-lab/apps/external-secrets/manifests/kustomization.yaml new file mode 100644 index 0000000..c8b58b6 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/manifests/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - issuer.yaml + - certificate.yaml + - clustersecretstore.yaml diff --git a/clusters/oci-lab/apps/external-secrets/values.yaml b/clusters/oci-lab/apps/external-secrets/values.yaml new file mode 100644 index 0000000..ec23767 --- /dev/null +++ b/clusters/oci-lab/apps/external-secrets/values.yaml @@ -0,0 +1,5 @@ +bitwarden-sdk-server: + enabled: true + +serviceMonitor: + enabled: false diff --git a/clusters/oci-lab/apps/oci-pivot-secrets/application.yaml b/clusters/oci-lab/apps/oci-pivot-secrets/application.yaml index 723727f..972e859 100644 --- a/clusters/oci-lab/apps/oci-pivot-secrets/application.yaml +++ b/clusters/oci-lab/apps/oci-pivot-secrets/application.yaml @@ -1,8 +1,6 @@ -# oci-lab-only. BitwardenSecret for oci-pivot-controller's Bitwarden-sourced -# secrets, synced by the bitwarden-sm-operator chart-app. Stays unhealthy -# until the owner delivers the bw-auth-token Secret + fills in -# organizationId/projectId/bwSecretId — see add-external-secrets-bitwarden -# (syscode-ai-internal-plans). +# oci-lab-only. ESO syncs oci-pivot-controller's Bitwarden-sourced secret. +# It stays unhealthy until secret zero (`bw-auth-token`) is restored. ESO is +# assigned an earlier sync wave so the ExternalSecret CRD exists first. # # Applied by unraid-lab's Argo (oci-raw lane) to the registered oci-lab # cluster — oci-lab runs no local Argo CD. @@ -11,6 +9,8 @@ kind: Application metadata: name: oci-pivot-secrets namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-1" spec: project: default source: @@ -26,3 +26,4 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true + - SkipDryRunOnMissingResource=true diff --git a/clusters/oci-lab/apps/oci-pivot-secrets/manifests/bitwardensecret.yaml b/clusters/oci-lab/apps/oci-pivot-secrets/manifests/bitwardensecret.yaml deleted file mode 100644 index c9d53f1..0000000 --- a/clusters/oci-lab/apps/oci-pivot-secrets/manifests/bitwardensecret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: k8s.bitwarden.com/v1 -kind: BitwardenSecret -metadata: - name: oci-pivot-controller-config - namespace: oci-pivot-system -spec: - organizationId: "28d10c96-b6fb-4e33-b486-b1f700017837" - projectId: "306ed959-eb18-49e0-a9cf-b4a10008446b" - secretName: oci-pivot-controller-config - authToken: - secretName: bw-auth-token - secretKey: token - map: - - bwSecretId: "e2432bdb-b635-4049-94dd-b4a100e69991" # gitleaks:allow -- Bitwarden secret UUID, not a credential - secretKeyName: OCI_COMPARTMENT_ID diff --git a/clusters/oci-lab/apps/oci-pivot-secrets/manifests/external-secret.yaml b/clusters/oci-lab/apps/oci-pivot-secrets/manifests/external-secret.yaml new file mode 100644 index 0000000..e395b17 --- /dev/null +++ b/clusters/oci-lab/apps/oci-pivot-secrets/manifests/external-secret.yaml @@ -0,0 +1,20 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: oci-pivot-controller-config + namespace: oci-pivot-system +spec: + refreshInterval: 1h + secretStoreRef: + name: bitwarden-oci + kind: ClusterSecretStore + target: + name: oci-pivot-controller-config + creationPolicy: Owner + deletionPolicy: Retain + template: + type: Opaque + data: + - secretKey: OCI_COMPARTMENT_ID + remoteRef: + key: "e2432bdb-b635-4049-94dd-b4a100e69991" # gitleaks:allow -- Bitwarden secret UUID, not a credential diff --git a/clusters/oci-lab/apps/oci-pivot-secrets/manifests/kustomization.yaml b/clusters/oci-lab/apps/oci-pivot-secrets/manifests/kustomization.yaml index 31c4ba9..219c748 100644 --- a/clusters/oci-lab/apps/oci-pivot-secrets/manifests/kustomization.yaml +++ b/clusters/oci-lab/apps/oci-pivot-secrets/manifests/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - bitwardensecret.yaml + - external-secret.yaml diff --git a/clusters/unraid-lab/apps/external-secrets/manifests/clustersecretstore.yaml b/clusters/unraid-lab/apps/external-secrets/manifests/clustersecretstore.yaml index ec01de1..9dcadaf 100644 --- a/clusters/unraid-lab/apps/external-secrets/manifests/clustersecretstore.yaml +++ b/clusters/unraid-lab/apps/external-secrets/manifests/clustersecretstore.yaml @@ -9,7 +9,7 @@ metadata: name: bitwarden spec: conditions: - - namespaces: ["monitoring", "argocd", "external-secrets"] + - namespaces: ["monitoring", "argocd", "external-secrets", "tailscale"] provider: bitwardensecretsmanager: apiURL: https://api.bitwarden.com diff --git a/clusters/unraid-lab/apps/external-secrets/manifests/kustomization.yaml b/clusters/unraid-lab/apps/external-secrets/manifests/kustomization.yaml index c8b58b6..9c4da37 100644 --- a/clusters/unraid-lab/apps/external-secrets/manifests/kustomization.yaml +++ b/clusters/unraid-lab/apps/external-secrets/manifests/kustomization.yaml @@ -4,3 +4,4 @@ resources: - issuer.yaml - certificate.yaml - clustersecretstore.yaml + - tailscale-operator-oauth.yaml diff --git a/clusters/unraid-lab/apps/external-secrets/manifests/tailscale-operator-oauth.yaml b/clusters/unraid-lab/apps/external-secrets/manifests/tailscale-operator-oauth.yaml new file mode 100644 index 0000000..1f4e99b --- /dev/null +++ b/clusters/unraid-lab/apps/external-secrets/manifests/tailscale-operator-oauth.yaml @@ -0,0 +1,26 @@ +# The Bitwarden value has exactly two newline-separated fields: client ID then +# client secret. ESO owns the target so it is recreated after a cluster reset. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: tailscale-operator-oauth + namespace: tailscale +spec: + refreshInterval: 1h + secretStoreRef: + name: bitwarden + kind: ClusterSecretStore + target: + name: operator-oauth + creationPolicy: Owner + deletionPolicy: Retain + template: + engineVersion: v2 + type: Opaque + data: + client_id: '{{ index (splitList "\n" .oauth) 0 }}' + client_secret: '{{ index (splitList "\n" .oauth) 1 }}' + data: + - secretKey: oauth + remoteRef: + key: Tailscale Operator Omni diff --git a/docs/deploy/runbook.md b/docs/deploy/runbook.md index a9640fa..711e5de 100644 --- a/docs/deploy/runbook.md +++ b/docs/deploy/runbook.md @@ -129,6 +129,18 @@ sops -d /path/to/homelab-secrets/unraid-lab/bws-token.enc.yaml | \ kubectl --context omni-unraid-lab apply -f - ``` -Everything else (grafana-cloud, argocd-pocketid-oidc) then syncs via -ExternalSecrets against ClusterSecretStore `bitwarden`. Never create -per-namespace provider tokens; never paste the token into any agent session. +Everything else (grafana-cloud, argocd-pocketid-oidc, and +tailscale/operator-oauth) then syncs via ExternalSecrets against +ClusterSecretStore `bitwarden`. The Tailscale Bitwarden value has the client +ID on line one and client secret on line two; ESO templates those into the +keys the operator requires. Never create per-namespace provider tokens; never +paste the token into any agent session. + +## Secrets bootstrap (oci-lab, ESO + Bitwarden) + +OCI runs the same vendored ESO chart and Bitwarden SDK-server plugin as +unraid-lab. Restore its existing machine-account token as +`oci-pivot-system/bw-auth-token` (key `token`) before ESO can reconcile the +`bitwarden-oci` ClusterSecretStore. The token is secret zero and stays outside +this repository; the OCI pivot compartment ID then arrives through its +ExternalSecret. diff --git a/values/clusters/oci-lab/oci-pivot-controller.yaml b/values/clusters/oci-lab/oci-pivot-controller.yaml index f9be776..8c353b3 100644 --- a/values/clusters/oci-lab/oci-pivot-controller.yaml +++ b/values/clusters/oci-lab/oci-pivot-controller.yaml @@ -1,4 +1,4 @@ -# compartmentId comes from Bitwarden Secrets Manager via bitwarden-sm-operator +# compartmentId comes from Bitwarden Secrets Manager via External Secrets Operator # (see clusters/oci-lab/apps/oci-pivot-secrets) as OCI_COMPARTMENT_ID, not a # plain Helm value — was committed here once, had to be scrubbed from git # history. Never put the real OCID in this file again.