fix(secrets): recover Tailscale OAuth through ESO

README.md

@@ -34,7 +34,7 @@ on more than one):
 | [harbor](clusters/unraid-lab/apps/harbor/) | goharbor/harbor | container registry backing the Image Factory |
 | [arc-controller](clusters/unraid-lab/apps/arc-controller/) | actions/gha-runner-scale-set-controller | GitHub Actions Runner Controller |
 | [arc-runners](clusters/unraid-lab/apps/arc-runners/) | ARC runner scale set (ApplicationSet) | self-hosted GH Actions runners, Harbor↔GHCR mirror + homelab jobs |
-| [bitwarden-sm-operator](clusters/unraid-lab/apps/bitwarden-sm-operator/) | charts.bitwarden.com/sm-operator | Bitwarden Secrets Manager operator (also on `oci-lab`) |
+| [external-secrets](clusters/unraid-lab/apps/external-secrets/) | vendored external-secrets | ESO plus Bitwarden SDK server |
 | [radar-pg](clusters/unraid-lab/apps/radar-pg/) | syscode-labs/radar-postgre | radar's Postgres backend |
 | [argocd-ingress](clusters/unraid-lab/apps/argocd-ingress/) | raw manifest | tailscale ingress for the Argo CD UI |
 | [hubble-proxyclass](clusters/unraid-lab/apps/hubble-proxyclass/) | raw manifest | kernel-network ProxyClass for the hubble-ui tailscale ingress proxy |
@@ -43,9 +43,9 @@ on more than one):
 
 | App | Chart / source | Role |
 | --- | --- | --- |
-| [bitwarden-sm-operator](clusters/oci-lab/apps/bitwarden-sm-operator/) | charts.bitwarden.com/sm-operator | Bitwarden Secrets Manager operator (also on `unraid-lab`) |
+| [external-secrets](clusters/oci-lab/apps/external-secrets/) | vendored external-secrets | ESO plus Bitwarden SDK server |
 | [oci-pivot-controller](clusters/oci-lab/apps/oci-pivot-controller/) | ghcr.io/syscode-labs/charts | OCI free-tier node lifecycle controller |
-| [oci-pivot-secrets](clusters/oci-lab/apps/oci-pivot-secrets/) | raw manifest | `BitwardenSecret` feeding oci-pivot-controller's OCI credentials |
+| [oci-pivot-secrets](clusters/oci-lab/apps/oci-pivot-secrets/) | raw manifest | `ExternalSecret` feeding oci-pivot-controller's OCI credentials |
 
 ## Adding or changing an app
 

clusters/oci-lab/apps/external-secrets/application.yaml

@@ -0,0 +1,37 @@
+# ESO is an OCI raw-lane application because the vendored chart, its values,
+# and the Bitwarden store resources must be applied together.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: oci-external-secrets
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "-2"
+    argocd.argoproj.io/compare-options: ServerSideDiff=true
+spec:
+  project: default
+  sources:
+    - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git
+      targetRevision: HEAD
+      path: vendor/external-secrets
+      helm:
+        releaseName: external-secrets
+        valueFiles:
+          - $values/clusters/oci-lab/apps/external-secrets/values.yaml
+    - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git
+      targetRevision: HEAD
+      ref: values
+    - repoURL: https://github.com/syscode-labs/syscode-homelab-gitops-apps.git
+      targetRevision: HEAD
+      path: clusters/oci-lab/apps/external-secrets/manifests
+  destination:
+    name: oci-lab
+    namespace: external-secrets
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+      - SkipDryRunOnMissingResource=true

clusters/oci-lab/apps/external-secrets/manifests/certificate.yaml

@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: bitwarden-sdk-server
+  namespace: external-secrets
+spec:
+  secretName: bitwarden-tls-certs
+  dnsNames:
+    - bitwarden-sdk-server.external-secrets.svc.cluster.local
+    - localhost
+  ipAddresses:
+    - 127.0.0.1
+  issuerRef:
+    name: eso-selfsigned
+    kind: Issuer
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS8
+    size: 2048

clusters/oci-lab/apps/external-secrets/manifests/clustersecretstore.yaml

@@ -0,0 +1,27 @@
+# This reuses the existing OCI Bitwarden machine-account token secret. The
+# token is secret zero and stays out of Git; ESO replaces only its consumer.
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: bitwarden-oci
+spec:
+  conditions:
+    - namespaces: ["oci-pivot-system"]
+  provider:
+    bitwardensecretsmanager:
+      apiURL: https://api.bitwarden.com
+      identityURL: https://identity.bitwarden.com
+      auth:
+        secretRef:
+          credentials:
+            name: bw-auth-token
+            key: token
+            namespace: oci-pivot-system
+      bitwardenServerSDKURL: https://bitwarden-sdk-server.external-secrets.svc.cluster.local:9998
+      caProvider:
+        type: Secret
+        name: bitwarden-tls-certs
+        key: ca.crt
+        namespace: external-secrets
+      organizationID: 28d10c96-b6fb-4e33-b486-b1f700017837
+      projectID: 306ed959-eb18-49e0-a9cf-b4a10008446b

clusters/oci-lab/apps/external-secrets/manifests/issuer.yaml

@@ -0,0 +1,7 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: eso-selfsigned
+  namespace: external-secrets
+spec:
+  selfSigned: {}

clusters/oci-lab/apps/external-secrets/manifests/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - issuer.yaml
+  - certificate.yaml
+  - clustersecretstore.yaml

clusters/oci-lab/apps/external-secrets/values.yaml

@@ -0,0 +1,5 @@
+bitwarden-sdk-server:
+  enabled: true
+
+serviceMonitor:
+  enabled: false

clusters/oci-lab/apps/oci-pivot-secrets/application.yaml

@@ -1,8 +1,6 @@
-# oci-lab-only. BitwardenSecret for oci-pivot-controller's Bitwarden-sourced
-# secrets, synced by the bitwarden-sm-operator chart-app. Stays unhealthy
-# until the owner delivers the bw-auth-token Secret + fills in
-# organizationId/projectId/bwSecretId — see add-external-secrets-bitwarden
-# (syscode-ai-internal-plans).
+# oci-lab-only. ESO syncs oci-pivot-controller's Bitwarden-sourced secret.
+# It stays unhealthy until secret zero (`bw-auth-token`) is restored. ESO is
+# assigned an earlier sync wave so the ExternalSecret CRD exists first.
 #
 # Applied by unraid-lab's Argo (oci-raw lane) to the registered oci-lab
 # cluster — oci-lab runs no local Argo CD.
@@ -11,6 +9,8 @@ kind: Application
 metadata:
   name: oci-pivot-secrets
   namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
 spec:
   project: default
   source:
@@ -26,3 +26,4 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+      - SkipDryRunOnMissingResource=true

clusters/oci-lab/apps/oci-pivot-secrets/manifests/external-secret.yaml

@@ -0,0 +1,20 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: oci-pivot-controller-config
+  namespace: oci-pivot-system
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: bitwarden-oci
+    kind: ClusterSecretStore
+  target:
+    name: oci-pivot-controller-config
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+  data:
+    - secretKey: OCI_COMPARTMENT_ID
+      remoteRef:
+        key: "e2432bdb-b635-4049-94dd-b4a100e69991" # gitleaks:allow -- Bitwarden secret UUID, not a credential

clusters/oci-lab/apps/oci-pivot-secrets/manifests/kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - bitwardensecret.yaml
+  - external-secret.yaml

clusters/unraid-lab/apps/external-secrets/manifests/clustersecretstore.yaml

@@ -9,7 +9,7 @@ metadata:
   name: bitwarden
 spec:
   conditions:
-    - namespaces: ["monitoring", "argocd", "external-secrets"]
+    - namespaces: ["monitoring", "argocd", "external-secrets", "tailscale"]
   provider:
     bitwardensecretsmanager:
       apiURL: https://api.bitwarden.com

clusters/unraid-lab/apps/external-secrets/manifests/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - issuer.yaml
   - certificate.yaml
   - clustersecretstore.yaml
+  - tailscale-operator-oauth.yaml

clusters/unraid-lab/apps/external-secrets/manifests/tailscale-operator-oauth.yaml

@@ -0,0 +1,26 @@
+# The Bitwarden value has exactly two newline-separated fields: client ID then
+# client secret. ESO owns the target so it is recreated after a cluster reset.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: tailscale-operator-oauth
+  namespace: tailscale
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: bitwarden
+    kind: ClusterSecretStore
+  target:
+    name: operator-oauth
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      engineVersion: v2
+      type: Opaque
+      data:
+        client_id: '{{ index (splitList "\n" .oauth) 0 }}'
+        client_secret: '{{ index (splitList "\n" .oauth) 1 }}'
+  data:
+    - secretKey: oauth
+      remoteRef:
+        key: Tailscale Operator Omni

docs/deploy/runbook.md

@@ -129,6 +129,18 @@ sops -d /path/to/homelab-secrets/unraid-lab/bws-token.enc.yaml | \
   kubectl --context omni-unraid-lab apply -f -
 ```
 
-Everything else (grafana-cloud, argocd-pocketid-oidc) then syncs via
-ExternalSecrets against ClusterSecretStore `bitwarden`. Never create
-per-namespace provider tokens; never paste the token into any agent session.
+Everything else (grafana-cloud, argocd-pocketid-oidc, and
+tailscale/operator-oauth) then syncs via ExternalSecrets against
+ClusterSecretStore `bitwarden`. The Tailscale Bitwarden value has the client
+ID on line one and client secret on line two; ESO templates those into the
+keys the operator requires. Never create per-namespace provider tokens; never
+paste the token into any agent session.
+
+## Secrets bootstrap (oci-lab, ESO + Bitwarden)
+
+OCI runs the same vendored ESO chart and Bitwarden SDK-server plugin as
+unraid-lab. Restore its existing machine-account token as
+`oci-pivot-system/bw-auth-token` (key `token`) before ESO can reconcile the
+`bitwarden-oci` ClusterSecretStore. The token is secret zero and stays outside
+this repository; the OCI pivot compartment ID then arrives through its
+ExternalSecret.

values/clusters/oci-lab/oci-pivot-controller.yaml

@@ -1,4 +1,4 @@
-# compartmentId comes from Bitwarden Secrets Manager via bitwarden-sm-operator
+# compartmentId comes from Bitwarden Secrets Manager via External Secrets Operator
 # (see clusters/oci-lab/apps/oci-pivot-secrets) as OCI_COMPARTMENT_ID, not a
 # plain Helm value — was committed here once, had to be scrubbed from git
 # history. Never put the real OCID in this file again.