Add supervised remote Mongo recovery

.env.template

@@ -25,13 +25,28 @@ MONGO_URI=mongodb://localhost:27017/
 # Database name (default: von_db)
 VON_DB_NAME=von_db
 
-# Local fallback URI (used if primary connection fails)
+# Local fallback is disabled by default so a remote primary cannot silently
+# downgrade to an old workstation database. Set this to 1 only as an explicit
+# local-development choice.
 MONGO_LOCAL_URI=mongodb://127.0.0.1:27017/?directConnection=true
-MONGO_ALLOW_LOCAL_FALLBACK=1
-
-# Hosted/cloud recommendation: disable local fallback so Atlas failures do not
-# silently fall back to localhost.
-# MONGO_ALLOW_LOCAL_FALLBACK=0
+MONGO_ALLOW_LOCAL_FALLBACK=0
+# MONGO_ALLOW_LOCAL_FALLBACK=1
+
+# Optional remote recovery paths after the primary URI. For a pure DNS/SRV
+# failure Von tries the lower-latency direct-host Atlas URI first. For TLS,
+# TCP, health-check transport, or shared-retry operation failures it tries the
+# supervised SSH forwards first. The explicit local-development fallback, if
+# enabled, remains last.
+#
+# Supervised SSH forwards on loopback. Von derives the credential-bearing
+# Mongo URIs in memory and selects only the writable replica member.
+# MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS=127.0.0.1:27018,127.0.0.1:27019,127.0.0.1:27020
+# MONGO_SSH_TUNNEL_EXPECTED_REPLICA_SET=atlas-example-shard-0
+# A direct-host Atlas URI, useful when only SRV lookup is failing.
+# MONGO_DNS_FALLBACK_URI=mongodb://YOUR_DIRECT_ATLAS_URI
+#
+# How long to keep a working fallback before probing the primary route again.
+# VON_MONGO_FALLBACK_STICKY_SECONDS=300
 #
 # Optional hosted Mongo startup hardening:
 # VON_MONGO_STRICT_STARTUP=1

README.md

@@ -111,10 +111,18 @@ MongoDB will be automatically installed during the setup process (via `setup_py.
      MONGO_PROJECT=Your Project Name
      ```
 
-     For hosted/cloud development, also set `MONGO_ALLOW_LOCAL_FALLBACK=0` so
-     Atlas failures do not silently fall back to localhost. Once connectivity is
-     stable, prefer `VON_MONGO_STRICT_STARTUP=1` and
-     `VON_MONGO_STARTUP_PROBE=1`.
+     Keep the default `MONGO_ALLOW_LOCAL_FALLBACK=0` so Atlas failures do not
+     silently fall back to an old localhost database. After installing a
+     separately supervised SSH tunnel, expose its loopback listeners through
+     `MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS`; Von derives its Mongo credentials in
+     memory, discovers the writable forwarded replica member, and probes the
+     direct primary again on later database use after a bounded recovery
+     window. Once connectivity is stable, prefer `VON_MONGO_STRICT_STARTUP=1`
+     and `VON_MONGO_STARTUP_PROBE=1`.
+
+     On macOS, the persistent per-user launchd tunnel and its install, status,
+     and uninstall commands are documented in
+     [`docs/engineering/environment_minimums.md`](docs/engineering/environment_minimums.md#macos-supervised-atlas-ssh-fallback).
 
 Von decides local versus remote MongoDB from the effective `MONGO_URI` host and
 logs either `[Von Database] Connecting to LOCAL MongoDB ...` or

docs/engineering/environment_minimums.md

@@ -11,6 +11,7 @@ Ollama path, Von can start without any explicit environment variables because:
 
 - `MONGO_URI` defaults to `mongodb://localhost:27017/`
 - `VON_DB_NAME` defaults to `von_db`
+- fallback to a different local Mongo instance is disabled by default
 - the active LLM provider falls back to Ollama when no explicit remote provider
   is configured
 
@@ -81,12 +82,104 @@ Why these matter:
 
 - `MONGO_ALLOW_LOCAL_FALLBACK=0` prevents Atlas failures from silently falling
   back to localhost
+- `MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS=<loopback host:port list>` enables a
+  secret-free, supervised SSH transport fallback. Von derives credentials from
+  the primary URI in memory, validates the expected replica set when configured,
+  and selects only the writable forwarded member. The SSH process itself must
+  be supervised separately; on macOS use
+  `scripts/install_macos_mongo_ssh_tunnel.py`.
+- `MONGO_SSH_TUNNEL_EXPECTED_REPLICA_SET=<set name>` pins the forwarded members
+  when the set name cannot be derived from an existing URI
 - `MONGO_DNS_FALLBACK_URI=<direct-host Atlas URI>` can still be used as a
   non-local recovery path when SRV/DNS resolution is flaky
+- `VON_MONGO_FALLBACK_STICKY_SECONDS=300` controls how long a working fallback
+  is reused before Von probes the direct primary route again
 - `VON_SKIP_BROWSER_LAUNCH=1` avoids remote hosts trying to open a local browser
 - `FLASK_SECRET_KEY` is required for sane hosted session handling and is
   mandatory when strict hosted OAuth startup is enabled
 
+### macOS supervised Atlas SSH fallback
+
+Use a per-user launchd agent to own a persistent tunnel independently of the
+Von process. It starts within that user's GUI login domain, restarts after SSH
+or network failures, and is not a pre-login system daemon. Supply one forward
+for every Atlas replica-set member so Von can select the current writable
+primary after an election. The installer writes no Mongo credentials to the
+LaunchAgent; it accepts SSH transport details only.
+
+```sh
+.venv/bin/python scripts/install_macos_mongo_ssh_tunnel.py install \
+  --ssh-destination operator@relay.example \
+  --identity-file /absolute/path/to/private-key \
+  --known-hosts-file /absolute/path/to/known_hosts \
+  --forward 27018:atlas-member-00.example:27017 \
+  --forward 27019:atlas-member-01.example:27017 \
+  --forward 27020:atlas-member-02.example:27017
+```
+
+The SSH identity must be owner-only, the relay must already have a pinned host
+key in the selected known-hosts file, and the local ports must not be owned by
+another process. The generated service uses strict host-key checking and
+isolates itself from ambient SSH configuration. Then configure Von and restart
+it, because these values are loaded when the backend module starts:
+
+```dotenv
+MONGO_ALLOW_LOCAL_FALLBACK=0
+MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS=127.0.0.1:27018,127.0.0.1:27019,127.0.0.1:27020
+MONGO_SSH_TUNNEL_EXPECTED_REPLICA_SET=atlas-example-shard-0
+```
+
+Explicitly pinning the actual replica-set name is strongly recommended,
+especially for an SRV primary URI that does not carry a `replicaSet` option.
+Check launchd and listener ownership without repeating the forward list:
+
+```sh
+.venv/bin/python scripts/install_macos_mongo_ssh_tunnel.py status
+./run.sh restart -NoBrowser -HealthTimeoutSec 180
+./run.sh status
+```
+
+Installer status proves only that the launchd-managed SSH PID owns each
+configured loopback listener; it does not probe Mongo. While the primary route
+is healthy, `./run.sh status` should continue to report `Mongo: Atlas`. A
+functional fallback test must report `Mongo: Atlas via SSH tunnel (...)` and
+complete a Mongo ping or canonical read through that route.
+
+The status command exits successfully when the query itself completed, even
+when the service is absent or degraded. For a healthy listener layer, require
+`configuration_status="configured"`, `plist_valid=true`,
+`label_matches=true`, `installed=true`, `loaded=true`,
+`listeners_checked=true`, and `listener_coverage_complete=true`.
+`mongo_route_probed=false` is expected until a separate Mongo operation is
+performed.
+
+Before removal, delete or disable
+`MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS`, restart Von, and verify that its live
+Mongo route no longer depends on the tunnel. Removal is then explicit:
+
+```sh
+.venv/bin/python scripts/install_macos_mongo_ssh_tunnel.py uninstall \
+  --confirm-uninstall
+.venv/bin/python scripts/install_macos_mongo_ssh_tunnel.py status
+```
+
+Uninstall removes the LaunchAgent plist, but retains the SSH key,
+known-hosts file, and `~/Library/Logs/Von` tunnel logs.
+
+Mongo still validates the remote certificate chain. Because a forwarded Atlas
+member is contacted at a loopback hostname, only hostname matching is relaxed
+for URIs derived from syntactically validated loopback endpoints. The
+authenticated SSH tunnel, strict relay host-key check, writable-primary check,
+and expected replica-set check when configured bound that exception. Do not
+expose these ports on a non-loopback interface.
+
+Fallback selection is failure-sensitive: a pure DNS/SRV failure tries the
+direct-host Atlas URI first; TLS, TCP, periodic health-check transport failures,
+and operation failures handled by the shared retry path try the SSH route
+first. An explicitly enabled local-development Mongo fallback is always last.
+A working remote fallback is reused for a bounded window and Von then probes
+the direct primary again.
+
 For stricter hosted Mongo startup, also set:
 
 ```powershell

run.sh

@@ -1487,6 +1487,7 @@ start_server() {
         echo "To fix: export VON_DB_NAME='von_db'" >&2
         return 1
     fi
+    log_mongo_ssh_tunnel_status
 
     local existing
     existing="$(get_pid || true)"
@@ -1936,16 +1937,24 @@ try:
 except Exception:
     sys.exit(1)
 using_fallback=data.get("using_fallback")
+fallback_kind=data.get("fallback_kind")
 atlas=data.get("atlas_detected")
 host=data.get("effective_host")
 if using_fallback:
-    label="Mongo: local fallback"
+    if fallback_kind == "ssh_tunnel":
+        label="Mongo: Atlas via SSH tunnel"
+    elif fallback_kind == "dns":
+        label="Mongo: Atlas direct-host fallback"
+    elif fallback_kind == "local":
+        label="Mongo: local fallback"
+    else:
+        label="Mongo: fallback"
 elif atlas:
     label="Mongo: Atlas"
 else:
     label="Mongo: "+(host or "unknown")
-if host and label.startswith("Mongo: local fallback"):
-    label=f"Mongo: local fallback ({host})"
+if host and using_fallback:
+    label=f"{label} ({host})"
 elif host and label.startswith("Mongo: Atlas"):
     label=f"Mongo: Atlas ({host})"
 elif host and label.startswith("Mongo: "):
@@ -1960,6 +1969,41 @@ print(label)' 2>/dev/null)"
     log "$line"
 }
 
+log_mongo_ssh_tunnel_status() {
+    local endpoints="${MONGO_SSH_TUNNEL_FALLBACK_ENDPOINTS:-}"
+    if [ -z "$endpoints" ]; then
+        return 0
+    fi
+    if ! command -v lsof >/dev/null 2>&1; then
+        log "Mongo SSH tunnel: configured; listener status unavailable"
+        return 0
+    fi
+    local ready=0
+    local total=0
+    local value=""
+    local port=""
+    local values=()
+    IFS=',' read -r -a values <<<"$endpoints"
+    for value in "${values[@]}"; do
+        port="${value##*:}"
+        port="${port//[[:space:]]/}"
+        if ! [[ "$port" =~ ^[0-9]+$ ]] || [ "$port" -lt 1 ] || [ "$port" -gt 65535 ]; then
+            continue
+        fi
+        total=$((total + 1))
+        if lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1; then
+            ready=$((ready + 1))
+        fi
+    done
+    if [ "$total" -eq 0 ]; then
+        log "Mongo SSH tunnel: configured; listener status unavailable"
+    elif [ "$ready" -eq "$total" ]; then
+        log "Mongo SSH tunnel: listeners present (${ready}/${total}); Mongo route not probed"
+    else
+        log "Mongo SSH tunnel: listener coverage incomplete (${ready}/${total}); inspect live Mongo route"
+    fi
+}
+
 parse_scan_counts() {
     local output="$1"
     local py
@@ -3215,6 +3259,7 @@ run_relation_alias_cleanup() {
 status_server() {
     # Match run.ps1: if PID file stale, try to sync from current listener.
     sync_pidfile_to_listener
+    log_mongo_ssh_tunnel_status
     local run_maintenance=0
     if [ "$STATUS_RUN_MAINTENANCE" -eq 1 ] || is_truthy "${VON_STATUS_RUN_MAINTENANCE:-}"; then
         run_maintenance=1