Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
751 changes: 640 additions & 111 deletions run.sh

Large diffs are not rendered by default.

47 changes: 37 additions & 10 deletions scripts/backup_von_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
- VON_BACKUP_COMPRESSION_ENABLED (truthy/falsey)
- VON_BACKUP_ENCRYPTION_ENABLED (truthy/falsey)
- VON_BACKUP_ENCRYPTION_KEY (Fernet key; required if encryption enabled)
- VON_BACKUP_MONGODUMP_TIMEOUT_SECONDS (int; default 21600 / 6 hours)
"""

from __future__ import annotations
Expand All @@ -22,6 +23,7 @@
import os
import subprocess
import sys
import tempfile
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
Expand All @@ -33,6 +35,7 @@

BACKUP_SUCCESS_RECEIPT_SCHEMA_VERSION = "backup_success_receipt.v1"
BACKUP_RECEIPT_SIDECAR_SUFFIX = ".backup_receipt.json"
DEFAULT_MONGODUMP_TIMEOUT_SECONDS = 6 * 60 * 60


def _env_truthy(value: str | None) -> bool:
Expand Down Expand Up @@ -366,16 +369,40 @@ def _upload_backup_to_blob(


def _run_mongodump(*, mongo_uri: str, db_name: str, out_path: Path) -> None:
cmd = [
"mongodump",
"--uri",
mongo_uri,
"--db",
db_name,
"--out",
str(out_path),
]
subprocess.run(cmd, check=True)
timeout_seconds = _env_int(
os.environ.get("VON_BACKUP_MONGODUMP_TIMEOUT_SECONDS"),
default=DEFAULT_MONGODUMP_TIMEOUT_SECONDS,
)
if timeout_seconds < 1:
timeout_seconds = DEFAULT_MONGODUMP_TIMEOUT_SECONDS
with tempfile.TemporaryDirectory(prefix="von-mongodump-") as config_dir:
config_root = Path(config_dir)
os.chmod(config_root, 0o700)
config_path = config_root / "config.yml"
config_path.write_text(
f"uri: {json.dumps(mongo_uri)}\n",
encoding="utf-8",
)
os.chmod(config_path, 0o600)
cmd = [
"mongodump",
"--config",
str(config_path),
"--db",
db_name,
"--out",
str(out_path),
]
try:
subprocess.run(cmd, check=True, timeout=timeout_seconds)
except subprocess.TimeoutExpired:
raise RuntimeError(
f"mongodump exceeded the configured {timeout_seconds}s timeout"
) from None
except subprocess.CalledProcessError as exc:
raise RuntimeError(
f"mongodump failed with exit code {exc.returncode}"
) from None


def main(argv: list[str] | None = None) -> int:
Expand Down
66 changes: 51 additions & 15 deletions scripts/restore_von_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,18 @@
from backup_von_db import BACKUP_RECEIPT_SIDECAR_SUFFIX


DEFAULT_MONGORESTORE_TIMEOUT_SECONDS = 6 * 60 * 60


def _env_int(value: str | None, *, default: int) -> int:
if value is None:
return default
try:
return int(value.strip())
except Exception:
return default


def _redact_mongo_uri(uri: str) -> str:
if not uri or "@" not in uri:
return uri
Expand Down Expand Up @@ -258,21 +270,45 @@ def _run_mongorestore(
target_db_name: str,
drop_target: bool,
) -> None:
cmd = [
"mongorestore",
"--uri",
mongo_uri,
"--dir",
str(dump_root),
"--nsFrom",
f"{source_db_name}.*",
"--nsTo",
f"{target_db_name}.*",
"--stopOnError",
]
if drop_target:
cmd.append("--drop")
subprocess.run(cmd, check=True)
timeout_seconds = _env_int(
os.environ.get("VON_BACKUP_MONGORESTORE_TIMEOUT_SECONDS"),
default=DEFAULT_MONGORESTORE_TIMEOUT_SECONDS,
)
if timeout_seconds < 1:
timeout_seconds = DEFAULT_MONGORESTORE_TIMEOUT_SECONDS
with tempfile.TemporaryDirectory(prefix="von-mongorestore-") as config_dir:
config_root = Path(config_dir)
os.chmod(config_root, 0o700)
config_path = config_root / "config.yml"
config_path.write_text(
f"uri: {json.dumps(mongo_uri)}\n",
encoding="utf-8",
)
os.chmod(config_path, 0o600)
cmd = [
"mongorestore",
"--config",
str(config_path),
"--dir",
str(dump_root),
"--nsFrom",
f"{source_db_name}.*",
"--nsTo",
f"{target_db_name}.*",
"--stopOnError",
]
if drop_target:
cmd.append("--drop")
try:
subprocess.run(cmd, check=True, timeout=timeout_seconds)
except subprocess.TimeoutExpired:
raise RuntimeError(
f"mongorestore exceeded the configured {timeout_seconds}s timeout"
) from None
except subprocess.CalledProcessError as exc:
raise RuntimeError(
f"mongorestore failed with exit code {exc.returncode}"
) from None


def main(argv: list[str] | None = None) -> int:
Expand Down
Loading
Loading