diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f5abfee..f3fcf9db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: - uses: actions/checkout@v4 - name: Run check-consistency.sh run: bash scripts/check-consistency.sh + - name: Test deployment image retention + run: python3 -m unittest discover -s scripts/tests -p 'test_cleanup_images.py' -v - name: Validate self-host Compose run: | bash scripts/selfhost.sh init diff --git a/infra/README.md b/infra/README.md index 212ba407..38961a1a 100644 --- a/infra/README.md +++ b/infra/README.md @@ -72,5 +72,22 @@ TimescaleDB extension 应存在；hypertable 至少包括 bars/ticks。架构和 ## 清理 +生产部署脚本 `scripts/deploy.sh` 需要宿主机 `python3`、`flock` 及支持 `up --wait` +的 Docker Compose。部署之间使用文件锁互斥；服务在 300 秒内通过健康检查后，才清理 +Compose 中明确声明的 Inalpha 应用镜像仓库。每个仓库保留最近两个构建版本，同时保留 +所有容器（包括已停止容器）引用、本次部署前引用、以及构建未满 24 小时的镜像。 +失败部署不清理；清理失败会输出告警。数据库、volume、容器及其他仓库镜像不参与清理。 +无仓库信息的悬空镜像会保留，需要人工确认来源后处理。 + +手动检查时默认只预览，确认后添加 `--apply`： + +```bash +python3 scripts/cleanup-images.py --repository ghcr.io/mirror29/inalpha-dashboard +``` + +生产 Compose 的全部服务均配置 `json-file` 日志轮转，每个文件最多 `10m`，保留 +`3` 个文件，每容器约 30 MB。新配置在容器下次创建或重建时生效，普通 restart +不会改变已有容器的日志配置。更新配置后应随计划内部署启用，不必重启 Docker daemon。 + `docker compose down` 只停止容器并保留 volume。`docker compose down -v` 会永久删除开发 数据库 volume，只能在明确要重建本地数据时使用。 diff --git a/infra/docker-compose.prod.yml b/infra/docker-compose.prod.yml index 5d8abe61..bf1c2b41 100644 --- a/infra/docker-compose.prod.yml +++ b/infra/docker-compose.prod.yml @@ -15,11 +15,18 @@ name: inalpha-prod # - 应急本地构建:`up -d --build`(小内存机注意 OOM)。 # image 路径由 IMAGE_PREFIX/IMAGE_TAG 注入(见 .env.prod.example);CI 见 build-images.yml。 +x-logging: &logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + x-python-build: &python-build context: ../services dockerfile: ../infra/docker/Dockerfile.python x-svc-common: &svc-common + logging: *logging restart: unless-stopped depends_on: postgres: @@ -30,6 +37,7 @@ x-svc-common: &svc-common services: postgres: + logging: *logging # 数据库镜像必须钉版本:latest-pg17 浮动 tag 在 pull/重建时可能拉到新版 # TimescaleDB,轻则起不来重则静默破坏已有 hypertable。升级走显式改版本号。 image: timescale/timescaledb:2.27.2-pg17 @@ -52,6 +60,7 @@ services: # 不映射端口到公网；仅容器内网可达 redis: + logging: *logging image: redis:8-alpine container_name: inalpha-redis restart: unless-stopped @@ -70,6 +79,7 @@ services: # 一次性：跑 DB 迁移到最新，app 服务依赖它 completed_successfully migrate: + logging: *logging image: ${IMAGE_PREFIX:-ghcr.io/__CHANGE_ME__}/inalpha-paper:${IMAGE_TAG:-latest} build: <<: *python-build @@ -190,6 +200,7 @@ services: start_period: 30s mastra: + logging: *logging image: ${IMAGE_PREFIX:-ghcr.io/__CHANGE_ME__}/inalpha-mastra:${IMAGE_TAG:-latest} build: context: ../packages/orchestration @@ -220,6 +231,7 @@ services: # 操作者控制台(Next16 standalone)。BFF 经容器名直连后端,零 CORS、JWT 同源(共用 .env.prod)。 # 公网仅经 Cloudflare Tunnel 暴露 dashboard.inalpha.dev → dashboard:3001;后端服务一律不暴露。 dashboard: + logging: *logging image: ${IMAGE_PREFIX:-ghcr.io/__CHANGE_ME__}/inalpha-dashboard:${IMAGE_TAG:-latest} build: context: ../apps/dashboard @@ -269,6 +281,7 @@ services: # 公网入口：Cloudflare Tunnel（无需在 VPS 开入站端口，TLS/防护走 CF） cloudflared: + logging: *logging profiles: ["tunnel"] image: cloudflare/cloudflared:2026.6.0 container_name: inalpha-cloudflared diff --git a/scripts/cleanup-images.py b/scripts/cleanup-images.py new file mode 100755 index 00000000..4b942952 --- /dev/null +++ b/scripts/cleanup-images.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import subprocess +from datetime import datetime, timedelta, timezone +from pathlib import Path + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True).strip() + + +def container_images(): + containers = docker("ps", "-aq").split() + if not containers: + return set() + return set(docker("inspect", "--format", "{{.Image}}", *containers).split()) + + +def repositories(image): + tags = image.get("RepoTags") or [] + digests = image.get("RepoDigests") or [] + return {tag.rsplit(":", 1)[0] for tag in tags} | { + digest.split("@", 1)[0] for digest in digests + } + + +def created_at(value): + normalized = re.sub( + r"\.(\d+)", lambda match: "." + match[1][:6].ljust(6, "0"), value + ) + return datetime.fromisoformat(normalized.replace("Z", "+00:00")) + + +def candidates(images, allowed, protected, keep, cutoff): + grouped = {repo: [] for repo in allowed} + for image in images: + for repo in repositories(image) & allowed: + grouped[repo].append(image) + retained = set(protected) + for group in grouped.values(): + ordered = sorted(group, key=lambda image: image["Created"], reverse=True) + retained.update(image["Id"] for image in ordered[:keep]) + return [ + image + for image in images + if repositories(image) + and repositories(image) <= allowed + and image["Id"] not in retained + and created_at(image["Created"]) < cutoff + ] + + +def remove_image(image, protected): + image_id = image["Id"] + refs = image.get("RepoTags") or [image_id] + for ref in refs: + if image_id in protected | container_images(): + print("SKIP in-use:", image_id) + return + actual = docker("image", "inspect", "--format", "{{.Id}}", ref) + if actual != image_id: + raise RuntimeError("Image reference changed; stopping cleanup: " + ref) + subprocess.run(["docker", "image", "rm", "--no-prune", ref], check=True) + + +def main(): + parser = argparse.ArgumentParser( + description="Preview or remove old Inalpha images." + ) + parser.add_argument("--repository", action="append", required=True) + parser.add_argument("--protect-file", type=Path) + parser.add_argument("--keep", type=int, default=2) + parser.add_argument("--min-age-hours", type=int, default=24) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + if args.keep < 2 or args.min_age_hours < 24: + parser.error("Keep at least two versions and a minimum age of 24 hours.") + allowed = set(args.repository) + services = {"data", "paper", "research", "factor", "evolver", "mastra", "dashboard"} + if any( + repo.rsplit("/", 1)[-1] not in {"inalpha-" + s for s in services} + for repo in allowed + ): + parser.error("Only explicit Inalpha application repositories are supported.") + protected = container_images() + if args.protect_file: + protected.update(args.protect_file.read_text().split()) + ids = sorted(set(docker("image", "ls", "-aq", "--no-trunc").split())) + images = json.loads(docker("image", "inspect", *ids)) if ids else [] + selected = candidates( + images, + allowed, + protected, + args.keep, + datetime.now(timezone.utc) - timedelta(hours=args.min_age_hours), + ) + print("APPLY" if args.apply else "DRY RUN", "candidate images:", len(selected)) + for image in selected: + print(image["Id"], ", ".join(image.get("RepoTags") or [])) + if args.apply: + remove_image(image, protected) + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 82a75f50..7ea0db98 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -48,22 +48,55 @@ if ! grep -qE '^CLOUDFLARE_TUNNEL_TOKEN=.+$' "$HOST_ENV_FILE"; then exit 1 fi +command -v python3 >/dev/null +command -v flock >/dev/null +mkdir -p "$ROOT/.cache" +exec 9>"$ROOT/.cache/deploy.lock" +flock -n 9 || { echo "已有部署正在运行" >&2; exit 1; } + if [ "$do_git_pull" -eq 1 ]; then echo "==> git pull --ff-only" git pull --ff-only fi +protected_images="$(mktemp)" +trap 'rm -f "$protected_images"' EXIT +container_ids="$(docker ps -aq)" +if [ -n "$container_ids" ]; then + docker inspect --format '{{.Image}}' $container_ids > "$protected_images" +fi +image_refs="$("${DC[@]}" config --images)" +cleanup_args=() +while IFS= read -r ref; do + repository="${ref%@*}" + if [[ "${repository##*/}" == *:* ]]; then + repository="${repository%:*}" + fi + case "${repository##*/}" in + inalpha-data|inalpha-paper|inalpha-research|inalpha-factor|inalpha-evolver|inalpha-mastra|inalpha-dashboard) + cleanup_args+=(--repository "$repository") ;; + esac +done <<< "$image_refs" + if [ "$mode" = "image" ]; then echo "==> 拉取镜像(GHCR)" "${DC[@]}" pull echo "==> 起栈(migrate 先跑 alembic upgrade head,再起各服务)" - "${DC[@]}" up -d + "${DC[@]}" up -d --wait --wait-timeout 300 else echo "==> 就地构建并起栈(--build;小内存机注意 OOM)" - "${DC[@]}" up -d --build + "${DC[@]}" up -d --build --wait --wait-timeout 300 fi echo "==> 当前状态" "${DC[@]}" ps +if [ "${#cleanup_args[@]}" -gt 0 ]; then + echo "==> 清理旧应用镜像（保留最近两个版本、部署前版本及容器引用）" + if ! python3 scripts/cleanup-images.py "${cleanup_args[@]}" \ + --protect-file "$protected_images" --apply; then + echo "警告：服务已部署，但镜像清理失败；请检查后重试。" >&2 + fi +fi + echo "==> 完成。排障:'${DC[*]} logs -f ';健康看上方 ps 的 STATUS。" diff --git a/scripts/tests/test_cleanup_images.py b/scripts/tests/test_cleanup_images.py new file mode 100644 index 00000000..bc8982ce --- /dev/null +++ b/scripts/tests/test_cleanup_images.py @@ -0,0 +1,188 @@ +import os +import runpy +import shutil +import subprocess +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +module = runpy.run_path(str(Path(__file__).parents[1] / "cleanup-images.py")) +candidates = module["candidates"] +remove_image = module["remove_image"] +created_at = module["created_at"] +REPO = "ghcr.io/mirror29/inalpha-data" +CUTOFF = datetime(2026, 10, 13, tzinfo=timezone.utc) + + +def image(number, repo=REPO, tags=None): + return { + "Id": "sha256:" + str(number), + "Created": f"2026-10-{number:02d}T00:00:00Z", + "RepoTags": tags if tags is not None else [f"{repo}:v{number}"], + "RepoDigests": [], + } + + +class CleanupImagesTests(unittest.TestCase): + def test_normalizes_docker_fractional_timestamps_for_older_python(self): + self.assertEqual( + created_at("2026-10-13T00:00:00.123456789Z"), + datetime(2026, 10, 13, microsecond=123456, tzinfo=timezone.utc), + ) + self.assertEqual( + created_at("2026-10-13T00:00:00.1Z"), + datetime(2026, 10, 13, microsecond=100000, tzinfo=timezone.utc), + ) + + def test_preserves_two_latest_and_container_and_rollback_images(self): + images = [image(n) for n in range(1, 8)] + selected = candidates(images, {REPO}, {"sha256:1", "sha256:2"}, 2, CUTOFF) + self.assertEqual( + [i["Id"] for i in selected], ["sha256:3", "sha256:4", "sha256:5"] + ) + + def test_keeps_at_least_two_for_each_repository(self): + second = "ghcr.io/mirror29/inalpha-paper" + images = [image(n) for n in (1, 2, 3)] + [image(n, second) for n in (4, 5, 6)] + selected = candidates(images, {REPO, second}, set(), 2, CUTOFF) + self.assertEqual([i["Id"] for i in selected], ["sha256:1", "sha256:4"]) + + def test_multiple_tags_count_as_one_version(self): + images = [image(1), image(2), image(3, tags=[REPO + ":a", REPO + ":b"])] + self.assertEqual(candidates(images, {REPO}, set(), 2, CUTOFF), [images[0]]) + + def test_preserves_other_repositories_and_shared_tags(self): + foreign = image(1, "redis") + shared = image(2, tags=[REPO + ":old", "other/data:backup"]) + images = [foreign, shared, image(3), image(4)] + self.assertEqual(candidates(images, {REPO}, set(), 2, CUTOFF), []) + + def test_preserves_recent_builds_and_unknown_dangling_images(self): + images = [image(1, tags=[]), image(13), image(14), image(15)] + self.assertEqual(candidates(images, {REPO}, set(), 2, CUTOFF), []) + + def test_recognizes_untagged_images_only_with_explicit_digest_repository(self): + old = image(1, tags=[]) + old["RepoDigests"] = [REPO + "@sha256:old"] + self.assertEqual( + candidates([old, image(2), image(3)], {REPO}, set(), 2, CUTOFF), [old] + ) + + def test_rechecks_containers_before_deleting_each_tag(self): + old = image(1, tags=[REPO + ":a", REPO + ":b"]) + with ( + patch.dict( + remove_image.__globals__, + { + "container_images": unittest.mock.Mock( + side_effect=[set(), {old["Id"]}] + ), + "docker": unittest.mock.Mock(return_value=old["Id"]), + }, + ), + patch("subprocess.run") as run, + ): + remove_image(old, set()) + run.assert_called_once_with( + ["docker", "image", "rm", "--no-prune", REPO + ":a"], check=True + ) + + def test_refuses_changed_image_reference(self): + with ( + patch.dict( + remove_image.__globals__, + { + "container_images": lambda: set(), + "docker": lambda *args: "sha256:replacement", + }, + ), + patch("subprocess.run") as run, + self.assertRaises(RuntimeError), + ): + remove_image(image(1), set()) + run.assert_not_called() + + def test_delete_failure_is_not_ignored(self): + with ( + patch.dict( + remove_image.__globals__, + { + "container_images": lambda: set(), + "docker": lambda *args: "sha256:1", + }, + ), + patch("subprocess.run", side_effect=RuntimeError("in use")), + self.assertRaises(RuntimeError), + ): + remove_image(image(1), set()) + + +class DeploymentTests(unittest.TestCase): + def check_deployment(self, up_result): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in ("scripts", "infra", "bin"): + (root / name).mkdir() + shutil.copy( + Path(__file__).parents[1] / "deploy.sh", root / "scripts/deploy.sh" + ) + (root / "infra/.env.prod").write_text("CLOUDFLARE_TUNNEL_TOKEN=test\n") + fake_docker = """#!/usr/bin/env bash +echo "$*" >> "$TEST_CALLS" +case "$*" in + "ps -aq") echo container1 ;; + "inspect "*) echo sha256:previous ;; + *"config --images") echo ghcr.io/mirror29/inalpha-data:v3 ;; + *" up "*) exit "$TEST_UP_RESULT" ;; +esac +""" + fake_python = """#!/usr/bin/env bash +echo "cleanup $*" >> "$TEST_CALLS" +while [ "$#" -gt 0 ]; do + if [ "$1" = "--protect-file" ]; then cat "$2" >> "$TEST_CALLS"; break; fi + shift +done +""" + for name, content in { + "docker": fake_docker, + "python3": fake_python, + "flock": "#!/bin/sh\nexit 0\n", + }.items(): + executable = root / "bin" / name + executable.write_text(content) + executable.chmod(0o755) + calls = root / "calls" + env = dict( + os.environ, + PATH=str(root / "bin") + os.pathsep + os.environ["PATH"], + TEST_CALLS=str(calls), + TEST_UP_RESULT=str(up_result), + ) + result = subprocess.run( + ["bash", str(root / "scripts/deploy.sh"), "--no-pull"], + env=env, + check=False, + capture_output=True, + text=True, + ) + return result, calls.read_text() + + def test_failed_deployment_never_cleans_images(self): + result, calls = self.check_deployment(1) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("cleanup ", calls) + + def test_successful_deployment_waits_then_cleans_with_rollback_protection(self): + result, calls = self.check_deployment(0) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertLess( + calls.index("--wait --wait-timeout 300"), calls.index("cleanup ") + ) + self.assertIn("--repository ghcr.io/mirror29/inalpha-data", calls) + self.assertTrue(calls.endswith("sha256:previous\n")) + + +if __name__ == "__main__": + unittest.main()