diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fa0b754..218a1177 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.1 with: submodules: true - run: ./scripts/validate-stack.sh @@ -28,7 +28,7 @@ jobs: if: github.event_name != 'pull_request' runs-on: [self-hosted, Linux, X64, arko, typetype] steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.1 with: submodules: true - run: ./scripts/validate-stack.sh diff --git a/.github/workflows/component-image.yml b/.github/workflows/component-image.yml index df0506de..a9b052e4 100644 --- a/.github/workflows/component-image.yml +++ b/.github/workflows/component-image.yml @@ -127,11 +127,17 @@ jobs: - name: Update beta env: + COMPONENT: ${{ needs.validate.outputs.component }} DEPLOY_HOST: ${{ secrets.TYPE_TYPE_DEPLOY_HOST }} DEPLOY_PORT: ${{ secrets.TYPE_TYPE_DEPLOY_PORT }} DEPLOY_USER: ${{ secrets.TYPE_TYPE_DEPLOY_USER }} + DIGEST: ${{ needs.validate.outputs.digest }} + IMAGE: ${{ needs.validate.outputs.image }} run: | - ssh -i "$RUNNER_TEMP/typetype_deploy_key" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" "$DEPLOY_USER@$DEPLOY_HOST" /usr/local/sbin/typetype-beta-update + printf -v update_command \ + 'TYPETYPE_DEPLOY_COMPONENT=%q TYPETYPE_DEPLOY_IMAGE=%q TYPETYPE_DEPLOY_DIGEST=%q /usr/local/sbin/typetype-beta-update' \ + "$COMPONENT" "$IMAGE" "$DIGEST" + ssh -i "$RUNNER_TEMP/typetype_deploy_key" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" "$DEPLOY_USER@$DEPLOY_HOST" "$update_command" curl --fail --retry 12 --retry-delay 2 --retry-all-errors --silent --show-error https://beta.typetype.video/api/health curl --fail --retry 12 --retry-delay 2 --retry-all-errors --silent --show-error https://beta.typetype.video/api/downloader/health diff --git a/.gitignore b/.gitignore index 19a93c6c..d5d214e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ tmp/ .DS_Store *.tsbuildinfo .env +.runtime/ diff --git a/Docs-TypeType b/Docs-TypeType index d50f45e8..7dc60778 160000 --- a/Docs-TypeType +++ b/Docs-TypeType @@ -1 +1 @@ -Subproject commit d50f45e841b5cd657305c97f20a96d9f69cd24c0 +Subproject commit 7dc607786d4ac85527de8fc852c803a7c583fbb6 diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f36763ce..8d0b6879 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,17 +1,98 @@ -# TypeType 1.2.4 +# TypeType 1.3.0 -## Summary +TypeType 1.3.0 brings new backup and content controls, makes SABR the only +YouTube playback path on the web, and improves downloads and self-hosted +updates. -- fix intermittent SABR startup failures when resuming videos from a saved position -- restore reliable audio and video initialization on cold playback sessions -- prevent stalled preparation from retrying indefinitely -- improve recovery when a SABR session cannot produce media -- keep repeated long-distance seeks responsive while SABR recovery is pending +## Playback -Normal seeks and quality changes continue to preserve the current playback position. +- Use SABR/MSE for all YouTube web playback, including watch pages, embeds and + autoplay. The old Classic toggle and transition prompt have been removed. + NicoNico and BiliBili playback remain unchanged. + [#168](https://github.com/TypeType-Video/TypeType/issues/168) +- Recover SABR sessions when YouTube authorization expires and stop stalled + preparation attempts instead of retrying indefinitely. +- Apply automatic SponsorBlock skips in audio-only mode. + [#178](https://github.com/TypeType-Video/TypeType/issues/178) +- Add a default playback speed from `0.25x` to `4x`, applied to regular videos + and Shorts. -No configuration or database migration is required. +## Backups And Content Controls -Thx to everyone who reported playback and buffering issues. +- Export a complete TypeType backup or select individual categories, including + subscriptions, history, playlists, watch later, favorites, playback + progress, search history, saved playlists, settings and content filters. +- Restore versioned TypeType JSON backups with an immediate confirmation + showing how many items were restored. +- Add blocked title keywords to filter unwanted videos from recommendations. +- Support very large history exports without exceeding PostgreSQL query + limits. +- Preserve more original dates and metadata when importing YouTube Takeout + favorites, history and playlists. -If u want to support TypeType, please share it with others. If u want to support it financially, u can do so through [GitHub Sponsors](https://github.com/sponsors/Priveetee). +## Subscriptions + +- Keep subscription feeds updating when one subscribed channel is deleted, + private or temporarily unavailable. + [#180](https://github.com/TypeType-Video/TypeType/issues/180) +- Keep live videos ordered by their actual promotion time instead of + permanently placing every active live stream first. + +## Downloads And Performance + +- Rework SABR downloads around bounded multipart streams and concurrent range + workers. +- Reserve the required disk space before starting large downloads to prevent + active jobs from exhausting storage. +- Detect stalled streams while allowing long downloads to continue as long as + data is still arriving. +- Reduce progress-update contention and reuse download buffers. +- Serve precompressed frontend assets through Nginx. + +## Self-Hosting + +- Bundle the default Nginx configuration directly in the Frontend image. +- Generate the default Garage configuration inside a managed Docker volume. +- Back up existing stack files and preserve custom Compose and Garage + configuration during updates. +- Validate the resolved Compose configuration, wait for service health and + provide a rollback directory when an update fails. +- Show the Frontend, Server, Token and Downloader versions, revisions and build + times directly in Settings. +- Expand the self-hosting documentation with resource requirements and new + French and Spanish prerequisites guides. + [#176](https://github.com/TypeType-Video/TypeType/issues/176) + +Self-hosters should follow the official update procedure for this release +because the Compose stack and managed configuration have changed. + +## Thx + +A big thx to @BuggyPasta for the detailed testing and for the ideas behind +backups, blocked keywords and default playback speed. + +Thx to @Toastienergy for reporting the missing SponsorBlock behavior in +audio-only mode, and to @tam1m for the precise subscription-feed report. + +Thx to @jerkiz and @coral-hungus for testing playback and SponsorBlock on real +iPhones. + +Thx to @arcoast for the Nginx, Garage and update-flow suggestions, to +@filippobaroni for helping expose problems caused by stale self-hosted stack +files, and to @EdgarsJudovics for helping keep the script-free installation +path documented. + +Thx as well to everyone testing the beta, reporting playback and buffering +problems, or helping other self-hosters. + +## Updating + +Follow the +[update guide](https://typetype-video.github.io/Docs-TypeType/self-hosting/maintenance). + +If necessary, follow the +[rollback guide](https://typetype-video.github.io/Docs-TypeType/self-hosting/rollback). + +If u want to support TypeType, please share it with others. If u want to +support it financially, u can do so through +[GitHub Sponsors](https://github.com/sponsors/Priveetee). diff --git a/TypeType-Downloader b/TypeType-Downloader index 7763ee16..29da6984 160000 --- a/TypeType-Downloader +++ b/TypeType-Downloader @@ -1 +1 @@ -Subproject commit 7763ee165333a0132a59ec25b4548a4b9084bb16 +Subproject commit 29da6984c60274eec510aa422dfd748455e4e012 diff --git a/TypeType-Frontend b/TypeType-Frontend index df97e1b2..60c14c3c 160000 --- a/TypeType-Frontend +++ b/TypeType-Frontend @@ -1 +1 @@ -Subproject commit df97e1b2c8786aa571e29d4fe596f6844b3b67c6 +Subproject commit 60c14c3c6bbb6914c44cce2649b70544087c523c diff --git a/TypeType-Player b/TypeType-Player index ee0b3c05..f4844c6e 160000 --- a/TypeType-Player +++ b/TypeType-Player @@ -1 +1 @@ -Subproject commit ee0b3c053340dacad1647236ed153af31db0f0cf +Subproject commit f4844c6e65d021c0ad1ab61169f4f51cfcd77694 diff --git a/TypeType-Server b/TypeType-Server index dfa23c12..22dcd6dd 160000 --- a/TypeType-Server +++ b/TypeType-Server @@ -1 +1 @@ -Subproject commit dfa23c12636b7157ef498eb7d8f330f409000cc0 +Subproject commit 22dcd6ddcd6b7babd3ab9f2cb184d6dcb37136d2 diff --git a/TypeType-Token b/TypeType-Token index 6f6bc1e0..0f0f217d 160000 --- a/TypeType-Token +++ b/TypeType-Token @@ -1 +1 @@ -Subproject commit 6f6bc1e0b2b909ba3eb83813f95ed2b7401adabc +Subproject commit 0f0f217da9d78ec5bb7e4458a3699b914dd8abd8 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 68df035d..f9c26c60 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,11 +3,14 @@ services: image: ${TYPETYPE_WEB_BETA_IMAGE:-ghcr.io/typetype-video/typetype-beta:latest} ports: - "${HOST_PORT_WEB_BETA:-18082}:80" - volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro depends_on: typetype-server: condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/api/version/web"] + interval: 5s + timeout: 3s + retries: 12 restart: unless-stopped typetype-server: @@ -21,7 +24,7 @@ services: DATABASE_PASSWORD: ${DATABASE_PASSWORD:-${POSTGRES_PASSWORD:-typetype}} DRAGONFLY_URL: ${DRAGONFLY_URL:-redis://dragonfly:6379} DOWNLOADER_SERVICE_URL: http://typetype-downloader:18093 - YOUTUBE_OUTBOUND_PROXY_URL: ${YOUTUBE_OUTBOUND_PROXY_URL:-} + YOUTUBE_OUTBOUND_PROXY_URL: ${YOUTUBE_OUTBOUND_PROXY_URL:?YOUTUBE_OUTBOUND_PROXY_URL is required} YOUTUBE_REMOTE_LOGIN_ENABLED: ${YOUTUBE_REMOTE_LOGIN_ENABLED:-false} YOUTUBE_REMOTE_LOGIN_SERVICE_URL: http://typetype-token:8081 YOUTUBE_REMOTE_LOGIN_CALLBACK_BASE_URL: http://typetype-server:8080 @@ -90,7 +93,7 @@ services: ipc: host environment: NODE_ENV: production - YOUTUBE_OUTBOUND_PROXY_URL: ${YOUTUBE_OUTBOUND_PROXY_URL:-} + YOUTUBE_OUTBOUND_PROXY_URL: ${YOUTUBE_OUTBOUND_PROXY_URL:?YOUTUBE_OUTBOUND_PROXY_URL is required} YOUTUBE_REMOTE_LOGIN_ENABLED: ${YOUTUBE_REMOTE_LOGIN_ENABLED:-false} YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN: ${YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN:-} YOUTUBE_REMOTE_LOGIN_CALLBACK_ORIGIN: ${YOUTUBE_REMOTE_LOGIN_CALLBACK_ORIGIN:-http://typetype-server:8080} @@ -136,26 +139,70 @@ services: restart: "no" dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.39.0 ulimits: memlock: -1 healthcheck: test: ["CMD", "redis-cli", "ping"] restart: unless-stopped + garage-config: + image: busybox:1.37 + command: + - /bin/sh + - -ec + - | + config=/etc/garage/garage.toml + if [ -s "$$config" ]; then + exit 0 + fi + umask 077 + if [ -s /migration/garage.toml ]; then + cp /migration/garage.toml "$$config" + exit 0 + fi + cat > "$$config" <<'EOF' + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "sqlite" + + replication_factor = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + EOF + volumes: + - garage_config:/etc/garage + - ./.typetype-migration:/migration:ro + restart: "no" + garage: image: dxflrs/garage:v2.2.0 ports: - "${HOST_BIND_GARAGE_S3_BETA:-127.0.0.1}:${HOST_PORT_GARAGE_S3_BETA:-3900}:3900" environment: + GARAGE_CONFIG_FILE: /etc/garage/garage.toml GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:-} volumes: - - ./garage.toml:/etc/garage.toml:ro + - garage_config:/etc/garage - garage_meta:/var/lib/garage/meta - garage_data:/var/lib/garage/data + depends_on: + garage-config: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "/garage", "status"] + interval: 5s + timeout: 3s + retries: 12 restart: unless-stopped volumes: postgres_data: + garage_config: garage_meta: garage_data: diff --git a/docker-compose.yml b/docker-compose.yml index 5437e992..07e1e70b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,14 @@ services: image: ${TYPETYPE_WEB_IMAGE:-ghcr.io/typetype-video/typetype:latest} ports: - "${HOST_PORT_WEB:-8082}:80" - volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro depends_on: typetype-server: condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/api/version/web"] + interval: 5s + timeout: 3s + retries: 12 restart: unless-stopped typetype-server: image: ${TYPETYPE_SERVER_IMAGE:-ghcr.io/typetype-video/typetype-server:latest} @@ -50,7 +53,7 @@ services: restart: unless-stopped typetype-secrets: - image: busybox:1.37 + image: busybox:1.38 command: - /bin/sh - -ec @@ -176,25 +179,69 @@ services: restart: "no" dragonfly: - image: docker.dragonflydb.io/dragonflydb/dragonfly:latest + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.39.0 ulimits: memlock: -1 healthcheck: test: ["CMD", "redis-cli", "ping"] restart: unless-stopped + garage-config: + image: busybox:1.37 + command: + - /bin/sh + - -ec + - | + config=/etc/garage/garage.toml + if [ -s "$$config" ]; then + exit 0 + fi + umask 077 + if [ -s /migration/garage.toml ]; then + cp /migration/garage.toml "$$config" + exit 0 + fi + cat > "$$config" <<'EOF' + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "sqlite" + + replication_factor = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + EOF + volumes: + - garage_config:/etc/garage + - ./.typetype-migration:/migration:ro + restart: "no" + garage: image: dxflrs/garage:v2.2.0 ports: - "${HOST_BIND_GARAGE_S3:-127.0.0.1}:${HOST_PORT_GARAGE_S3:-3900}:3900" environment: + GARAGE_CONFIG_FILE: /etc/garage/garage.toml GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:-} volumes: - - ./garage.toml:/etc/garage.toml:ro + - garage_config:/etc/garage - garage_meta:/var/lib/garage/meta - garage_data:/var/lib/garage/data + depends_on: + garage-config: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "/garage", "status"] + interval: 5s + timeout: 3s + retries: 12 restart: unless-stopped volumes: postgres_data: typetype_secrets: + garage_config: garage_meta: garage_data: diff --git a/garage.toml b/garage.toml deleted file mode 100644 index 9a803faa..00000000 --- a/garage.toml +++ /dev/null @@ -1,12 +0,0 @@ -metadata_dir = "/var/lib/garage/meta" -data_dir = "/var/lib/garage/data" -db_engine = "sqlite" - -replication_factor = 1 - -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "127.0.0.1:3901" - -[s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index b6c3553f..00000000 --- a/nginx.conf +++ /dev/null @@ -1,94 +0,0 @@ -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -resolver 127.0.0.11 valid=10s ipv6=off; - -upstream typetype_server_backend { - zone typetype_server_backend 64k; - server typetype-server:8080 resolve; -} - -upstream typetype_token_backend { - zone typetype_token_backend 64k; - server typetype-token:8081 resolve; -} - -upstream typetype_downloader_backend { - zone typetype_downloader_backend 64k; - server typetype-downloader:18093 resolve; -} - -server { - listen 80; - root /usr/share/nginx/html; - index index.html; - client_max_body_size 2g; - - gzip on; - gzip_types text/plain text/css application/javascript application/json image/svg+xml; - - location = /api/version { - alias /usr/share/nginx/html/version.json; - default_type application/json; - add_header Cache-Control "no-cache"; - } - - location = /api/version/web { - alias /usr/share/nginx/html/version-web.json; - default_type application/json; - add_header Cache-Control "no-cache"; - } - - location = /api/version/server { - proxy_pass http://typetype_server_backend/version; - add_header Cache-Control "no-cache"; - } - - location = /api/version/token { - proxy_pass http://typetype_token_backend/version; - add_header Cache-Control "no-cache"; - } - - location = /api/version/downloader { - proxy_pass http://typetype_downloader_backend/version; - add_header Cache-Control "no-cache"; - } - - location ^~ /api/ { - client_max_body_size 2g; - proxy_pass http://typetype_server_backend/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_read_timeout 300; - proxy_send_timeout 300; - proxy_buffering off; - } - - location ^~ /sabr/ { - proxy_pass http://typetype_server_backend/sabr/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_read_timeout 300; - proxy_send_timeout 300; - proxy_buffering off; - } - - location / { - try_files $uri $uri/ /index.html; - } - - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } -} diff --git a/scripts/bootstrap-garage.sh b/scripts/bootstrap-garage.sh index fe913b1f..d6b02d4a 100755 --- a/scripts/bootstrap-garage.sh +++ b/scripts/bootstrap-garage.sh @@ -4,6 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" COMPOSE_OVERRIDE_FILE="${COMPOSE_OVERRIDE_FILE:-}" +COMPOSE_CUSTOM_FILE="${COMPOSE_CUSTOM_FILE:-}" ENV_FILE="${ROOT_DIR}/.env" PLACEHOLDER_ACCESS_KEY="SET_ME_ACCESS_KEY" @@ -121,13 +122,16 @@ COMPOSE_ARGS=(-f "${COMPOSE_FILE}") if [[ -n "${COMPOSE_OVERRIDE_FILE}" ]]; then COMPOSE_ARGS+=(-f "${COMPOSE_OVERRIDE_FILE}") fi +if [[ -n "${COMPOSE_CUSTOM_FILE}" ]]; then + COMPOSE_ARGS+=(-f "${COMPOSE_CUSTOM_FILE}") +fi compose() { docker compose "${COMPOSE_ARGS[@]}" "$@" } garage_exec() { - compose exec -T garage /garage -c /etc/garage.toml "$@" + compose exec -T garage /garage "$@" } echo "[garage-bootstrap] starting garage service if needed..." diff --git a/scripts/check-youtube-egress.sh b/scripts/check-youtube-egress.sh new file mode 100755 index 00000000..86569e30 --- /dev/null +++ b/scripts/check-youtube-egress.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +project="${1:?Compose project is required}" +proxy_url="${2:?YouTube outbound proxy URL is required}" +anchor=$(docker ps -q \ + --filter "label=com.docker.compose.project=${project}" | head -n 1) +if [[ -z "$anchor" ]]; then + echo "No running ${project} container is available for the egress preflight." >&2 + exit 1 +fi +pid=$(docker inspect "$anchor" --format '{{.State.Pid}}') +if [[ ! "$pid" =~ ^[1-9][0-9]*$ ]]; then + echo "Cannot resolve the network namespace for ${project}." >&2 + exit 1 +fi + +nsenter --target "$pid" --net -- curl \ + --fail \ + --proxy "$proxy_url" \ + --noproxy "" \ + --connect-timeout 5 \ + --max-time 10 \ + --retry 2 \ + --retry-delay 1 \ + --retry-all-errors \ + --silent \ + --show-error \ + --output /dev/null \ + https://www.youtube.com/generate_204 diff --git a/scripts/deploy-beta.sh b/scripts/deploy-beta.sh index caa432b0..510534d6 100755 --- a/scripts/deploy-beta.sh +++ b/scripts/deploy-beta.sh @@ -2,15 +2,48 @@ set -euo pipefail source_root="${1:?deployment source is required}" +component="${TYPETYPE_DEPLOY_COMPONENT:-${2:-all}}" +image="${TYPETYPE_DEPLOY_IMAGE:-${3:-}}" +digest="${TYPETYPE_DEPLOY_DIGEST:-${4:-}}" project=typetype-beta-stack -server=$(docker ps -q \ - --filter "label=com.docker.compose.project=${project}" \ - --filter label=com.docker.compose.service=typetype-server) -test -n "$server" -root=$(docker inspect "$server" --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}') +anchor=$(docker ps -a -q \ + --filter "label=com.docker.compose.project=${project}" | head -n 1) +test -n "$anchor" +root=$(docker inspect "$anchor" --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}') test -d "$root" test -f "$root/.env" +target_service= +image_variable= +expected_image= +case "$component" in + all) ;; + frontend) + target_service=typetype + image_variable=TYPETYPE_WEB_BETA_IMAGE + expected_image=ghcr.io/typetype-video/typetype-beta + ;; + server) + target_service=typetype-server + image_variable=TYPETYPE_SERVER_BETA_IMAGE + expected_image=ghcr.io/typetype-video/typetype-server-beta + ;; + downloader) + target_service=typetype-downloader + image_variable=TYPETYPE_DOWNLOADER_BETA_IMAGE + expected_image=ghcr.io/typetype-video/typetype-downloader-beta + ;; + token) + target_service=typetype-token + image_variable=TYPETYPE_TOKEN_BETA_IMAGE + expected_image=ghcr.io/typetype-video/typetype-token-beta + ;; + *) exit 64 ;; +esac +if [[ "$component" != all ]]; then + [[ "$image" == "$expected_image" ]] + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] +fi export COMPOSE_FILE="$root/docker-compose.dev.yml" export COMPOSE_PROJECT_NAME="$project" @@ -18,12 +51,30 @@ compose() { docker compose --env-file "$root/.env" "$@" } +set_env_value() { + local key="$1" + local value="$2" + local temporary + temporary=$(mktemp "$root/.env.XXXXXX") + awk -v key="$key" -v value="$value" ' + BEGIN { found = 0 } + index($0, key "=") == 1 { + if (!found) print key "=" value + found = 1 + next + } + { print } + END { if (!found) print key "=" value } + ' "$root/.env" > "$temporary" + chmod --reference="$root/.env" "$temporary" + chown --reference="$root/.env" "$temporary" + mv "$temporary" "$root/.env" +} managed_files=( .env.example docker-compose.dev.yml - garage.toml - nginx.conf scripts/bootstrap-garage.sh + scripts/check-youtube-egress.sh scripts/deploy-beta.sh ) services=( @@ -34,14 +85,22 @@ services=( postgres postgres-init dragonfly + garage-config garage ) docker compose --project-directory "$root" --env-file "$root/.env" \ -f "$source_root/docker-compose.dev.yml" config -q +if [[ "$component" == all ]]; then + proxy_url=$(docker compose --project-directory "$root" --env-file "$root/.env" \ + -f "$source_root/docker-compose.dev.yml" config --environment \ + | sed -n 's/^YOUTUBE_OUTBOUND_PROXY_URL=//p') + "$source_root/scripts/check-youtube-egress.sh" "$project" "$proxy_url" +fi rollback_root="$root/.deploy-rollbacks" backup="$rollback_root/$(date -u +'%Y%m%dT%H%M%SZ')-$$" mkdir -p "$backup/scripts" +cp -a "$root/.env" "$backup/.env" for file in "${managed_files[@]}"; do if [[ -e "$root/$file" ]]; then @@ -50,13 +109,21 @@ for file in "${managed_files[@]}"; do printf '%s\n' "$file" >> "$backup/existing-files" fi done - printf 'services:\n' > "$backup/rollback.yml" -for service in "${services[@]}"; do - container=$(compose ps -a -q "$service" | head -n 1) +declare -A current_services=() +rollback_services=() +while read -r current_service; do + current_services["$current_service"]=1 + rollback_services+=("$current_service") +done < <(compose config --services) +for current_service in "${services[@]}"; do + if [[ -z "${current_services[$current_service]:-}" ]]; then + continue + fi + container=$(compose ps -a -q "$current_service" | head -n 1) if [[ -n "$container" ]]; then - image=$(docker inspect "$container" --format '{{.Image}}') - printf ' %s:\n image: "%s"\n' "$service" "$image" >> "$backup/rollback.yml" + current_image=$(docker inspect "$container" --format '{{.Image}}') + printf ' %s:\n image: "%s"\n' "$current_service" "$current_image" >> "$backup/rollback.yml" fi done @@ -74,6 +141,7 @@ finish() { printf 'failed\n' > "$backup/status" set +e + cp -a "$backup/.env" "$root/.env" for file in "${managed_files[@]}"; do if grep -Fxq "$file" "$backup/existing-files" 2>/dev/null; then mkdir -p "$root/$(dirname "$file")" @@ -83,8 +151,13 @@ finish() { fi done cd "$root" - docker compose --env-file .env -f docker-compose.dev.yml -f "$backup/rollback.yml" \ - up -d --remove-orphans + if [[ "$component" == all ]]; then + docker compose --env-file .env -f docker-compose.dev.yml -f "$backup/rollback.yml" \ + up -d --no-deps "${rollback_services[@]}" + else + docker compose --env-file .env -f docker-compose.dev.yml -f "$backup/rollback.yml" \ + up -d --no-deps "$target_service" + fi docker compose --env-file .env -f docker-compose.dev.yml -f "$backup/rollback.yml" ps exit "$status" } @@ -92,16 +165,15 @@ trap finish EXIT install -m 644 "$source_root/.env.example" "$root/.env.example" install -m 644 "$source_root/docker-compose.dev.yml" "$root/docker-compose.dev.yml" -install -m 644 "$source_root/garage.toml" "$root/garage.toml" -install -m 644 "$source_root/nginx.conf" "$root/nginx.conf" install -m 755 "$source_root/scripts/bootstrap-garage.sh" "$root/scripts/bootstrap-garage.sh" +install -m 755 "$source_root/scripts/check-youtube-egress.sh" "$root/scripts/check-youtube-egress.sh" install -m 755 "$source_root/scripts/deploy-beta.sh" "$root/scripts/deploy-beta.sh" +install -d -m 700 "$root/.typetype-migration" +if [[ -s "$root/garage.toml" ]]; then + install -D -m 600 "$root/garage.toml" "$root/.typetype-migration/garage.toml" +fi cd "$root" -./scripts/bootstrap-garage.sh -compose pull -compose up -d --remove-orphans --wait --wait-timeout 180 - probe() { local url="$1" for attempt in $(seq 1 30); do @@ -113,10 +185,36 @@ probe() { return 1 } -probe http://127.0.0.1:8080/health -probe http://typetype-token:8081/health -probe 'http://typetype-token:8081/potoken?videoId=dQw4w9WgXcQ' -probe http://typetype-downloader:18093/health/deep -compose exec -T typetype wget -q -T 20 -t 1 -O /dev/null http://127.0.0.1/api/health +if [[ "$component" == all ]]; then + compose pull + compose up -d --remove-orphans --wait --wait-timeout 180 + ./scripts/bootstrap-garage.sh + probe http://127.0.0.1:8080/health + probe http://typetype-token:8081/health + probe 'http://typetype-token:8081/potoken?videoId=dQw4w9WgXcQ' + probe http://typetype-downloader:18093/health/deep + compose exec -T typetype wget -q -T 20 -t 1 -O /dev/null http://127.0.0.1/api/health +else + set_env_value "$image_variable" "$image@$digest" + compose pull "$target_service" + compose up -d --no-deps --wait --wait-timeout 180 "$target_service" + case "$component" in + frontend) + compose exec -T typetype wget -q -T 20 -t 1 -O /dev/null \ + http://127.0.0.1/api/health + ;; + server) + probe http://127.0.0.1:8080/health + ;; + token) + compose exec -T typetype-token curl --fail --silent \ + http://127.0.0.1:8081/health >/dev/null + probe 'http://typetype-token:8081/potoken?videoId=dQw4w9WgXcQ' + ;; + downloader) + probe http://typetype-downloader:18093/health/deep + ;; + esac +fi compose ps succeeded=true diff --git a/scripts/deploy-beta.test.sh b/scripts/deploy-beta.test.sh new file mode 100755 index 00000000..fa1f59f7 --- /dev/null +++ b/scripts/deploy-beta.test.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +repository="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT + +stack="$temporary/stack" +fake_bin="$temporary/bin" +mkdir -p "$stack/scripts" "$fake_bin" +printf 'POSTGRES_PASSWORD=test\n' > "$stack/.env" +for file in \ + .env.example \ + docker-compose.dev.yml \ + scripts/bootstrap-garage.sh \ + scripts/check-youtube-egress.sh \ + scripts/deploy-beta.sh; do + mkdir -p "$stack/$(dirname "$file")" + cp "$repository/$file" "$stack/$file" +done + +cat > "$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%q ' "$@" >> "$FAKE_DOCKER_LOG" +printf '\n' >> "$FAKE_DOCKER_LOG" + +case "$1" in + ps) + if [[ " $* " == *" -q "* ]]; then + echo anchor + fi + ;; + inspect) + if [[ "$2" == anchor ]]; then + if [[ "$*" == *"State.Pid"* ]]; then + echo 12345 + else + echo "$FAKE_STACK_ROOT" + fi + else + echo sha256:old + fi + ;; + compose) + if [[ "${FAKE_FAIL_UPDATE:-0}" == 1 && "$*" == *"up -d --no-deps --wait"* ]]; then + exit 1 + fi + case "$*" in + *"config --services"*) + printf '%s\n' typetype typetype-server typetype-downloader \ + typetype-token postgres postgres-init \ + dragonfly garage-config garage + ;; + *"config --environment"*) + echo "YOUTUBE_OUTBOUND_PROXY_URL=http://127.0.0.1:29083" + ;; + *"ps -a -q"*) + echo "container-${*: -1}" + ;; + esac + ;; +esac +EOF +chmod +x "$fake_bin/docker" + +cat > "$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${FAKE_EGRESS_FAIL:-0}" == 1 ]]; then + exit 1 +fi +EOF +chmod +x "$fake_bin/curl" + +cat > "$fake_bin/nsenter" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%q ' "$@" >> "$FAKE_NSENTER_LOG" +printf '\n' >> "$FAKE_NSENTER_LOG" +while [[ "$1" != -- ]]; do + shift +done +shift +exec "$@" +EOF +chmod +x "$fake_bin/nsenter" + +export FAKE_DOCKER_LOG="$temporary/docker.log" +export FAKE_NSENTER_LOG="$temporary/nsenter.log" +export FAKE_STACK_ROOT="$stack" +export PATH="$fake_bin:$PATH" +digest="sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +TYPETYPE_DEPLOY_COMPONENT=server \ +TYPETYPE_DEPLOY_IMAGE=ghcr.io/typetype-video/typetype-server-beta \ +TYPETYPE_DEPLOY_DIGEST="$digest" \ + "$repository/scripts/deploy-beta.sh" "$repository" + +if ! grep -Fq "TYPETYPE_SERVER_BETA_IMAGE=ghcr.io/typetype-video/typetype-server-beta@$digest" \ + "$stack/.env"; then + echo "the server image digest was not persisted" >&2 + sed -n '/^TYPETYPE_.*_IMAGE=/p' "$stack/.env" >&2 + exit 1 +fi +if ! grep -Fq 'pull typetype-server' "$FAKE_DOCKER_LOG"; then + echo "the server rollout did not pull only the server image" >&2 + cat "$FAKE_DOCKER_LOG" >&2 + exit 1 +fi +if ! grep -Fq 'up -d --no-deps --wait --wait-timeout 180 typetype-server' \ + "$FAKE_DOCKER_LOG"; then + echo "the server rollout did not restart only the server service" >&2 + cat "$FAKE_DOCKER_LOG" >&2 + exit 1 +fi +if grep -Eq 'compose .* pull $' "$FAKE_DOCKER_LOG"; then + echo "a server rollout must not pull the full stack" >&2 + exit 1 +fi + +previous_pin=$(grep '^TYPETYPE_SERVER_BETA_IMAGE=' "$stack/.env") +: > "$FAKE_DOCKER_LOG" +failed_digest="sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" +if FAKE_FAIL_UPDATE=1 \ + TYPETYPE_DEPLOY_COMPONENT=server \ + TYPETYPE_DEPLOY_IMAGE=ghcr.io/typetype-video/typetype-server-beta \ + TYPETYPE_DEPLOY_DIGEST="$failed_digest" \ + "$repository/scripts/deploy-beta.sh" "$repository"; then + echo "the simulated failed rollout unexpectedly succeeded" >&2 + exit 1 +fi +grep -Fxq "$previous_pin" "$stack/.env" +grep -Fq 'up -d --no-deps typetype-server' "$FAKE_DOCKER_LOG" +if grep -Eq 'compose .* up .*typetype($| )' "$FAKE_DOCKER_LOG"; then + echo "a server rollback must not recreate the frontend" >&2 + exit 1 +fi + +: > "$FAKE_DOCKER_LOG" +if FAKE_EGRESS_FAIL=1 TYPETYPE_DEPLOY_COMPONENT=all \ + "$repository/scripts/deploy-beta.sh" "$repository"; then + echo "a full rollout passed with an unavailable egress proxy" >&2 + exit 1 +fi +if ! grep -Fq -- '--target 12345 --net --' "$FAKE_NSENTER_LOG"; then + echo "the egress preflight did not use a beta container network namespace" >&2 + cat "$FAKE_NSENTER_LOG" >&2 + exit 1 +fi +if grep -Eq 'compose .* (pull|up) ' "$FAKE_DOCKER_LOG"; then + echo "a failed egress preflight mutated the stack" >&2 + cat "$FAKE_DOCKER_LOG" >&2 + exit 1 +fi diff --git a/scripts/deploy-stable.sh b/scripts/deploy-stable.sh index 028a65d0..71ec6b34 100755 --- a/scripts/deploy-stable.sh +++ b/scripts/deploy-stable.sh @@ -22,8 +22,6 @@ managed_files=( .env.example docker-compose.yml docker-compose.arm64.yml - garage.toml - nginx.conf scripts/bootstrap-env.sh scripts/bootstrap-garage.sh scripts/deploy-stable.sh @@ -37,6 +35,7 @@ services=( postgres postgres-init dragonfly + garage-config garage ) @@ -106,6 +105,10 @@ finish() { } trap finish EXIT +install -d -m 700 "$root/.typetype-migration" +if [[ -s "$root/garage.toml" ]]; then + install -D -m 600 "$root/garage.toml" "$root/.typetype-migration/garage.toml" +fi for file in "${managed_files[@]}"; do install -D -m 644 "$source_root/$file" "$root/$file" done @@ -115,9 +118,9 @@ chmod 755 "$root/scripts/deploy-stable.sh" cd "$root" ./scripts/bootstrap-env.sh -./scripts/bootstrap-garage.sh compose pull compose up -d --remove-orphans --wait --wait-timeout 180 +./scripts/bootstrap-garage.sh probe() { local service="$1" diff --git a/scripts/install-stack.sh b/scripts/install-stack.sh index 883ea662..9e452bf7 100755 --- a/scripts/install-stack.sh +++ b/scripts/install-stack.sh @@ -11,6 +11,8 @@ AUTO_APPROVE=0 REF_SET=0 INSTALL_DIR_SET=0 ENV_FILE_CREATED=0 +BACKUP_DIR="" +CUSTOM_COMPOSE_FILE="" DEFAULT_DOWNLOADER_S3_ACCESS_KEY="SET_ME_ACCESS_KEY" DEFAULT_DOWNLOADER_S3_SECRET_KEY="SET_ME_SECRET_KEY" @@ -340,6 +342,9 @@ compose_command_hint() { if [[ ${USE_ARM64_OVERRIDE} -eq 1 ]]; then args="${args} -f docker-compose.arm64.yml" fi + if [[ -n "${CUSTOM_COMPOSE_FILE}" ]]; then + args="${args} -f docker-compose.override.yml" + fi printf 'docker compose %s --env-file .env %s' "${args}" "${command}" } @@ -379,15 +384,79 @@ choose_stack_port() { fetch_file() { local relative_path="$1" local out_path="$2" + local temporary mkdir -p "$(dirname "${out_path}")" + temporary="$(mktemp "${out_path}.tmp.XXXXXX")" if [[ -n "${SOURCE_DIR}" ]]; then - cp "${SOURCE_DIR}/${relative_path}" "${out_path}" + cp "${SOURCE_DIR}/${relative_path}" "${temporary}" else local base_url="https://raw.githubusercontent.com/${REPO}/${REF}" - curl -fsSL "${base_url}/${relative_path}" -o "${out_path}" + curl -fsSL --retry 3 "${base_url}/${relative_path}" -o "${temporary}" fi + mv "${temporary}" "${out_path}" +} + +backup_existing_stack() { + local env_file="${INSTALL_DIR}/.env" + local existing=() + local source + local timestamp + + shopt -s nullglob + existing+=("${INSTALL_DIR}"/docker-compose*.yml) + shopt -u nullglob + for source in "${env_file}" "${INSTALL_DIR}/nginx.conf" "${INSTALL_DIR}/garage.toml"; do + [[ -e "${source}" ]] && existing+=("${source}") + done + [[ -d "${INSTALL_DIR}/scripts" ]] && existing+=("${INSTALL_DIR}/scripts") + [[ ${#existing[@]} -gt 0 ]] || return 0 + + timestamp="$(date -u +%Y%m%d-%H%M%S)" + BACKUP_DIR="${INSTALL_DIR}/.typetype-backups/update-${timestamp}" + mkdir -p "${BACKUP_DIR}" + chmod 0700 "${INSTALL_DIR}/.typetype-backups" "${BACKUP_DIR}" + for source in "${existing[@]}"; do + cp -a "${source}" "${BACKUP_DIR}/" + done + + if [[ -f "${COMPOSE_FILE}" ]]; then + local old_args=(-f "${COMPOSE_FILE}") + if is_arm64_host && [[ -f "${ARM64_COMPOSE_FILE}" ]]; then + old_args+=(-f "${ARM64_COMPOSE_FILE}") + fi + if [[ -f "${INSTALL_DIR}/docker-compose.override.yml" ]]; then + old_args+=(-f "${INSTALL_DIR}/docker-compose.override.yml") + fi + if docker compose "${old_args[@]}" --env-file "${env_file}" config -q 2>/dev/null; then + docker compose "${old_args[@]}" --env-file "${env_file}" config --images \ + > "${BACKUP_DIR}/resolved-images.txt" + docker compose "${old_args[@]}" --env-file "${env_file}" images \ + > "${BACKUP_DIR}/running-images.txt" 2>/dev/null || true + fi + fi + echo "[install] Backed up the existing stack to ${BACKUP_DIR}" +} + +stage_legacy_garage_config() { + local legacy_config="${INSTALL_DIR}/garage.toml" + + mkdir -p "${INSTALL_DIR}/.typetype-migration" + chmod 0700 "${INSTALL_DIR}/.typetype-migration" + [[ -s "${legacy_config}" ]] || return 0 + + cp "${legacy_config}" "${INSTALL_DIR}/.typetype-migration/garage.toml" + chmod 0600 "${INSTALL_DIR}/.typetype-migration/garage.toml" + echo "[install] Staged the existing garage.toml for one-time import into the managed config volume." +} + +verify_component_versions() { + local base_url="http://127.0.0.1:${HOST_PORT_WEB_RESOLVED}/api/version" + local endpoint + for endpoint in "" "/web" "/server" "/token" "/downloader"; do + curl -fsS "${base_url}${endpoint}" >/dev/null + done } need_cmd curl @@ -424,20 +493,33 @@ COMPOSE_FILE="${INSTALL_DIR}/${COMPOSE_NAME}" ARM64_COMPOSE_FILE="${INSTALL_DIR}/docker-compose.arm64.yml" USE_ARM64_OVERRIDE=0 +if [[ -s "${INSTALL_DIR}/nginx.conf" ]]; then + echo "[install] Existing nginx.conf detected; it will be backed up and is no longer mounted by default." + echo "[install] Keep using it only through an explicit Compose override." +fi +if [[ -s "${INSTALL_DIR}/garage.toml" ]]; then + echo "[install] Existing garage.toml detected; it will be preserved in the managed config volume." +fi + +backup_existing_stack + +stage_legacy_garage_config + fetch_file "${COMPOSE_SOURCE}" "${COMPOSE_FILE}" fetch_file "docker-compose.arm64.yml" "${ARM64_COMPOSE_FILE}" -fetch_file "nginx.conf" "${INSTALL_DIR}/nginx.conf" -fetch_file "garage.toml" "${INSTALL_DIR}/garage.toml" fetch_file ".env.example" "${INSTALL_DIR}/.env.example" fetch_file "scripts/install-stack.sh" "${INSTALL_DIR}/scripts/install-stack.sh" fetch_file "scripts/bootstrap-env.sh" "${INSTALL_DIR}/scripts/bootstrap-env.sh" fetch_file "scripts/bootstrap-garage.sh" "${INSTALL_DIR}/scripts/bootstrap-garage.sh" fetch_file "scripts/setup-stack.sh" "${INSTALL_DIR}/scripts/setup-stack.sh" +fetch_file "scripts/validate-stack.sh" "${INSTALL_DIR}/scripts/validate-stack.sh" +fetch_file "scripts/youtube-egress-relay.mjs" "${INSTALL_DIR}/scripts/youtube-egress-relay.mjs" chmod +x "${INSTALL_DIR}/scripts/install-stack.sh" chmod +x "${INSTALL_DIR}/scripts/bootstrap-env.sh" chmod +x "${INSTALL_DIR}/scripts/bootstrap-garage.sh" chmod +x "${INSTALL_DIR}/scripts/setup-stack.sh" +chmod +x "${INSTALL_DIR}/scripts/validate-stack.sh" COMPOSE_ARGS=(-f "${COMPOSE_FILE}") COMPOSE_OVERRIDE_FILE="" @@ -447,6 +529,11 @@ if is_arm64_host; then COMPOSE_ARGS+=(-f "${COMPOSE_OVERRIDE_FILE}") echo "[install] ARM64 host detected, using Redis cache override." fi +if [[ -f "${INSTALL_DIR}/docker-compose.override.yml" ]]; then + CUSTOM_COMPOSE_FILE="${INSTALL_DIR}/docker-compose.override.yml" + COMPOSE_ARGS+=(-f "${CUSTOM_COMPOSE_FILE}") + echo "[install] Custom Compose override detected and preserved." +fi if [[ ! -f "${INSTALL_DIR}/.env" ]]; then cp "${INSTALL_DIR}/.env.example" "${INSTALL_DIR}/.env" @@ -494,8 +581,13 @@ else printf '\nALLOWED_ORIGINS=%s\n' "${input_origins}" >> "${INSTALL_DIR}/.env" fi +echo "[install] Validating the resolved Compose stack..." +docker compose "${COMPOSE_ARGS[@]}" --env-file "${INSTALL_DIR}/.env" config -q + if [[ ${START_STACK} -eq 0 ]]; then echo "[install] Download-only complete." + [[ ! -s "${INSTALL_DIR}/garage.toml" ]] || echo "[install] The existing garage.toml will be imported on the next Compose startup." + [[ -z "${BACKUP_DIR}" ]] || echo "[install] Rollback files: ${BACKUP_DIR}" echo "[install] Next step: cd ${INSTALL_DIR} && $(compose_command_hint 'up -d')" exit 0 fi @@ -516,12 +608,19 @@ docker compose "${COMPOSE_ARGS[@]}" --env-file "${INSTALL_DIR}/.env" up -d --wai echo "[install] Bootstrapping Garage..." ( cd "${INSTALL_DIR}" - COMPOSE_FILE="${COMPOSE_FILE}" COMPOSE_OVERRIDE_FILE="${COMPOSE_OVERRIDE_FILE}" ./scripts/bootstrap-garage.sh + COMPOSE_FILE="${COMPOSE_FILE}" \ + COMPOSE_OVERRIDE_FILE="${COMPOSE_OVERRIDE_FILE}" \ + COMPOSE_CUSTOM_FILE="${CUSTOM_COMPOSE_FILE}" \ + ./scripts/bootstrap-garage.sh ) +echo "[install] Checking component version endpoints..." +verify_component_versions + echo "[install] Service status:" docker compose "${COMPOSE_ARGS[@]}" --env-file "${INSTALL_DIR}/.env" ps echo echo "Done. Open http://localhost:${HOST_PORT_WEB_RESOLVED}" echo "Install directory: ${INSTALL_DIR}" +[[ -z "${BACKUP_DIR}" ]] || echo "Rollback files: ${BACKUP_DIR}" diff --git a/scripts/setup-stack.sh b/scripts/setup-stack.sh index 47a75270..a3803882 100755 --- a/scripts/setup-stack.sh +++ b/scripts/setup-stack.sh @@ -267,11 +267,22 @@ cd "${ROOT_DIR}" COMPOSE_ARGS=(-f "${ROOT_DIR}/docker-compose.yml") COMPOSE_OVERRIDE_FILE="" +COMPOSE_CUSTOM_FILE="" if is_arm64_host && [[ -f "${ROOT_DIR}/docker-compose.arm64.yml" ]]; then COMPOSE_OVERRIDE_FILE="${ROOT_DIR}/docker-compose.arm64.yml" COMPOSE_ARGS+=(-f "${COMPOSE_OVERRIDE_FILE}") echo "[setup] ARM64 host detected, using Redis cache override." fi +if [[ -f "${ROOT_DIR}/docker-compose.override.yml" ]]; then + COMPOSE_CUSTOM_FILE="${ROOT_DIR}/docker-compose.override.yml" + COMPOSE_ARGS+=(-f "${COMPOSE_CUSTOM_FILE}") + echo "[setup] Custom Compose override detected." +fi +install -d -m 700 "${ROOT_DIR}/.typetype-migration" +if [[ -s "${ROOT_DIR}/garage.toml" ]]; then + install -D -m 600 "${ROOT_DIR}/garage.toml" "${ROOT_DIR}/.typetype-migration/garage.toml" + echo "[setup] Staged the existing garage.toml for import into the managed config volume." +fi ensure_youtube_remote_login_env "${ENV_FILE}" "${ROOT_DIR}/scripts/bootstrap-env.sh" @@ -291,14 +302,20 @@ if [[ -z "${CURRENT_ALLOWED_ORIGINS}" || "${CURRENT_ALLOWED_ORIGINS}" == "${PACK fi echo +echo "[setup] Validating the resolved Compose stack..." +docker compose "${COMPOSE_ARGS[@]}" config -q + echo "[setup] Pulling images..." docker compose "${COMPOSE_ARGS[@]}" pull echo "[setup] Starting services..." -docker compose "${COMPOSE_ARGS[@]}" up -d +docker compose "${COMPOSE_ARGS[@]}" up -d --wait --wait-timeout 180 echo "[setup] Bootstrapping Garage for downloader..." -COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml" COMPOSE_OVERRIDE_FILE="${COMPOSE_OVERRIDE_FILE}" "${ROOT_DIR}/scripts/bootstrap-garage.sh" +COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml" \ + COMPOSE_OVERRIDE_FILE="${COMPOSE_OVERRIDE_FILE}" \ + COMPOSE_CUSTOM_FILE="${COMPOSE_CUSTOM_FILE}" \ + "${ROOT_DIR}/scripts/bootstrap-garage.sh" echo "[setup] Current service status:" docker compose "${COMPOSE_ARGS[@]}" ps diff --git a/scripts/validate-stack.sh b/scripts/validate-stack.sh index 5f531f30..427b1a25 100755 --- a/scripts/validate-stack.sh +++ b/scripts/validate-stack.sh @@ -19,7 +19,37 @@ trap cleanup EXIT for script in scripts/*.sh; do bash -n "$script" done +./scripts/deploy-beta.test.sh docker compose --env-file .env.example -f docker-compose.yml config -q docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.arm64.yml config -q -docker compose --env-file .env.example -f docker-compose.dev.yml config -q +if docker compose --env-file .env.example -f docker-compose.dev.yml config -q 2>/dev/null; then + echo "beta Compose must reject a missing outbound proxy" >&2 + exit 1 +fi +YOUTUBE_OUTBOUND_PROXY_URL=http://127.0.0.1:29083 \ + docker compose --env-file .env.example -f docker-compose.dev.yml config -q + +stable_config="$(docker compose --env-file .env.example -f docker-compose.yml config)" +dev_config="$(YOUTUBE_OUTBOUND_PROXY_URL=http://127.0.0.1:29083 \ + docker compose --env-file .env.example -f docker-compose.dev.yml config)" +if grep -q '/etc/nginx/conf.d/default.conf' <<<"${stable_config}${dev_config}"; then + echo "default Compose must use the nginx configuration bundled in the web image" >&2 + exit 1 +fi +if ! grep -q 'source: garage_config' <<<"${stable_config}"; then + echo "stable Compose must use the managed Garage config volume" >&2 + exit 1 +fi +if ! grep -q 'source: garage_config' <<<"${dev_config}"; then + echo "beta Compose must use the managed Garage config volume" >&2 + exit 1 +fi +if grep -q 'youtube-egress-relay' <<<"$dev_config"; then + echo "beta Compose must use the externally validated egress proxy" >&2 + exit 1 +fi +if [[ $(grep -c 'YOUTUBE_OUTBOUND_PROXY_URL: http://127.0.0.1:29083' <<<"$dev_config") -ne 2 ]]; then + echo "beta Server and Token must share the configured outbound proxy" >&2 + exit 1 +fi