diff --git a/.github/skills/ctf-testing/deploy_and_test.sh b/.github/skills/ctf-testing/deploy_and_test.sh index 7182faa..f5f64ca 100755 --- a/.github/skills/ctf-testing/deploy_and_test.sh +++ b/.github/skills/ctf-testing/deploy_and_test.sh @@ -15,7 +15,7 @@ # reboot and progress persists # # Prerequisites: -# - terraform (>= 1.0) +# - terraform (>= 1.0; Azure requires >= 1.14.0) # - jq (for AWS terraform config) # - sshpass (macOS: brew install hudochenkov/sshpass/sshpass) # - aws CLI (for AWS, must be logged in) @@ -398,7 +398,8 @@ _reboot_vm() { local provider="$1" local ip="$2" - _log INFO "Rebooting VM (${provider})..." + # This function returns the VM IP via stdout, so logs must go to stderr. + _log INFO "Rebooting VM (${provider})..." >&2 case "${provider}" in aws) @@ -416,10 +417,10 @@ _reboot_vm() { return 1 fi - echo " Stopping instance ${instance_id}..." + echo " Stopping instance ${instance_id}..." >&2 aws ec2 stop-instances --instance-ids "${instance_id}" > /dev/null aws ec2 wait instance-stopped --instance-ids "${instance_id}" - echo " Starting instance ${instance_id}..." + echo " Starting instance ${instance_id}..." >&2 aws ec2 start-instances --instance-ids "${instance_id}" > /dev/null aws ec2 wait instance-running --instance-ids "${instance_id}" # IP may change, get new one @@ -427,22 +428,16 @@ _reboot_vm() { ip=$(_get_public_ip "${provider}") ;; azure) - echo " Restarting Azure VM..." - az vm restart --resource-group ctf-resources --name ctf-vm - # az vm restart waits by default, but add explicit wait for running state - az vm wait \ - --resource-group ctf-resources \ - --name ctf-vm \ - --created \ - --timeout 120 2>/dev/null || true + echo " Restarting Azure VM..." >&2 + az vm restart --resource-group ctf-resources --name ctf-vm >&2 || return 1 ;; gcp) - echo " Restarting GCP VM..." + echo " Restarting GCP VM..." >&2 local zone zone=$(cd "${REPO_ROOT}/${provider}" \ && terraform output -raw zone 2>/dev/null \ || echo "us-central1-a") - gcloud compute instances reset ctf-instance --zone="${zone}" --quiet + gcloud compute instances reset ctf-instance --zone="${zone}" --quiet >&2 || return 1 # Wait for VM to be running local attempts=0 while [[ ${attempts} -lt 30 ]]; do @@ -467,6 +462,19 @@ _reboot_vm() { # TEST EXECUTION # ============================================================================= +# Copy test script to VM +# Arguments: +# $1 - Cloud provider name +# $2 - IP address of the VM +_copy_test_script() { + local provider="$1" + local ip="$2" + + _log INFO "Copying test script to ${provider} VM..." + # shellcheck disable=SC2086 + _sshpass_cmd scp ${SSH_OPTS} "${TEST_SCRIPT}" "${SSH_USER}@${ip}:/tmp/test_ctf_challenges.sh" +} + # Copy test script to VM and execute it # Arguments: # $1 - Cloud provider name @@ -482,9 +490,7 @@ _run_tests() { test_flags="${test_flags} --with-reboot" fi - _log INFO "Copying test script to VM..." - # shellcheck disable=SC2086 - _sshpass_cmd scp ${SSH_OPTS} "${TEST_SCRIPT}" "${SSH_USER}@${ip}:/tmp/test_ctf_challenges.sh" + _copy_test_script "${provider}" "${ip}" _log INFO "Running tests on ${provider} VM (${ip})..." echo "" @@ -508,11 +514,14 @@ _run_post_reboot_tests() { local provider="$1" local ip="$2" + _copy_test_script "${provider}" "${ip}" + _log INFO "Running post-reboot verification on ${provider}..." local exit_code=0 # shellcheck disable=SC2086 - _sshpass_cmd ssh ${SSH_OPTS} "${SSH_USER}@${ip}" "/tmp/test_ctf_challenges.sh" \ + _sshpass_cmd ssh ${SSH_OPTS} "${SSH_USER}@${ip}" \ + "chmod +x /tmp/test_ctf_challenges.sh && /tmp/test_ctf_challenges.sh --post-reboot" \ || exit_code=$? return "${exit_code}" @@ -592,13 +601,15 @@ _test_provider() { _log WARN "Reboot requested - performing VM reboot..." local new_ip - new_ip=$(_reboot_vm "${provider}" "${ip}") - - # Wait for SSH after reboot - _wait_for_ssh "${new_ip}" - - # Run post-reboot tests - _run_post_reboot_tests "${provider}" "${new_ip}" || test_exit_code=$? + if ! new_ip=$(_reboot_vm "${provider}" "${ip}"); then + _log ERROR "VM reboot failed for ${provider}" + result=1 + elif ! _wait_for_ssh "${new_ip}"; then + _log ERROR "SSH connection failed after reboot for ${provider}" + result=1 + elif ! _run_post_reboot_tests "${provider}" "${new_ip}"; then + result=1 + fi elif [[ ${test_exit_code} -ne 0 ]]; then result=1 fi diff --git a/.github/skills/ctf-testing/test_ctf_challenges.sh b/.github/skills/ctf-testing/test_ctf_challenges.sh index 25a29dc..dbd465d 100644 --- a/.github/skills/ctf-testing/test_ctf_challenges.sh +++ b/.github/skills/ctf-testing/test_ctf_challenges.sh @@ -9,11 +9,12 @@ # can complete the CTF. # # Usage: -# ./test_ctf_challenges.sh [--with-reboot] +# ./test_ctf_challenges.sh [--with-reboot|--post-reboot] # DEBUG=true ./test_ctf_challenges.sh # Enable debug tracing # # Flags: # --with-reboot After tests pass, signal reboot to verify services persist +# --post-reboot Run only the post-reboot verification phase # # Exit codes: # 0 - All tests passed @@ -44,9 +45,10 @@ readonly GREEN='\033[0;32m' readonly YELLOW='\033[1;33m' readonly NC='\033[0m' # No Color -# File paths for reboot test coordination -readonly REBOOT_MARKER="/tmp/.ctf_reboot_test_marker" -readonly PROGRESS_SNAPSHOT="/tmp/.ctf_progress_snapshot" +# File paths for reboot test coordination. These must survive a VM reboot. +readonly TEST_STATE_DIR="${HOME}/.linux-ctfs-test" +readonly REBOOT_MARKER="${TEST_STATE_DIR}/.ctf_reboot_test_marker" +readonly PROGRESS_SNAPSHOT="${TEST_STATE_DIR}/.ctf_progress_snapshot" # ============================================================================= # GLOBAL STATE @@ -58,15 +60,34 @@ FAILED=0 # Parse arguments WITH_REBOOT=false -for arg in "$@"; do - case $arg in +POST_REBOOT=false +usage() { + echo "Usage: $0 [--with-reboot|--post-reboot]" +} + +while [[ $# -gt 0 ]]; do + case "$1" in --with-reboot) WITH_REBOOT=true - shift + ;; + --post-reboot) + POST_REBOOT=true + ;; + *) + echo "Unknown argument: $1" + usage + exit 1 ;; esac + shift done +if [[ "${WITH_REBOOT}" == true && "${POST_REBOOT}" == true ]]; then + echo "--with-reboot and --post-reboot cannot be used together." + usage + exit 1 +fi + # ============================================================================= # HELPER FUNCTIONS # ============================================================================= @@ -130,11 +151,18 @@ _verify_flag() { # ============================================================================ # POST-REBOOT VERIFICATION # ============================================================================ -if [[ -f "${REBOOT_MARKER}" ]]; then +if [[ "${POST_REBOOT}" == true ]]; then _section "POST-REBOOT VERIFICATION" - + + if [[ ! -f "${REBOOT_MARKER}" ]]; then + _fail "Reboot marker not found - reboot verification was not prepared" + echo "" + echo "Passed: ${PASSED} | Failed: ${FAILED}" + exit 1 + fi + echo "Verifying services survived reboot..." - + for service in ctf-secret-service ctf-monitor-directory ctf-ping-message ctf-secret-process nginx; do if systemctl is-active "${service}" &>/dev/null; then _pass "${service} is running after reboot" @@ -142,19 +170,20 @@ if [[ -f "${REBOOT_MARKER}" ]]; then _fail "${service} failed to start after reboot - SETUP BUG" fi done - + if [ -f "$PROGRESS_SNAPSHOT" ]; then EXPECTED=$(cat "$PROGRESS_SNAPSHOT") - ACTUAL=$(sort -u /var/ctf/completed_challenges 2>/dev/null | wc -l) + ACTUAL=$( { sort -u /var/ctf/completed_challenges 2>/dev/null || true; } | wc -l ) if [ "$ACTUAL" -ge "$EXPECTED" ]; then _pass "Progress persisted after reboot ($ACTUAL checks)" else _fail "Progress lost after reboot (expected ${EXPECTED}, got ${ACTUAL})" fi fi - + rm -f "${REBOOT_MARKER}" "${PROGRESS_SNAPSHOT}" - + rmdir "${TEST_STATE_DIR}" 2>/dev/null || true + echo "" echo "Passed: ${PASSED} | Failed: ${FAILED}" [[ ${FAILED} -eq 0 ]] && exit 0 || exit 1 @@ -644,9 +673,10 @@ echo "Flags captured: ${#FLAGS[@]}" echo "" if [ "$WITH_REBOOT" = true ] && [ $FAILED -eq 0 ]; then + mkdir -p "${TEST_STATE_DIR}" sort -u /var/ctf/completed_challenges 2>/dev/null | wc -l > "$PROGRESS_SNAPSHOT" touch "$REBOOT_MARKER" - echo "Reboot marker created. Re-run after reboot to verify services." + echo "Reboot marker created. After reboot, re-run with --post-reboot to verify services." exit 100 fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 973a6fe..51dc3c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ All PRs that change setup, challenges, verify behavior, or Terraform should be t Install: -1. `terraform` 1.0 or newer +1. `terraform` 1.0 or newer; Azure requires Terraform 1.14.0 or newer 2. `jq` 3. `sshpass` 4. The cloud CLI for the provider you want to test @@ -120,6 +120,12 @@ Most contributors only need to know this: - Contributor testing uses local files. - `deploy_and_test.sh` handles contributor mode for you. +Setup readiness differs by provider: + +- Azure release mode uses VM Custom Script Extension (Terraform 1.14.0 or newer), so Terraform waits for extension success or failure. +- AWS and GCP release mode still use the shared SSH marker wait. +- Contributor mode stays on `use_local_setup=true`, uploading local files over SSH for test runs. + If you manually run Terraform to test local setup changes, pass: ```bash diff --git a/azure/README.md b/azure/README.md index 8cfa09c..29a5ff9 100644 --- a/azure/README.md +++ b/azure/README.md @@ -93,9 +93,17 @@ Type `yes` when prompted. ## Troubleshooting 1. Ensure your Azure CLI is logged in with valid credentials -2. Check that you're using Terraform v1.9.0 or later +2. Check that you're using Terraform v1.14.0 or later 3. Verify you have permissions to create VMs, VNets, and Network Security Groups +If release setup fails during `terraform apply`, Azure reports the failure through the VM Custom Script Extension. Useful VM-side logs are: + +```text +/var/log/ctf_setup.log +/var/log/waagent.log +/var/log/azure/custom-script/handler.log +``` + If problems persist, please open an issue: https://github.com/learntocloud/linux-ctfs/issues diff --git a/azure/main.tf b/azure/main.tf index 2edc825..8733f2c 100644 --- a/azure/main.tf +++ b/azure/main.tf @@ -73,7 +73,7 @@ locals { rm -f "$${FAILED_MARKER}" fail_setup() { - echo "CTF setup failed. Check /var/log/cloud-init-output.log and /var/log/ctf_setup.log." >&2 + echo "CTF setup failed. Check /var/log/ctf_setup.log and Azure Custom Script Extension logs." >&2 touch "$${FAILED_MARKER}" } trap fail_setup ERR @@ -92,8 +92,28 @@ locals { return 1 } - apt-get update - apt-get install -y ca-certificates curl tar gzip coreutils + wait_for_cloud_init() { + if command -v cloud-init >/dev/null 2>&1; then + cloud-init status --wait || true + fi + } + + apt_get_update_with_retry() { + local attempt + for attempt in 1 2 3 4 5; do + if apt-get -o DPkg::Lock::Timeout=120 -o Acquire::Retries=3 update; then + return 0 + fi + echo "apt-get update failed. Attempt $${attempt}/5." + rm -rf /var/lib/apt/lists/partial/* + sleep 10 + done + return 1 + } + + wait_for_cloud_init + apt_get_update_with_retry + apt-get -o DPkg::Lock::Timeout=120 -o Acquire::Retries=3 install -y ca-certificates curl tar gzip coreutils cd "$${WORK_DIR}" download_with_retry "$${SETUP_URL}" "$${ASSET_NAME}" @@ -111,26 +131,11 @@ locals { trap - ERR EOF - release_readiness_script = <<-EOF - set -eu - echo "Waiting for CTF setup to finish..." - for attempt in $(seq 1 180); do - if test -f /var/lib/linux-ctfs/setup.failed; then - echo "CTF setup failed. Check /var/log/ctf_setup.log and /var/log/cloud-init-output.log." >&2 - exit 1 - fi - - if test -f /var/lib/linux-ctfs/setup.done || test -f /var/lib/cloud/instance/ctf-setup.done || test -f /var/log/setup_complete; then - echo "CTF setup is complete." - exit 0 - fi - - echo "CTF setup is still running. Attempt $attempt/180." - sleep 10 - done - - echo "Timed out waiting for CTF setup. Check /var/log/ctf_setup.log and /var/log/cloud-init-output.log." >&2 - exit 1 + azure_release_extension_script = <<-EOF +#!/bin/sh +exec /bin/bash <<'LINUX_CTFS_SETUP' +${local.release_setup_script} +LINUX_CTFS_SETUP EOF } @@ -270,7 +275,24 @@ resource "azurerm_linux_virtual_machine" "ctf_vm" { version = "latest" } - custom_data = base64encode(var.use_local_setup ? local.local_bootstrap_script : local.release_setup_script) + custom_data = var.use_local_setup ? base64encode(local.local_bootstrap_script) : null +} + +resource "azurerm_virtual_machine_extension" "release_setup" { + count = var.use_local_setup ? 0 : 1 + name = "linux-ctfs-release-setup" + virtual_machine_id = azurerm_linux_virtual_machine.ctf_vm.id + publisher = "Microsoft.Azure.Extensions" + type = "CustomScript" + type_handler_version = "2.1" + + protected_settings = jsonencode({ + script = base64encode(local.azure_release_extension_script) + }) + + tags = { + setup_release_tag = var.setup_release_tag + } } action "azurerm_virtual_machine_power" "ctf_power_off" { @@ -323,28 +345,8 @@ resource "null_resource" "local_setup" { } } -resource "null_resource" "release_setup_ready" { - count = var.use_local_setup ? 0 : 1 - depends_on = [azurerm_linux_virtual_machine.ctf_vm] - - triggers = { - instance_id = azurerm_linux_virtual_machine.ctf_vm.id - } - - connection { - host = azurerm_linux_virtual_machine.ctf_vm.public_ip_address - user = "ctf_user" - password = "CTFpassword123!" - timeout = "30m" - } - - provisioner "remote-exec" { - inline = [local.release_readiness_script] - } -} - # Output the public IP address output "public_ip_address" { value = azurerm_linux_virtual_machine.ctf_vm.public_ip_address - depends_on = [null_resource.local_setup, null_resource.release_setup_ready] -} \ No newline at end of file + depends_on = [null_resource.local_setup, azurerm_virtual_machine_extension.release_setup] +}