diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5268cd860..1bf2f469a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1557,6 +1557,58 @@ jobs: retention-days: 7 if-no-files-found: ignore + # Native Intel Mac proof of gateway readiness as docs/LAUNCHER_CONTRACT.md + # defines it: `elastos gateway --addr localhost:8090`, then + # GET http://localhost:8090/healthz returns HTTP 200 and OK while that pid + # holds host-process.lock. The Home is empty. This is not a signed release. + gateway-smoke-macos-intel: + name: gateway-smoke-macos-intel + runs-on: macos-15-intel + timeout-minutes: 90 + needs: source-gate + env: + CARGO_BUILD_JOBS: "2" + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + - uses: dtolnay/rust-toolchain@72504271e02fb719eb4eb3eaece3a8e3d06f57c0 # 1.91.0 + - name: require an Intel Mac with room to compile + run: | + set -euo pipefail + [[ "$RUNNER_OS/$RUNNER_ARCH" == macOS/X64 ]] + df -Pk . + df -Pk . | awk 'NR == 2 { if ($4 < 8 * 1024 * 1024) { print "Intel runner has under 8 GiB free"; exit 1 } }' + # source-gate runs these tests on Linux. Run them here on macOS too, + # where the lock path, lsof and localhost order differ. + - name: smoke script tests on macOS + timeout-minutes: 5 + run: python3 scripts/gateway-healthz-smoke-test.py -v + - name: build the gateway binary + timeout-minutes: 70 + run: | + set -euo pipefail + # GITHUB_ENV applies on the next step. Export it here so this build + # writes the binary the /healthz probe executes. + export CARGO_TARGET_DIR="$RUNNER_TEMP/gateway-intel-target" + echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR" >> "$GITHUB_ENV" + cargo build --locked --release --manifest-path elastos/Cargo.toml -p elastos-server --bin elastos + test -x "$CARGO_TARGET_DIR/release/elastos" + - name: gateway is ready at localhost:8090 with an empty Home + timeout-minutes: 10 + run: | + set -euo pipefail + export ELASTOS_GATEWAY_LOG="$RUNNER_TEMP/gateway.log" + scripts/gateway-healthz-smoke.sh "$CARGO_TARGET_DIR/release/elastos" + - name: upload gateway log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: gateway-smoke-macos-intel-log + path: ${{ runner.temp }}/gateway.log + retention-days: 7 + if-no-files-found: ignore + # A pull request run stops at its first failed job, so the macOS and Linux # journeys do not run on for 30+ minutes after a decisive failure. Pushes, # merge queues and dispatches keep running every job to the end. This job diff --git a/docs/MAC.md b/docs/MAC.md index e2ee5830a..28f7a1b86 100644 --- a/docs/MAC.md +++ b/docs/MAC.md @@ -4,6 +4,40 @@ The public installer looks up Apple silicon (`aarch64-darwin`) releases. This runbook is the source-home staging path for local Mac verification, Browser VM artifacts, and `.dmg` packaging work. +## Intel Mac + +The public release id for an Intel Mac is `x86_64-darwin`. `scripts/install.sh` +stops on that host before a download lookup. The message is +`macOS release lookup requires Apple silicon`. + +A person with an Intel Mac has no signed download. The next step is the signed +installer on an Apple silicon Mac. The source-home commands later in this +runbook are a developer build. They are not the installer, and this note does +not claim that build has passed on Intel. + +CI job `gateway-smoke-macos-intel` builds the native `elastos` binary on +`macos-15-intel`. It runs `scripts/gateway-healthz-smoke.sh`, which starts the +command in [the launcher contract](LAUNCHER_CONTRACT.md) and requires the +readiness that contract defines. The probe uses `http://localhost:8090/healthz` +and passes only when all of these are true: + +- the answer is HTTP 200 with the two bytes `OK`, without a proxy, +- the address that answered is a listen socket of the gateway process, +- `host-process.lock` names that process, a gateway role and `localhost:8090`. + +The job uses an empty Home. A pass does not prove providers, capsules, +sign-in, or a signed install. `elastos serve` reports operator status at +`GET /api/health`. + +For `--addr localhost:8090` the gateway listens on every loopback address that +[the launcher contract](LAUNCHER_CONTRACT.md) names for `localhost`. After the +readiness check, the job log prints facts, not pass or fail: the `localhost` +lookup order, the addresses the gateway listens on, and what `GET /healthz` +returns at `localhost:8090`, `127.0.0.1:8090` and `[::1]:8090`. + +This runbook claims no local Intel Browser engine. See +[Browser support](BROWSER_SUPPORT.md). The job uploads no release tarball. + ## From Fresh Mac To Home Install the host tools first: diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index a576b012e..77f03f5ec 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -1643,6 +1643,18 @@ def test_restored_inputs_are_verified_before_use(self): with self.subTest(mutation=name), self.assertRaises(AssertionError): validate_restores_are_verified(changed) + def test_intel_gateway_smoke_probes_the_binary_it_built(self): + build, = [step for step in steps("gateway-smoke-macos-intel") + if step.startswith("name: build the gateway binary\n")] + probe, = [step for step in steps("gateway-smoke-macos-intel") + if "scripts/gateway-healthz-smoke.sh" in step] + self.assertLess(build.index('export CARGO_TARGET_DIR="$RUNNER_TEMP/gateway-intel-target"'), + build.index("cargo build --locked --release")) + self.assertIn('test -x "$CARGO_TARGET_DIR/release/elastos"', build) + self.assertIn('scripts/gateway-healthz-smoke.sh "$CARGO_TARGET_DIR/release/elastos"', probe) + # The probe keeps the contract address localhost:8090 (the script default). + self.assertNotIn("ELASTOS_GATEWAY_ADDR", probe) + def test_github_runners_and_check_names_stay_fixed(self): expected = { "source-gate": ("source-gate", "ubuntu-24.04"), @@ -1658,6 +1670,9 @@ def test_github_runners_and_check_names_stay_fixed(self): "source-home-linux": ("source-home-linux (${{ matrix.os }})", "${{ matrix.os }}"), "source-home-linux-arm64": ("source-home-linux (${{ matrix.check_name || matrix.os }})", "${{ matrix.os }}"), "source-home-macos": ("source-home-macos", "macos-14"), + # Intel gateway /healthz smoke. Signed x86_64-darwin admission is + # still the signer and installer gate in docs/MAC.md. + "gateway-smoke-macos-intel": ("gateway-smoke-macos-intel", "macos-15-intel"), "cancel-on-failure": ("cancel-on-failure", "ubuntu-24.04"), # Anders decides whether this becomes a required check (#320). "update-hop-linux": ("update-hop-linux", "ubuntu-24.04"), diff --git a/scripts/gateway-healthz-smoke-test.py b/scripts/gateway-healthz-smoke-test.py new file mode 100644 index 000000000..c2ef96a66 --- /dev/null +++ b/scripts/gateway-healthz-smoke-test.py @@ -0,0 +1,547 @@ +#!/usr/bin/env python3 +"""The gateway smoke accepts contract readiness and refuses every other case. + +Readiness is defined in docs/LAUNCHER_CONTRACT.md "Readiness and Home URL". +A stub stands in for the elastos binary so each refusal path runs on Linux and +macOS. The stub keeps host-process.lock where the real binary does (the dirs +crate data_dir): $XDG_DATA_HOME/elastos on Linux and +$HOME/Library/Application Support/elastos on macOS, where XDG_DATA_HOME is not +read. It asks `uname -s`, as the smoke does, so a test can run the macOS path +on Linux with a uname shim. +""" + +import os +import shutil +import signal +import socket +import stat +import subprocess +import tempfile +import textwrap +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SMOKE = ROOT / "scripts" / "gateway-healthz-smoke.sh" + +# No test may depend on how fast the stub starts. A slow python3 start (the +# macOS /usr/bin/python3 shim, a cold cache, a busy host) can take seconds. +# Tests that end on an event (an answer, a refusal, the stub's exit) give the +# smoke the largest deadline it accepts, so the deadline never ends them. +# Tests that need the deadline to pass first wait for the stub's own event +# before the first probe (see gated_curl_path). HANG_GUARD only stops a hung +# run; no passing run comes near it. +NO_DEADLINE = "9999" +HANG_GUARD = 600 + +STUB = textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json, os, signal, socket, socketserver, subprocess, sys, time + from http.server import BaseHTTPRequestHandler, HTTPServer + mode = os.environ.get("SMOKE_STUB", "ok") + lock_mode = os.environ.get("SMOKE_LOCK", "good") + args = sys.argv[1:] + if "SMOKE_ARGS" in os.environ: + with open(os.environ["SMOKE_ARGS"], "w", encoding="utf-8") as handle: + json.dump(args, handle) + def fork_sleeper(ignore_term=False): + child = os.fork() + if child == 0: + if ignore_term: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with open(os.environ["SMOKE_CHILD_PID"], "w", encoding="utf-8") as handle: + handle.write(str(os.getpid())) + time.sleep(60) + os._exit(0) + if mode == "exit": + sys.exit(1) + if mode == "exit-after-probe": + # Stay alive until the smoke has run curl once, then exit. The smoke + # must then report the exit; no deadline decides the result. + probed = os.environ["SMOKE_PROBED"] + give_up = time.monotonic() + 600 + while not os.path.exists(probed) and time.monotonic() < give_up: + time.sleep(0.05) + sys.exit(1) + if mode == "secret": + print("attach_secret=planted-secret", flush=True) + sys.exit(1) + if mode == "sleep": + time.sleep(30) + sys.exit(0) + if mode in ("fork", "ignore-term"): + fork_sleeper(ignore_term=(mode == "ignore-term")) + time.sleep(60) + sys.exit(0) + if args[:2] != ["gateway", "--addr"] or len(args) != 3: + sys.exit(2) + addr = args[2] + host, port = addr.rsplit(":", 1) + host = host.strip("[]") + system = subprocess.run(["uname", "-s"], capture_output=True, text=True).stdout.strip() + if system == "Darwin" and lock_mode != "xdg": + data = os.path.join(os.environ["HOME"], "Library", "Application Support", "elastos") + else: + data = os.path.join(os.environ["XDG_DATA_HOME"], "elastos") + os.makedirs(data, exist_ok=True) + record = {"pid": os.getpid(), "role": "gateway", "addr": addr} + if lock_mode == "other-pid": + record["pid"] = os.getpid() + 100000 + elif lock_mode == "bad-role": + record["role"] = "serve" + elif lock_mode == "bad-addr": + record["addr"] = "127.0.0.1:1" + if lock_mode == "garbage": + with open(os.path.join(data, "host-process.lock"), "w") as handle: + handle.write("not json") + elif lock_mode != "none": + with open(os.path.join(data, "host-process.lock"), "w") as handle: + json.dump(record, handle) + if mode == "ok-fork": + fork_sleeper() + not_ready = int(os.environ.get("SMOKE_503_COUNT", "0")) + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + global not_ready + status, body = 200, b"OK" + if not_ready > 0: + not_ready -= 1 + status, body = 503, b"" + elif mode == "newline": + body = b"OK\\n" + elif mode == "redirect": + status = 302 + elif mode == "no-content": + status = 204 + elif mode == "server-error": + status = 500 + elif mode == "other": + body = b"NO" + elif mode == "always-503": + status, body = 503, b"" + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + return + if host == "localhost": + host = os.environ.get("SMOKE_LOCALHOST_BIND", "127.0.0.1") + class Server(HTTPServer): + address_family = socket.AF_INET6 if ":" in host else socket.AF_INET + def server_bind(self): + # HTTPServer.server_bind also calls socket.getfqdn, a reverse + # lookup that can take seconds on macOS. The smoke does not need it. + socketserver.TCPServer.server_bind(self) + self.server_name, self.server_port = host, int(port) + server = Server((host, int(port)), Handler) + if "SMOKE_READY" in os.environ: + # The socket is bound and listening, so a probe now reaches the stub. + with open(os.environ["SMOKE_READY"], "w") as handle: + handle.write("ready") + server.serve_forever() + """ +) + + +def ipv6_loopback(): + try: + with socket.socket(socket.AF_INET6) as sock: + sock.bind(("::1", 0)) + return True + except OSError: + return False + + +def free_port(family=socket.AF_INET, host="127.0.0.1"): + with socket.socket(family) as sock: + sock.bind((host, 0)) + return sock.getsockname()[1] + + +def process_gone(pid): + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + return False + + +class OkHandler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"OK" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + return + + +class GatewayHealthzSmokeTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.binary = Path(self.tmp.name) / "elastos" + self.binary.write_text(STUB) + self.binary.chmod(self.binary.stat().st_mode | stat.S_IEXEC) + self.port = free_port() + self.child_pid = Path(self.tmp.name) / "child.pid" + + def env(self, stub, addr=None, deadline=NO_DEADLINE, **extra): + env = os.environ.copy() + env["SMOKE_STUB"] = stub + env["SMOKE_CHILD_PID"] = str(self.child_pid) + env["ELASTOS_GATEWAY_HEALTH_DEADLINE"] = deadline + env["ELASTOS_GATEWAY_ADDR"] = addr if addr is not None else f"127.0.0.1:{self.port}" + env.update(extra) + return env + + def run_smoke(self, stub, addr=None, deadline=NO_DEADLINE, **extra): + return subprocess.run( + ["bash", str(SMOKE), str(self.binary)], + env=self.env(stub, addr, deadline, **extra), + capture_output=True, + text=True, + timeout=HANG_GUARD, + ) + + def gated_curl_path(self, marker): + # A curl that waits until the stub has written to marker, then runs the + # real curl. The smoke checks its deadline only after a probe returns, + # so the first probe always sees the stub at that point, however slow + # the stub was to start. + real_curl = shutil.which("curl") + self.assertIsNotNone(real_curl, "this host has no curl") + shim = Path(self.tmp.name) / "gated-bin" + shim.mkdir() + curl = shim / "curl" + # The wait stops after HANG_GUARD seconds so a broken run cannot + # leave it behind. + curl.write_text('#!/bin/sh\n' + 'i=0\n' + f'while [ ! -s "{marker}" ] && [ "$i" -lt {HANG_GUARD * 20} ]; do\n' + ' sleep 0.05\n' + ' i=$((i + 1))\n' + 'done\n' + f'exec "{real_curl}" "$@"\n') + curl.chmod(0o755) + return f"{shim}{os.pathsep}{os.environ.get('PATH', '')}" + + def serve_ok(self, family, host, port): + class Server(HTTPServer): + address_family = family + + server = Server((host, port), OkHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + # Ready. + + def test_ok_with_owned_lock_is_ready(self): + result = self.run_smoke("ok") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"returned HTTP 200 and OK from gateway pid", result.stdout) + self.assertIn("host-process.lock names pid", result.stdout) + self.assertIn(f"listens on: 127.0.0.1:{self.port}", result.stdout) + self.assertRegex(result.stdout, rf"fact: GET http://127\.0\.0\.1:{self.port}/healthz: " + r"HTTP 200, body OK, from gateway pid \d+ on 127\.0\.0\.1:") + self.assertIn(f"fact: GET http://[::1]:{self.port}/healthz: no answer", result.stdout) + + def test_503_then_200_is_ready(self): + result = self.run_smoke("ok", SMOKE_503_COUNT="2") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_localhost_name_is_probed_as_given(self): + result = self.run_smoke("ok", addr=f"localhost:{self.port}") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"GET http://localhost:{self.port}/healthz", result.stdout) + self.assertIn("resolves localhost to", result.stdout) + + def no_network_path(self, log): + # A curl that records its arguments and never connects. The default + # address is the contract's localhost:8090, and this test must not + # depend on whatever listens on that port on the test host. + shim = Path(self.tmp.name) / "no-network-bin" + shim.mkdir() + curl = shim / "curl" + curl.write_text('#!/bin/sh\n' + f'printf \'%s\\n\' "$@" >>"{log}"\n' + 'printf 000\n' + 'exit 7\n') + curl.chmod(0o755) + return f"{shim}{os.pathsep}{os.environ.get('PATH', '')}" + + def test_default_is_the_contract_command(self): + # The stub writes its arguments first, stays alive until the smoke has + # probed once, then exits. The smoke ends on that exit, not on its + # deadline, so a slow stub start cannot change the result. Nothing + # listens or connects, so it passes whether port 8090 is free or not. + args = Path(self.tmp.name) / "args.json" + curl_log = Path(self.tmp.name) / "curl-args.txt" + env = self.env("exit-after-probe", SMOKE_ARGS=str(args), + SMOKE_PROBED=str(curl_log), PATH=self.no_network_path(curl_log)) + env.pop("ELASTOS_GATEWAY_ADDR") + result = subprocess.run(["bash", str(SMOKE), str(self.binary)], env=env, + capture_output=True, text=True, timeout=HANG_GUARD) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(args.read_text(), '["gateway", "--addr", "localhost:8090"]') + self.assertIn("gateway exited before GET http://localhost:8090/healthz was ready", + result.stderr) + urls = [line for line in curl_log.read_text().splitlines() + if line.startswith("http")] + self.assertTrue(urls, curl_log.read_text()) + self.assertEqual(set(urls), {"http://localhost:8090/healthz"}) + + @unittest.skipUnless(ipv6_loopback(), "this host has no IPv6 loopback") + def test_ipv6_loopback_ok(self): + port = free_port(socket.AF_INET6, "::1") + result = self.run_smoke("ok", addr=f"[::1]:{port}") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn(f"on [::1]:{port}", result.stdout) + + @unittest.skipUnless(ipv6_loopback(), "this host has no IPv6 loopback") + def test_facts_name_another_process_on_the_other_family(self): + # The macOS case: localhost gives ::1 first, so the gateway holds only + # [::1]. Another process on 127.0.0.1 then answers the IPv4 URL. + port = free_port() + self.serve_ok(socket.AF_INET, "127.0.0.1", port) + ready = Path(self.tmp.name) / "stub-ready" + result = self.run_smoke("ok", addr=f"localhost:{port}", SMOKE_LOCALHOST_BIND="::1", + SMOKE_READY=str(ready), PATH=self.gated_curl_path(ready)) + if f"answered on 127.0.0.1:{port}" in result.stderr: + self.skipTest("curl on this host chose 127.0.0.1 for localhost") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(f"listens on: [::1]:{port}", result.stdout) + self.assertIn(f"fact: GET http://127.0.0.1:{port}/healthz: HTTP 200, body OK, " + f"from another process on 127.0.0.1:{port}", result.stdout) + + # Not ready. + + def test_503_until_deadline_is_not_ready(self): + # The first probe waits until the stub listens, so it gets a 503 + # before the 1 s deadline can pass. + ready = Path(self.tmp.name) / "stub-ready" + result = self.run_smoke("always-503", deadline="1", SMOKE_READY=str(ready), + PATH=self.gated_curl_path(ready)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("last: HTTP 503", result.stderr) + + def test_other_status_codes_are_refused(self): + for stub in ("redirect", "no-content", "server-error"): + with self.subTest(stub=stub): + result = self.run_smoke(stub) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Only 200 (ready) or 503 (not ready)", result.stderr) + + def test_other_body_is_refused(self): + for stub in ("other", "newline"): + with self.subTest(stub=stub): + result = self.run_smoke(stub) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("not the two bytes OK", result.stderr) + + def test_lock_that_is_not_this_gateway_is_refused(self): + cases = { + "other-pid": "not gateway pid", + "bad-role": "not gateway", + "bad-addr": "addr is '127.0.0.1:1'", + "none": "missing or is not JSON", + "garbage": "missing or is not JSON", + } + for lock_mode, message in cases.items(): + with self.subTest(lock=lock_mode): + result = self.run_smoke("ok", SMOKE_LOCK=lock_mode) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("host-process.lock", result.stderr) + self.assertIn(message, result.stderr) + self.assertNotIn("ready:", result.stdout) + + # macOS keeps the lock under Library/Application Support. A uname shim runs + # that path on Linux too, so Linux CI covers it. + + def darwin_path(self): + real_uname = shutil.which("uname") + self.assertIsNotNone(real_uname, "this host has no uname") + shim = Path(self.tmp.name) / "darwin-bin" + shim.mkdir() + uname = shim / "uname" + uname.write_text('#!/bin/sh\n' + 'if [ "$1" = "-s" ]; then echo Darwin; exit 0; fi\n' + f'exec "{real_uname}" "$@"\n') + uname.chmod(0o755) + return f"{shim}{os.pathsep}{os.environ.get('PATH', '')}" + + def test_macos_lock_is_read_from_application_support(self): + result = self.run_smoke("ok", PATH=self.darwin_path()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("host-process.lock names pid", result.stdout) + + def test_macos_lock_under_xdg_is_refused(self): + # The real binary does not read XDG_DATA_HOME on macOS, so a lock + # there belongs to nothing this smoke started. + result = self.run_smoke("ok", SMOKE_LOCK="xdg", PATH=self.darwin_path()) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("host-process.lock is missing or is not JSON", result.stderr) + self.assertNotIn("ready:", result.stdout) + + def test_ok_from_another_process_on_the_port_is_refused(self): + self.serve_ok(socket.AF_INET, "127.0.0.1", self.port) + result = self.run_smoke("sleep") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("not a listen socket of gateway pid", result.stderr) + + @unittest.skipUnless(ipv6_loopback(), "this host has no IPv6 loopback") + def test_localhost_answer_on_another_family_is_refused(self): + # The gateway holds 127.0.0.1 and another process holds [::1] on the + # same port. curl tries ::1 first for localhost and gets that OK. + port = free_port() + try: + self.serve_ok(socket.AF_INET6, "::1", port) + except OSError: + self.skipTest("[::1] port is not free") + result = self.run_smoke("ok", addr=f"localhost:{port}", + SMOKE_LOCALHOST_BIND="127.0.0.1") + if "on 127.0.0.1:" in result.stdout: + self.skipTest("curl on this host chose 127.0.0.1 for localhost") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn(f"answered on [::1]:{port}", result.stderr) + + def test_early_exit_is_not_ready(self): + result = self.run_smoke("exit") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("exited before", result.stderr) + + def test_no_answer_until_deadline_is_not_ready(self): + result = self.run_smoke("sleep", deadline="1") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("last: no answer", result.stderr) + + def test_proxy_cannot_supply_the_ok_body(self): + proxy = socket.socket() + proxy.bind(("127.0.0.1", 0)) + proxy.listen(1) + self.addCleanup(proxy.close) + + def serve_proxy(): + try: + connection, _ = proxy.accept() + except OSError: + return + connection.sendall(b"HTTP/1.0 200 OK\r\nContent-Length: 2\r\n\r\nOK") + connection.close() + + threading.Thread(target=serve_proxy, daemon=True).start() + proxy_url = f"http://127.0.0.1:{proxy.getsockname()[1]}" + result = self.run_smoke("other", http_proxy=proxy_url, HTTP_PROXY=proxy_url, + all_proxy=proxy_url, ALL_PROXY=proxy_url) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("not the two bytes OK", result.stderr) + + # Input checks fail closed before the gateway starts. + + def test_deadline_must_be_a_whole_number_of_seconds(self): + marker = Path(self.tmp.name) / "pwned" + for value in ("abc", "-5", "", "0", "1.5", " 5", "10000", "07", + "x[$(touch %s)]" % marker): + with self.subTest(value=value): + result = self.run_smoke("ok", deadline=value) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("ELASTOS_GATEWAY_HEALTH_DEADLINE", result.stderr) + self.assertFalse(marker.exists()) + + def test_address_must_be_loopback_host_and_port(self): + for addr in ("0.0.0.0:9", "example.com:80", "127.0.0.1:1@example.com:80", + "localhost", "localhost:0", "localhost:65536", "localhost:08090", + "[::]:8090", "127.0.0.2:8090", ""): + with self.subTest(addr=addr): + result = self.run_smoke("ok", addr=addr) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("loopback host:port", result.stderr) + + def test_missing_binary_exits_one(self): + missing = Path(self.tmp.name) / "missing" + result = subprocess.run(["bash", str(SMOKE), str(missing)], + capture_output=True, text=True, timeout=HANG_GUARD) + self.assertEqual(result.returncode, 1, result.stderr) + + # Cleanup stops the whole process group on every exit path. + + def assert_child_stopped(self, result): + self.assertTrue(self.child_pid.exists(), result.stderr) + self.assertTrue(process_gone(int(self.child_pid.read_text())), + "a gateway group member survived the smoke") + + def test_group_is_stopped_after_success(self): + result = self.run_smoke("ok-fork") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_child_stopped(result) + + # These two need the deadline to pass after the stub has forked. The first + # probe waits until the child has written its pid, so the 1 s deadline can + # only pass after the child exists, however slow the host is. + + def test_group_is_stopped_after_timeout(self): + result = self.run_smoke("fork", deadline="1", + PATH=self.gated_curl_path(self.child_pid)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("was not ready within 1s", result.stderr) + self.assert_child_stopped(result) + + def test_child_ignoring_term_is_killed(self): + result = self.run_smoke("ignore-term", deadline="1", + PATH=self.gated_curl_path(self.child_pid)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("was not ready within 1s", result.stderr) + self.assert_child_stopped(result) + + def test_group_is_stopped_when_the_smoke_is_interrupted(self): + proc = subprocess.Popen(["bash", str(SMOKE), str(self.binary)], + env=self.env("fork"), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + # Signal only after the child exists, however long that takes. + guard = time.monotonic() + HANG_GUARD + while not (self.child_pid.exists() and self.child_pid.read_text()): + self.assertIsNone(proc.poll(), "the smoke ended before the stub forked") + self.assertLess(time.monotonic(), guard, "the stub never forked") + time.sleep(0.05) + proc.send_signal(signal.SIGTERM) + _, stderr = proc.communicate(timeout=HANG_GUARD) + finally: + if proc.poll() is None: + proc.kill() + self.assertEqual(proc.returncode, 143, stderr) + self.assertTrue(process_gone(int(self.child_pid.read_text()))) + + def test_log_is_kept_and_not_echoed(self): + log = Path(self.tmp.name) / "gateway.log" + result = self.run_smoke("secret", ELASTOS_GATEWAY_LOG=str(log)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertNotIn("planted-secret", result.stdout) + self.assertNotIn("planted-secret", result.stderr) + self.assertIn("planted-secret", log.read_text()) + + # The smoke follows the contract text, which is the one source of truth. + + def test_contract_names_the_command_and_url_this_smoke_uses(self): + contract = (ROOT / "docs/LAUNCHER_CONTRACT.md").read_text() + smoke = SMOKE.read_text() + self.assertIn("elastos gateway --addr localhost:8090", contract) + self.assertIn("GET http://localhost:8090/healthz", contract) + self.assertIn("host-process.lock", contract) + self.assertIn("ELASTOS_GATEWAY_ADDR-localhost:8090", smoke) + mac = (ROOT / "docs/MAC.md").read_text() + install = (ROOT / "scripts/install.sh").read_text() + self.assertIn('die "macOS release lookup requires Apple silicon"', install) + self.assertIn("macOS release lookup requires Apple silicon", mac) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/gateway-healthz-smoke.sh b/scripts/gateway-healthz-smoke.sh new file mode 100755 index 000000000..b403e6961 --- /dev/null +++ b/scripts/gateway-healthz-smoke.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Start one elastos gateway and require the readiness that +# docs/LAUNCHER_CONTRACT.md "Readiness and Home URL" defines: +# GET http://localhost:8090/healthz returns HTTP 200 and the two bytes OK +# while this gateway pid holds {data}/host-process.lock with role gateway +# or gateway-public. +# By default the gateway gets the contract command +# `elastos gateway --addr localhost:8090` and the probe uses the contract URL. +# The probe passes only when: +# - curl gets HTTP 200 and the two bytes OK (no proxy, no redirect), +# - the address curl connected to is a listen socket of the gateway pid, +# - host-process.lock names that pid, a gateway role and the same addr. +# The script then prints facts, not pass or fail: the localhost lookup order, +# the addresses the gateway listens on, and what GET /healthz returns at +# localhost, 127.0.0.1 and [::1] on the same port. +# An empty Home can pass. This does not prove providers, capsules, sign-in, +# or a signed install. +set -euo pipefail + +usage() { + echo "usage: gateway-healthz-smoke.sh /path/to/elastos" >&2 + exit 2 +} +[[ $# -eq 1 && -n "${1:-}" ]] || usage +binary=$1 + +deadline=${ELASTOS_GATEWAY_HEALTH_DEADLINE-120} +if [[ ! "$deadline" =~ ^[1-9][0-9]{0,3}$ ]]; then + echo "ELASTOS_GATEWAY_HEALTH_DEADLINE must be a whole number of seconds from 1 to 9999" >&2 + exit 2 +fi + +addr=${ELASTOS_GATEWAY_ADDR-localhost:8090} +# Loopback only. localhost is the contract name; the numeric forms are for +# tests that need a free port. +if [[ ! "$addr" =~ ^(localhost|127\.0\.0\.1|\[::1\]):([1-9][0-9]{0,4})$ ]] \ + || (( BASH_REMATCH[2] > 65535 )); then + echo "gateway smoke needs a loopback host:port (localhost, 127.0.0.1 or [::1]): $addr" >&2 + exit 2 +fi +port=${BASH_REMATCH[2]} + +for tool in curl lsof python3; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "gateway smoke needs $tool" >&2 + exit 2 + fi +done +case "$(uname -s)" in + Darwin) data_subdir="Library/Application Support/elastos" ;; + Linux) data_subdir=".local/share/elastos" ;; + *) + echo "gateway smoke does not know the data directory on $(uname -s)" >&2 + exit 2 + ;; +esac +if [[ ! -x "$binary" ]]; then + echo "elastos binary is missing or is not executable: $binary" >&2 + exit 1 +fi + +root=$(mktemp -d) +pid= +log="$root/gateway.log" + +# kill -0 succeeds for the group while any member is alive, and for the pid +# until it exits. Check both, in case the group did not form. +gateway_alive() { + kill -0 -- "-$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null +} +signal_gateway() { + kill "-$1" -- "-$pid" 2>/dev/null || true + kill "-$1" "$pid" 2>/dev/null || true +} +wait_gone() { + local i + for i in $(seq 1 25); do + gateway_alive || return 0 + sleep 0.2 + done + return 1 +} +stop_gateway() { + [[ -n "$pid" ]] || return 0 + signal_gateway TERM + wait_gone && return 0 + signal_gateway KILL + wait_gone && return 0 + echo "gateway process group $pid is still running after SIGKILL" >&2 + return 1 +} +cleanup() { + local status=$1 + trap - EXIT INT TERM HUP + stop_gateway || status=1 + if [[ -n "${ELASTOS_GATEWAY_LOG:-}" && -f "$log" ]]; then + cp "$log" "$ELASTOS_GATEWAY_LOG" 2>/dev/null || true + fi + rm -rf "$root" + exit "$status" +} +trap 'cleanup $?' EXIT +trap 'cleanup 130' INT +trap 'cleanup 143' TERM +trap 'cleanup 129' HUP + +export HOME="$root/home" +export XDG_DATA_HOME="$HOME/.local/share" +mkdir -p "$HOME" "$XDG_DATA_HOME" +lock="$HOME/$data_subdir/host-process.lock" +expect="$root/healthz-expect" +body="$root/healthz-body" +printf 'OK' >"$expect" + +# The gateway leads a new session, so its pid is also its process group id. +python3 -c 'import os, sys; os.setsid(); os.execv(sys.argv[1], sys.argv[1:])' \ + "$binary" gateway --addr "$addr" >"$log" 2>&1 & +pid=$! + +url="http://${addr}/healthz" +end=$((SECONDS + deadline)) +last="no answer" +while true; do + if ! kill -0 "$pid" 2>/dev/null; then + echo "gateway exited before GET $url was ready" >&2 + exit 1 + fi + : >"$body" + answer=$(curl --noproxy '*' --proto =http -sS --max-time 2 -o "$body" \ + -w '%{http_code} %{remote_ip}' "$url" 2>/dev/null || true) + code=${answer%% *} + remote_ip=${answer#* } + case "$code" in + 200) break ;; + 503) last="HTTP 503" ;; + "" | 000) last="no answer" ;; + *) + echo "GET $url returned HTTP $code. Only 200 (ready) or 503 (not ready) is allowed." >&2 + exit 1 + ;; + esac + if (( SECONDS >= end )); then + echo "GET $url was not ready within ${deadline}s (last: $last)" >&2 + exit 1 + fi + sleep 1 +done + +if ! cmp -s "$expect" "$body"; then + echo "GET $url returned HTTP 200, but the body is not the two bytes OK" >&2 + exit 1 +fi +case "$remote_ip" in + 127.0.0.1) socket="127.0.0.1" ;; + ::1) socket="[::1]" ;; + *) + echo "GET $url connected to '$remote_ip', which is not a loopback address" >&2 + exit 1 + ;; +esac +if ! lsof -nP -a -p "$pid" -iTCP@"$socket":"$port" -sTCP:LISTEN >/dev/null 2>&1; then + echo "GET $url was answered on $socket:$port, which is not a listen socket of gateway pid $pid" >&2 + exit 1 +fi +if ! python3 - "$lock" "$pid" "$addr" <<'PY' +import json, sys +path, pid, addr = sys.argv[1], int(sys.argv[2]), sys.argv[3] +try: + with open(path, "rb") as handle: + meta = json.load(handle) +except (OSError, ValueError) as error: + sys.exit(f"host-process.lock is missing or is not JSON: {error}") +if not isinstance(meta, dict): + sys.exit("host-process.lock is not a JSON object") +if meta.get("pid") != pid: + sys.exit(f"host-process.lock names pid {meta.get('pid')!r}, not gateway pid {pid}") +if meta.get("role") not in ("gateway", "gateway-public"): + sys.exit(f"host-process.lock role is {meta.get('role')!r}, not gateway") +if meta.get("addr") != addr: + sys.exit(f"host-process.lock addr is {meta.get('addr')!r}, not {addr}") +PY +then + exit 1 +fi + +echo "ready: GET $url returned HTTP 200 and OK from gateway pid $pid on $socket:$port" +echo "ready: host-process.lock names pid $pid with role gateway and addr $addr" + +# Facts only, not pass or fail: what each loopback spelling of this port +# reaches. A client that uses a different spelling than the gateway's --addr +# can miss the gateway or reach another process. +listening=$(lsof -nP -a -p "$pid" -iTCP:"$port" -sTCP:LISTEN 2>/dev/null \ + | awk 'NR > 1 { print $(NF - 1) }' | sort -u | paste -sd ' ' - || true) +echo "fact: gateway pid $pid listens on: ${listening:-nothing}" +resolved=$(python3 -c 'import socket, sys +seen = [] +for info in socket.getaddrinfo("localhost", int(sys.argv[1]), type=socket.SOCK_STREAM): + if info[4][0] not in seen: + seen.append(info[4][0]) +print(" ".join(seen))' "$port" 2>/dev/null || echo "lookup failed") +echo "fact: this host resolves localhost to (in order): $resolved" +for probe in "localhost:$port" "127.0.0.1:$port" "[::1]:$port"; do + : >"$body" + answer=$(curl --noproxy '*' --proto =http -sS --max-time 2 -o "$body" \ + -w '%{http_code} %{remote_ip}' "http://$probe/healthz" 2>/dev/null || true) + code=${answer%% *} + ip=${answer#* } + if [[ -z "$code" || "$code" == 000 ]]; then + echo "fact: GET http://$probe/healthz: no answer" + continue + fi + case "$ip" in ::1) ip="[::1]" ;; esac + owner="another process" + if lsof -nP -a -p "$pid" -iTCP@"$ip":"$port" -sTCP:LISTEN >/dev/null 2>&1; then + owner="gateway pid $pid" + fi + shown="not OK" + cmp -s "$expect" "$body" && shown="OK" + echo "fact: GET http://$probe/healthz: HTTP $code, body $shown, from $owner on $ip:$port" +done diff --git a/scripts/test-source-gate.sh b/scripts/test-source-gate.sh index d1998f6e5..171d2a5ea 100755 --- a/scripts/test-source-gate.sh +++ b/scripts/test-source-gate.sh @@ -34,6 +34,7 @@ release() { python3 scripts/prepare-release-platform-test.py python3 scripts/media-tools-build-test.py python3 scripts/verify-linux-arm64-package-test.py + python3 scripts/gateway-healthz-smoke-test.py python3 scripts/verify-linux-engine-test.py python3 scripts/update-hop-compare-test.py python3 .github/scripts/release-mac-capacity-test.py