diff --git a/README.md b/README.md index b7a535d..4e599ca 100644 --- a/README.md +++ b/README.md @@ -1104,6 +1104,7 @@ AirOpsEnv v1 is a model of a day, not an airline's operations system. What it le | `scripts/build_stage.py`, `scripts/prebake_traffic.py`, `scripts/build_twin.py`, `scripts/prebake_radar.py` | the viewer's airport stages, traffic, hub twins and radar frames, into `cache/` | | `scripts/record.py` | films an episode in headless Chrome | | `scripts/hub_dataset.py`, `hub/` | the Hugging Face dataset and the static Space | +| `live/` | optional, not needed to use the plugin: streaming a model's days live to X or any RTMP target ([live/README.md](live/README.md)) | ```bash uv sync --extra dev --extra viewer --extra build diff --git a/live/.gitignore b/live/.gitignore new file mode 100644 index 0000000..55c66a1 --- /dev/null +++ b/live/.gitignore @@ -0,0 +1,2 @@ +live.env +runner/fetched/ diff --git a/live/CONTRACT.md b/live/CONTRACT.md new file mode 100644 index 0000000..b9a00e7 --- /dev/null +++ b/live/CONTRACT.md @@ -0,0 +1,192 @@ + +# AirOpsEnv live: the contracts + +What the runner, the live server, the viewer's live mode, the streamer and the sound promise each other. The runbook +is [README.md](README.md). + +## current.json (runner → server) +Written atomically (tmp + rename) by `scripts/live_runner.py` on every change and at least every 10 s while a day runs. +Default path `results/runs//current.json` in the plugin checkout; on the on-air box `/home/ubuntu/live/current.json` +(`--current`). +``` +{ + "instance_id": str|null, the running day's task instance (agent-env's id, "-xxxxxxxx"), set within seconds + of the launch, before the env is up; the finished day's while status is done/intermission + "run_id": "b6-20240803-0400-joint-a1", -joint-a: the key of days/.json and transcripts/-.json + "date": "2024-08-03", "hub": "JFK", "task_id": "b6-20240803-0400", "model": "openai/gpt-6-astra", "mode": "joint", + "started_utc": "2026-10-23T16:00:04Z", + "status": "starting" (launched, env not up yet) | "playing" (env deployed: env.base answers data/get) + | "done" (day ended; result + day_file set; held card_seconds) | "intermission" (waiting for next.starts_utc), + "next": {"date", "hub", "task_id", "starts_utc"} | null (null: no further day; see final), + "history": [{"date", "hub", "task_id", "run_id", "instance_id", "outcome": "scored"|"unscored"|"skipped", + "reward", "reward_L", "cost_usd", "wall_seconds", "desks", "attempts"}], finished days in play order + "attempt": 1|2, "day": n (1-based index on the stream), + "env": {"base": "http://localhost:PORT", "mcp_url": ".../mcp", "gateway_url": ...} | null data/get at env.base + (the verifier's own rule: mcp_url without /mcp); null until deployed. The env is torn + down seconds after the grade, so read the final record from day_file. + "day_file": absolute path of days/.json (the env's final data/get, graded) once status is done, else null + "result": history's entry for this day once done, else null + "spent_usd": finished attempts' model spend, "live_usd": the running day's estimate (sum of each desk's last + "request N: … $X spent" log line), "desks": {"crew": $, "customer": $, "ops": $}, "budget_usd": 430 + "pilot": "occ3-live-astra", "policy": "llm"|"script", "seq": n (+1 per write), "updated_utc", + "final": false|true, "reason": why the runner stopped (budget, window, dates, stopped) once final +} +``` +Before the first day: status "intermission", instance_id/date null, next set. The live server can attach as soon +as `env` is non-null (status "playing"). + +## Live server → viewer (`scripts/live_server.py`) + +### Endpoints (one origin, default port 8765 on the box that plays the day, which has current.json and the env; CORS `*`) +| Path | What | Caching | +|---|---|---| +| `GET /live/state.json` | tiny: `{"seq", "live": {...the live object...}}` | `no-store` | +| `GET /live/episode.json` | the live Episode JSON (below). gzip when the request sends `Accept-Encoding: gzip`; `ETag: ""`, answers `304` to `If-None-Match` | `no-store` | +| `GET /live/.stage-.json.gz` | the traffic sidecar the episode names in `stage.traffic` (a new `` per rebuild; the last 8 stay servable) | immutable | +| `GET /live/.json` (+ its `.stage.json.gz`) | each finished day's final episode (`live.status` "done"), kept beside the live one so its `../stage`, `../radar` URLs resolve, for the between-days replay | immutable | +| `GET /live/index.json` | the finished days as a viewer index (`export.entry` rows + `"live": true`, `url` relative to it) | `no-store` | +| `GET /stage/…`, `/radar/…`, `/twin/…` | the static assets: the episode's `../stage/JFK/stage.json.gz`, `../radar//.png` resolve here from `/live/` | long | +| `GET /web/…` | the viewer folder (`web/` of the branch the server runs from, or `--web DIR`) | short | + +Open the viewer at `http://HOST:8765/web/index.html?live=http://HOST:8765` (the viewer polls `/live/state.json` +and `/live/episode.json`; `?live=..` relative to the page works too). + +### Polling and sequence numbers +- Poll `/live/state.json` every 2 s. Fetch `/live/episode.json` only when `state.seq` differs from the episode you hold + (its `live.seq`). `seq` is one counter for the whole server life: it never goes back, across days and restarts of a + day (a server restart starts it at the wall-clock second, still higher). +- The day changed when `live.run_id` or `live.instance_id` changes (a retry is a new `run_id`, `...-a2`): drop the + old timeline. +- The episode is rebuilt (1) cheaply on every poll of the env that brought new calls (the room: `steps`, `occ.turns`, + `occ.requests`, `occ.decisions`, `occ.notes`, `occ.pages`, `live.desks`, `live.feed`), and (2) with the projection + (`executed`, `cost`, `passengers`, `occ.crew`, `occ.units`/`reserves`/`hotels`/`deadheads`, the open watch's + `projected`, a new sidecar `stage.traffic`) at each watch that opens and after each committed decision (at most one + rebuild per 15 s). The server polls the env's data/get every 3 s. + +### The live object (`episode.live`, also all of `/live/state.json`) +``` +"live": { + "status": "starting" current.json starting: the day's world, no env yet; the episode is the 04:00 projection + | "playing" the env answers data/get and the day runs + | "ending" the runner says done (or moved on) and the server is still reading the day: behind, or + the env gone before its last record was read (it reads the runner's day_file) + | "done" the grade is in: reward, cost and the record are final (exactly the recorded episode) + | "intermission" current.json intermission: the finished day's episode stays, `next` says what comes + | "stale", data/get has failed for 30 s; the last good episode stays (the field `error` says why) + "seq": 1729700412, + "run_id": "b6-20260715-0400-joint-a1", "task_id": "b6-20260715-0400", "date": "2026-07-15", "hub": "BOS", + "instance_id": "@local/~/…/b6-20260715-0400-joint-xxxxxxxx", "model": "openai/gpt-6-astra", + "model_label": "GPT-6 Astra", "mode": "joint", "pilot": "occ3-live-astra", + "watch": 7, "watches": 37, open watch index (0-based) and the day's watch slots + "t": 660, the open watch's minute (local minutes since midnight, as task.start) + "decided_until": 660, everything strictly before this minute is final; play up to it, never past it. + = t while playing; the day's last movement (max executed dep/arr) once done + "turn": {"n": 31, "desk": "customer", "kind": "customer", "calls": 4} | null, the turn in progress + "desks": [{"role": "crew"|"customer"|"ops"|"one", "state": "turn"|"waiting"|"off", "off": null|"done"|"stand_down"|…, + "last": {"seq": 412, "tool": "crew_confirm", "text": "assign P-A32X-0391 to B6 1381", "t": 660} | null, + "at": "2026-10-23T16:41:03Z"}], at: wall clock the server first saw `last` + "feed": [{"seq": 412, "at": "2026-10-23T16:41:03Z", "watch": 7, "t": 660, "desk": "crew", + "kind": "request"|"reply"|"decision"|"note"|"notice"|"off", "text": "…one line, flight ids as numbers…", + "request": "R14"|null, "flight": 154|null}], the last 60 room events, oldest first + "projected": {"total": 550467.2, "delay": …, "cancel": …, "notice": …, "aircraft": …, "lodging": …, "fare": …, + "deadhead": …, "hotel": …, "end": …}, the projection's cost now (moves with each decision) + "next_watch_at": "2026-10-23T16:42:30Z" | null, when the next watch should open: the open watch's wall start plus the + mean wall seconds of the watches so far (null before the second) + "watch_wall_s": 92.4, mean wall seconds per closed watch + "started_utc": "…", "updated_utc": "…", "wall_seconds": 655.0, + "spent_usd": 3.12, current.json live_usd: the running day's model spend so far (null if none) + "total_usd": 48.9, "budget_usd": 430, spent_usd of the finished days plus the running day's; the runner's budget + "result": null | {"reward", "reward_L", "cost", "best", "bound", "B0", "end_reason", "audit_ok", "cost_usd", + "wall_seconds"}, once done (cost_usd and wall_seconds from current.json's result) + "exact": true, "mismatches": 0, the server's replay against the env's result digests (false: a bug, the + episode still follows the env's calls) + "lag": 0, watches the server's replay is behind the env (it catches up two at a time, + publishing each; a server started mid-day shows the day from 04:00 on) + "next": current.json next, "history": current.json history, + "error": null | "…" +} +``` + +### The episode while the day runs (same schema as recorded v3 episodes, `web/episode.schema.json`, plus `live`) +- Present and final from the start: `version`, `task`, `model`, `model_label`, `run` (`"-"`, e.g. + `occ3-live-astra-joint`), `schedule`, `reference` (`anchor`, `best`, `bound`, `B0`, `gap_pct`; `flat` is the + projection's), `radar`, `twin`, `sources`, `caps: {}`, `nominal: {}`, `ferries: []`, `frozen: []`. +- `reward: null`, `end_reason: null`, `audit: null`, `occ.grade: null` until done. (The schema has `reward` a number + and `occ.grade` an object: the viewer's live mode must accept null for both; the tests validate the live episode + against the schema with those two made nullable, and nothing else changed.) +- `cost`: the projection's cost now, with the recorded keys (`total`, `pax_delay`, `cancel`, `delay`, `notice`, + `aircraft`, `lodging`, `fare`, `deadhead`, `hotel`, `end`; as recorded, `cancel` is the cancel part alone and + `notice` the late-notice part). Not the grade: the grade is on the truth. Same numbers as `live.projected`. +- `executed`: every JetBlue flight on the projection now (the plan so far on the truth up to `decided_until`, on the + day as known after it): rows before `decided_until` are what happened; later ones are the forecast and move. +- `watches`: one per watch opened so far, 0..`live.watch`. Closed ones exactly as recorded (`t`, `news`, `changes`, + `est_cost`, `projected` as the watch left it). The open one: `t`, `news`, `changes` so far, `projected` = now + (moves), no `est_cost`. +- `steps`: every call so far (as recorded: `t`, `tool`, `args`, `desk`, `watch`, `turn`). +- `news`: FAA advisories and the hub's METARs through `live.t` (as recorded, cut at the open watch). +- `occ`: as recorded (`mode`, `label`, `desks`, `turns`, `requests` with `shown`/`answered`, `decisions`, `notes`, + `pages`, `absent`, `refs`) so far, including the open watch's turns (their `shown` minutes are after `t`: held back + by `decided_until`, and a turn's window may still narrow until its watch closes). `crew`, `units`, `reserves`, + `hotels`, `deadheads` are the projection's. +- `passengers`: the passengers on the projection now (`flights`, `connections`, `events`; a flight's `hold` against + the projection without the customer desk's holds). +- `stage`: `{"hub", "url": "../stage//stage.json.gz", "traffic": ".stage-.json.gz"}`: the static part + (span start, the other airlines, runways from the METARs so far) stays; JetBlue's aircraft, crews and passengers + follow the projection. `` = `..`. +- `record`: `{"folder": pilot, "run": "-", "attempt", "cost_usd", "wall_seconds", "desks"}` from + current.json (`result`; null until the runner has them). + +### Once done (`live.status` "done", then "intermission") +The episode is the recorded one, built by the same code as `occ_viewer.py` from the day the env graded (our replay, +checked against the env's digest), plus `live`: `reward`, `occ.grade` (with `reward_L`), `cost` (the grade's), +`executed` (the truth), `end_reason`, `audit`, `passengers`, `stage.traffic` = `.stage.json.gz`, the final +`decided_until`, and `live.result`. It is also kept at `/live/.json` and listed in `/live/index.json`. + +### Where data/get comes from (for anyone attaching by hand) +The task instance store has the running day's env once `deploy` is done: `with namespace_routing(): +task_instances("@local/~//-joint", limit=1)[0].context["deployed_envs"]` → the entry with +`env_id == "airops-occ3"`, its `mcp_url` minus `/mcp` is the base; `POST /agentenv` +`{"jsonrpc":"2.0","id":1,"method":"data/get","params":{}}` → `result.parts[0].data` (the verifier's call; the gateway +forwards it to the env, whose handler runs on its event loop, so a snapshot is consistent). `live_server.py attach +BUNDLE TASK --current FILE` does this for a plain `agent-env run`. The server polls every 3 s (60 s timeout); a call +takes ~1-2 s on a loaded laptop and never changes the day. + +## Stream → viewer and server (`live/stream/`) +- **The page:** Google Chrome 156, kiosk, 1920x1080 at device pixel ratio 1, on an NVIDIA L4 (WebGL renderer + `ANGLE (NVIDIA Corporation, NVIDIA L4/PCIe/SSE2, OpenGL 4.5.0)`), one URL for the whole stream (`AIROPS_URL`), e.g. + `http://127.0.0.1:8765/web/index.html?live=http://127.0.0.1:8765&stream`. Flags that matter to the page: + `--autoplay-policy=no-user-gesture-required` (an AudioContext starts without a click), + `--disable-background-timer-throttling --disable-renderer-backgrounding`. The page's own reloads are fine. +- **Sound:** everything the page plays goes to PulseAudio's default sink and is encoded as is: AAC-LC 128k, 44.1 kHz, + stereo, over a quiet airport bed that only shows while the page is silent. No normalization or limiter after the page: + level the mix in the page. +- **Watchdog (what the page must do):** keep calling requestAnimationFrame (a page with no rAF callback for 20 s is + reloaded, and Chrome restarted if a reload doesn't help); answer DevTools within 45 s; a `.error` element makes it + reload (at most every 30 s); a lost WebGL context for 10 s makes it reload. The supervisor injects a rAF counter and a + `webglcontextlost` listener (`window.__airopsFrames`, `window.__airopsLost`) through DevTools; nothing else. +- **Frame rate:** the page renders at the display's 60 Hz unless `AIROPS_PAGE_FPS=30` (a rAF wrapper injected through + DevTools caps every rAF user at 30 fps; used on a 4-vCPU box). Capture is 30 fps. +- **Safe restarts:** with `AIROPS_SAFE_URL=http://127.0.0.1:8765/live/state.json`, Chrome is restarted for memory only + while `live.status` is `done` or `intermission` (PSS over 6 GB), or at once over 9 GB. +- **Network:** Chrome opens the live server on `127.0.0.1:8765`, and the viewer's map tiles from openfreemap.org and + the USGS basemap over the internet. + +## Sound (`web/audio.js`) +- **Loading:** `app.js` calls `mountSound(window.airops, timelineEl, {on, stream})` once `window.airops` is set. It isn't + called with `record=1` or `?sound=0`. Afterwards `window.airops.sound` is the `AirportSound`. +- **On and off:** `&sound=1` or `&stream` turns the sound on; otherwise a speaker button at the end of the timeline does. + With `&stream` there's no button. +- **Per frame (optional):** `window.airops.sound.update(state, camera)`. + - `state` needs `{main: "stage" | "room" | "map", view}`, as in `window.airops.state()`. + - `camera` is the THREE camera on screen. + - Without these calls, the sound runs its own rAF loop, reading `airops.state()` at 4 Hz. + - What it reads from the viewer: `stage` (its `fleet.live`, `T`, `rate`, `rig`, `env.weatherAt`, `lightning`, `gate()`) and `room.state()` (`turn`, `cards`). +- **Quiet bed:** `update({main: "map", view: "network"})` gives the quiet map bed, for example under a card. + `sound.setVolume(0..1)` trims the output. +- **Levels:** about -20 LUFS integrated; the limiter sits at -4.5 dBFS, because the stream's AAC encode adds about + 2 dB of true peak (-2.5 dBFS measured -0.6 dBTP on the stream); meters are in `sound.state().meter`. + - The leveller and limiter need an AudioWorklet, which needs a secure origin: https, localhost, or Chromium's + `--unsafely-treat-insecure-origin-as-secure=http://HOST:8765` (on one box the page is on 127.0.0.1, already secure). + - Elsewhere it falls back to the built-in compressors and a clipper at -2 dBFS. + - The AudioContext runs at 48 kHz with `latencyHint: "playback"` and starts without a click only with + `--autoplay-policy=no-user-gesture-required`. diff --git a/live/README.md b/live/README.md new file mode 100644 index 0000000..79c2d23 --- /dev/null +++ b/live/README.md @@ -0,0 +1,279 @@ +# AirOpsEnv live: a model's storm days, streamed + +Optional tooling for streaming a live run to X, or to any RTMP target: a model plays AirOpsEnv v3 storm days +(`airops-occ3`, three desks) back to back while the viewer shows each day as it is played, live in 3D, with airport +sound. Normal use of the plugin doesn't need any of it, and none of it ships in the wheel. + +The code it drives is in the plugin: `scripts/live_runner.py` (and its settings and day order, +`scripts/live_runner.toml`) plays the days, `scripts/live_server.py` (with `src/agentenv_airops/occ/onair.py`) serves +the day so far, and the viewer's live mode is `web/live.js`, `web/director.js` and `web/audio.js`. This folder sets +them up on a cloud box and streams them. [CONTRACT.md](CONTRACT.md) has what each part promises the next. + +## How it works + +One GPU box, a g6.4xlarge (NVIDIA L4, 16 vCPU, 64 GB) on the AWS Deep Learning Base OSS NVIDIA driver AMI (Ubuntu +22.04), runs everything as systemd units, reached only through SSM: + +``` + airops-live live_runner.py: the model's days back to back, each an `agent-env run` of its task (the env + airops-occ3 and three desk agents on local Docker); spend guard, slots and intermissions; + writes /home/ubuntu/live/current.json + airops-live-server live_server.py on 127.0.0.1:8765: polls the running env's data/get every 3 s, replays the + calls in its own day, serves the day so far as Episode JSON, the static assets and the viewer + airops-x Xorg on the L4, a virtual 1920x1080 screen; a slate on the root window + airops-capture streamer.py: PulseAudio (a 48 kHz null sink), Chrome 156 in kiosk on + http://127.0.0.1:8765/web/index.html?live=http://127.0.0.1:8765&stream, and ffmpeg: + x11grab + the sink -> h264_nvenc High 6000k CBR, 2 s GOP, no B-frames, AAC 128k + -> rtmp://127.0.0.1:1935/airops + airops-relay mediamtx: the local relay; records the stream in one-hour fMP4 segments + airops-push streamer.py: copies the relay to the target (X's server and key from SSM), reconnecting + airops-fresh fresh.py: each minute the stream's picture against Chrome's screenshot; restarts a stale encoder +``` + +Why one box: + +- **The sound.** `127.0.0.1` is a secure origin, so the page's AudioWorklet (its leveller and limiter) runs without + Chrome flags; on a second box's `http://` address the page falls back to the built-in compressors and a clipper. +- **No port between boxes.** The security group needs no inbound rule, and the server's port stays on loopback. +- **The GPU.** CPU rendering is unusable: on a c7i.8xlarge the page rendered at 0.4 fps with SwiftShader and 2 fps + with llvmpipe on 32 threads. The L4 renders it at 60 fps or more at 1920x1080, and NVENC encodes it. + +## Requirements + +- **AWS:** an account where you can launch g6 instances (16 vCPU of G-instance quota for a g6.4xlarge) as + SSM-managed instances: an instance profile whose role has `AmazonSSMManagedInstanceCore` (it also reads the + SecureString parameters below, encrypted with the default `aws/ssm` key), a subnet with a route to the internet + (apt, PyPI, GitHub, the model endpoint, X's ingest, the viewer's map tiles) and a security group (no inbound rule). +- **An S3 bucket** in the same region. Everything travels through it with presigned URLs, so the box needs no S3 + rights. +- **A LiteLLM-compatible endpoint** and a key that reaches the model (`openai/gpt-6-astra` by default). The scripted + test needs neither. +- **An RTMP target:** an X Media Studio RTMP source (server URL and stream key), or any `rtmp(s)://` URL. +- **On the laptop:** bash, the AWS CLI v2, python3 (with boto3, or [uv](https://docs.astral.sh/uv/)), git, curl, tar + and zstd, and this checkout synced with `uv sync --extra dev --extra viewer --extra build` to build the server's data. + +## Files + +Scripts at the top of each folder run on the laptop, from the checkout's root, and take the box's instance id. +Scripts under a `box/` folder run on the box: the laptop scripts send them through SSM, and +`aws/ssm.sh ID TIMEOUT SCRIPT [ARGS]` sends any of them. + +| Path | What | +|---|---| +| `live.env.example` | the settings: copy to `live/live.env` (git-ignored); `LIVE_ENV=FILE` names another | +| `aws/` | `common.sh` (the settings, `presign`, `ssm`), `launch-gpu.sh`, `launch.sh`, `teardown.sh`, `ssm.sh`, `put-secret.sh` | +| `runner/` | `setup.sh`, `start.sh`, `plan.sh`, `stop.sh`, `status.sh`, `fetch.sh`: the run box (`airops-live`) | +| `stream/` | `deploy.sh`, `start.sh`, `stop.sh`, `status.sh`, `save-recordings.sh`; `box/` is the streamer, installed in `/opt/airops-stream` with its settings in `/etc/airops-stream.env` | +| `onair/` | `prep.sh`, `upload-data.sh`, `deploy-server.sh`, `fresh.sh`, `health.sh`, `sync.sh`, `clip.sh`, `grab.sh`; `box/loudness.sh`, `box/metrics.sh`, `box/logs.sh` | +| `CONTRACT.md` | `current.json`, the live server's endpoints and live object, the stream's watchdog, the sound | + +## Setup + +**1. Settings and secrets.** Fill in the settings, then store the model's key in SSM. `put-secret.sh` reads a value +without echoing it, so a key never goes in a command you type, in a file or in a chat. + +```bash +cp live/live.env.example live/live.env # then edit it +live/aws/put-secret.sh litellm-key +``` + +**2. The server's data,** once, on the laptop: the raw public inputs in `cache/inputs/` (BTS, FAA and IEM, as +`scripts/build_pack.py` downloads them; the snippet fetches what the days' months still lack), the viewer's static +assets from the dataset, and each day's news, hub traffic and radar frames (`prebuild`: under a minute a day, most of +it the radar of a day the dataset has none for). The server prebuilds a day missing from them on the fly. + +```bash +python - <<'PY' +import re +from pathlib import Path +from agentenv_airops.build import pack +days = re.findall(r'"b6-(\d{4})(\d\d)\d\d-0400"', Path("scripts/live_runner.toml").read_text()) +pack.fetch_inputs(Path("cache/inputs"), sorted({f"{y}-{m}" for y, m in days}), ["JFK", "BOS"]) +PY +hf download earakely-scale/AirOpsEnv-AgentEnv --repo-type dataset --revision v0.5.0 \ + --include "viewer/stage/*" "viewer/radar/*" "viewer/twin/*" --local-dir cache/hub/dataset-v0.5.0 +python scripts/live_server.py prebuild $(grep -oE '"b6-[0-9]{8}-0400"' scripts/live_runner.toml | tr -d '"') \ + --live cache/live/prebuilt --inputs cache/inputs --assets cache/hub/dataset-v0.5.0/viewer +live/onair/upload-data.sh cache/live/prebuilt cache/hub/dataset-v0.5.0/viewer +``` + +**3. The box,** about 20 minutes. `launch-gpu.sh` tries each type and subnet until one comes up (GPU capacity moves +between zones) and schedules a power-off 48 h after boot: prep, the day, and slack. Then: Docker's address pools on +the AMI's own Docker, the runner (a local registry, the plugin at this checkout's branch with the pinned framework, +the env and desk images), the streamer, the live server with the data from step 2, the stream (recording only, +nothing pushed) and the watchdog. + +```bash +ID=$(live/aws/launch-gpu.sh airops-live-onair) +live/onair/prep.sh $ID +live/runner/setup.sh $ID # ends with the schedule's dry run +live/stream/deploy.sh $ID +live/onair/deploy-server.sh $ID +live/stream/start.sh $ID --target none +live/onair/fresh.sh $ID +live/onair/health.sh $ID +``` + +The stream's page defaults to the live server on the same box (`--url` and `--safe-url` change it); until the runner +starts, it waits for a day. + +## Test at $0: scripted desks + +The env's scripted desks play real days through the real `agent-env run`, with no model. A day takes 3 to 5 minutes +(the env deploys in about 40 s); with 8-minute slots each is followed by its score card and an intermission, as on +the stream. Its pilot is its own folder. + +```bash +live/runner/start.sh $ID --policy script --name live-script-box --set slot_minutes=8 --days 3 +live/onair/health.sh $ID # one JSON line; repeat +live/runner/status.sh $ID +live/onair/grab.sh $ID frames 20:05:00 20:09:30=room # frames of the recording, UTC +live/onair/clip.sh $ID clip.mp4 20:05:00=20 20:09:30=20 +``` + +`health.sh` prints `picture_vs_page` (the stream's frame against Chrome's screenshot, 96x54 grey: about 1 while the +encoder sees the page, tens on a stale picture), the runner's status and spend, the server's `status`, `watch`, `lag`, +`exact` and `mismatches` (`exact` true and 0 mismatches: the server's replay agrees with the env), and the last capture +and push metrics (`ff_fps` 30, `ff_drop` flat after the first minute, `chrome_restarts`). + +## Rehearse: one model day + +One real day in its own pilot, about $22 and an hour with GPT-6 Astra, to see a model day on the stream end to end: + +```bash +live/runner/start.sh $ID --name occ3-live-rehearsal --days 1 +``` + +Its spend isn't counted by the stream's pilot: lower the stream's `budget_usd` by it. To watch it privately, push to +a private RTMP server with `live/stream/start.sh $ID --target rtmp://HOST/APP/KEY`, or cut a clip from the recording. + +## Arm for a start time + +The runner holds an intermission (the page's "next storm" card) until `start_utc`, then plays the days of +`scripts/live_runner.toml` in order, in slots: a day starts only if the spend so far plus a worst day +(`worst_day_usd`, $42) stays within `budget_usd`, and only if its slot ends inside the `window_hours` (24) from +`start_utc`; a day that ends early is followed by an intermission until its slot ends. At $22.68 a typical Astra day, +18 days fit $430; 84-minute slots spread 17 over the 24 hours. The config is read again before every day, and `--set` +wins over it. Plan with the same arguments first; `plan.sh` plays nothing: + +```bash +live/runner/plan.sh $ID --set start_utc=2026-10-23T16:00:00Z --set budget_usd=405 --set slot_minutes=84 +live/runner/start.sh $ID --set start_utc=2026-10-23T16:00:00Z --set budget_usd=405 --set slot_minutes=84 +``` + +For another model, set `LIVE_MODEL` and, with `--set` or in the config, its `cap_joint_usd` (each desk's cost cap; +Astra's is $23.50), `worst_day_usd` and `typical_day_usd`. + +## Go live on X + +1. In X Media Studio (studio.x.com), **Producer**, **Sources**, **Create source**: type RTMP, the nearest region. + Copy the **RTMP URL** and the **Stream key** into SSM: + ```bash + live/aws/put-secret.sh x-server + live/aws/put-secret.sh x-key + ``` +2. About 30 minutes before the start, push the relay to X (this restarts only the push; capture and recording go + on): `live/stream/start.sh $ID --target ssm`. +3. In Producer, **Broadcasts**, **Create broadcast**: pick the source, set the title, wait for the preview (the + page, with sound, about 10 s after step 2), then **Go Live**. + +The key is read from SSM when the push starts and never printed (the logs show the parameter names). An RTMP drop to +X restarts only the push (1 s, doubling to 30 s); the encoder and the recording keep going. A new key in SSM takes +effect at the next `start.sh --target ssm`. + +## Monitor + +```bash +live/onair/health.sh $ID # every half hour or so +live/runner/status.sh $ID # the runner: day, spend, next, the last days, its log +live/stream/status.sh $ID # units, relay, metrics, restarts and warnings, GPU, disk +live/aws/ssm.sh $ID 60 live/onair/box/logs.sh 10 # the server's and runner's logs, the fresh watchdog's +live/aws/ssm.sh $ID 300 live/onair/box/metrics.sh 16:00:00 18:00:00 # encoder runs and drops, fps, CPU, GPU, memory +live/aws/ssm.sh $ID 600 live/onair/box/loudness.sh "2026-10-23 16:00" "2026-10-23 16:30" # LUFS, true peak, silences +``` + +The fresh watchdog logs a line a minute to `/var/log/airops-stream/fresh.jsonl` (`diff` about 1 while fresh, an +`action` when it restarted the encoder). To change code during the stream, commit on the branch and +`live/onair/sync.sh $ID`: `web/` changes are served at once (the page reloads between days), a restarted +`airops-live-server` resumes from its checkpoint, and the runner takes its changes between days. + +## Stop, save, tear down + +The runner stops by itself at the end of the window or the budget or the days (`current.json` `final`); the page then +holds its last card. + +```bash +live/runner/stop.sh $ID # finish the day in progress, then exit; `cancel` takes it back +live/runner/stop.sh $ID now # stop the day now (agent-env tears it down; the attempt is recorded) +``` + +End the broadcast in Producer, then `live/stream/stop.sh $ID --push-only` (recording goes on) or +`live/stream/stop.sh $ID` (everything). + +```bash +live/stream/save-recordings.sh $ID 2026-10-23_1 # finished segments matching a pattern, to s3://BUCKET/PREFIX/rec/ +live/runner/fetch.sh $ID # current.json, the logs and the pilot folder, into live/runner/fetched/ +live/aws/teardown.sh $ID airops-live-onair +``` + +Recordings are `/var/lib/airops-stream/rec/airops/.mp4`, one-hour fMP4 segments of about 2.7 GB (24 h is +about 65 GB); below 8 GB free the oldest is deleted, and every encoder restart starts a new one. Save a few hours at a +time, so that each SSM command stays small. The pilot folder is a board's: `agent-env airops-occ3 report ` +replays and audits it (about 5 minutes a day; not on the box while it streams). Afterwards, reset the stream key in +Media Studio, revoke the model key, delete the SSM parameters and empty `s3://BUCKET/PREFIX/`. + +## What restarts what + +| Fault | Detected by | Action, and the recovery measured | +|---|---|---| +| Chrome exits or crashes | the process | restart (2 s, doubling to 60 s), the slate meanwhile: back in about 20 s | +| the page hangs | no frame for 20 s (a frame counter injected through DevTools) | reload; if still none, restart Chrome: 75 s | +| the viewer shows its error | `.error` in the page | reload, at most every 30 s | +| a lost WebGL context | `webglcontextlost` | reload after 5 s, then the page's load: about 20 s | +| Chrome's memory | PSS every 30 s | over 6 GB: restart between days (the safe URL says done or intermission); over 9 GB: now | +| the encoder exits or stalls | the process, or no new frame for 20 s | restart (1 s, doubling to 30 s): 12 s | +| the X server restarts | the pid in `/tmp/.X0-lock` | restart the encoder | +| a stale picture at full rate | fresh.py, 3 minutes past a difference of 12 | kill the encoder; the supervisor starts it again | +| PulseAudio exits | the process | restart it and the encoder: 17 s | +| the push drops or stalls | the process, or no progress for 30 s | reconnect (1 s, doubling to 30 s): 11 s | +| the relay, the display, a supervisor | systemd | `Restart=always`: the relay in 14 s | +| the live server | systemd | `Restart=always`: answering in 3 s, back on the day from its checkpoint in about 6 s | +| the runner | systemd | restarted after 30 s; it picks the running day up (`KillMode=process` keeps `agent-env run` alive) | +| a full disk | under 8 GB free | delete the oldest recording segment | + +## Measured + +- **Picture:** 1920x1080 at 30 fps, NVENC High CBR 6000k, with 0 dropped frames after warm-up (each encoder start drops + about 120 frames in its first minute while Chrome loads): 48 minutes on a g6.xlarge with 0 drops, and on the + g6.4xlarge 29.97 fps at 1.00x speed with no new drops under 16 busy loops on every core for 2 minutes. +- **Restarts:** the live server checkpoints its replay at each watch, so a restarted server is back on the day in about + 6 s; without the checkpoint it replays the day from 04:00, single-threaded (about 80 s of CPU per Astra watch). +- **A GPT-6 Astra day** costs about $22 and takes about an hour of wall time here: the rehearsal day, 2024-06-26 at + JFK, cost $21.82 in 58 minutes (the board's Astra days: $22.68 on average, $41.46 at most). +- **Sound:** about -20 LUFS integrated, with the limiter at -4.5 dBFS (below). +- **Memory:** Chrome's PSS grew about 300 MB in 48 minutes; the page's heap after GC stayed flat across days. + +## Traps + +- **x11grab after an X restart encodes a stale screen.** ffmpeg keeps reading the old X server's shared memory at full + frame rate, so a frame-count watchdog stays green while the stream shows a frozen loop. Fix: the capture supervisor + restarts the encoder when the X server's pid changes, and the fresh watchdog compares the stream with Chrome's + screenshot each minute and restarts the encoder after three stale minutes. Check `picture_vs_page` in `health.sh`. +- **The framework is pinned on the box:** agentenv-framework 0.9.1339 and agentenv-framework-protocol 0.1.354 + (`LIVE_PINS`). 0.9.1343 breaks deploys from the repo directory and 0.9.1344 fixes it: check a newer pin with the $0 + test before you use it. +- **`--autoplay-policy=no-user-gesture-required`.** Without it Chrome won't start the page's AudioContext without a + click, and the stream carries only the quiet airport bed under the page. `streamer.py` passes it; keep it if you + change Chrome's flags. +- **AAC adds true peak.** The page's limiter sits at -4.5 dBFS because at -2.5 dBFS the stream measured -0.6 dBTP + after the AAC encode. Measure with `loudness.sh` before you raise it. +- **Keys belong in SSM.** The X stream key and the LiteLLM key go in with `put-secret.sh` and are read on the box when + they're needed (`with-model.sh` at every runner start, the push at every start). Never put one on a command line, in + `live.env` or in a chat. +- **The box powers itself off** MAX_HOURS after boot (48 by default for `launch-gpu.sh`); `runner/status.sh` and + `stream/status.sh` print when, and `shutdown -c` on the box cancels it. +- **The Deep Learning AMI ships Docker,** so the runner's setup leaves its `daemon.json` alone: run `onair/prep.sh` + first, or the env's Docker networks run out of addresses. +- **Leave the X output at the driver's 1024x768.** Set to 1920x1080, Chrome syncs to 60 Hz and the 3D falls to 34 to + 52 fps; the virtual screen is 1920x1080 and the slate fills the whole root window. +- **A rehearsal in its own pilot isn't counted** in the stream's spend: lower `budget_usd` by it. diff --git a/live/aws/common.sh b/live/aws/common.sh new file mode 100755 index 0000000..a38484b --- /dev/null +++ b/live/aws/common.sh @@ -0,0 +1,92 @@ +# Sourced by the other scripts: the settings (live/live.env, or the file LIVE_ENV names), the AWS profile and region, +# and the S3 and SSM helpers. +# shellcheck disable=SC2034 +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIVE_ENV="${LIVE_ENV:-$(cd "$HERE/.." && pwd)/live.env}" +[ -f "$LIVE_ENV" ] || { echo "ERROR: no $LIVE_ENV: copy live/live.env.example there and fill it in" >&2; exit 1; } +# shellcheck source=/dev/null +source "$LIVE_ENV" + +# Forced, not defaulted: a shell's own AWS_PROFILE (e.g. a production one) must never be used here. +if [ -n "${LIVE_AWS_PROFILE:-}" ]; then + export AWS_PROFILE="$LIVE_AWS_PROFILE" + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +fi +export AWS_REGION="${LIVE_AWS_REGION:-us-west-2}" +export AWS_DEFAULT_REGION="$AWS_REGION" +export AWS_PAGER="" +if [ -n "${LIVE_AWS_ACCOUNT:-}" ]; then + [ "$(aws sts get-caller-identity --query Account --output text)" = "$LIVE_AWS_ACCOUNT" ] \ + || { echo "ERROR: these credentials are not account $LIVE_AWS_ACCOUNT" >&2; exit 1; } +fi + +REPO_URL="${LIVE_REPO_URL:-https://github.com/earakely-scale/agentenv-airops-plugin.git}" +BASE_REF="${LIVE_BASE_REF:-v0.5.0}" +BUCKET="${LIVE_BUCKET:-}" +PREFIX="${LIVE_S3_PREFIX:-airops-live}" +SUBNET="${SUBNET:-${LIVE_SUBNET:-}}" +SG="${LIVE_SECURITY_GROUP:-}" +PROFILE_NAME="${LIVE_INSTANCE_PROFILE:-}" +TAG_OWNER="${LIVE_TAG_OWNER:-${USER:-airops}}" +TAG_PURPOSE="${LIVE_TAG_PURPOSE:-airops-live}" + +log() { echo "[$(date +%H:%M:%S)] $*" >&2; } +die() { echo "ERROR: $*" >&2; exit 1; } +need() { local v; for v in "$@"; do [ -n "${!v:-}" ] || die "set $v in $LIVE_ENV"; done; } + +# presign get|put KEY SECONDS: a presigned URL for s3://$BUCKET/KEY (aws s3 presign only does GET). +presign() { + local py=(python3) + python3 -c 'import boto3' 2>/dev/null || py=(uv run --quiet --no-project --with boto3 python) + "${py[@]}" - "$1" "$BUCKET" "$2" "$3" "$AWS_REGION" <<'PY' +import sys, boto3 +from botocore.config import Config +op, bucket, key, ttl, region = sys.argv[1:6] +s3 = boto3.client("s3", region_name=region, endpoint_url=f"https://s3.{region}.amazonaws.com", + config=Config(signature_version="s3v4", s3={"addressing_style": "virtual"})) +print(s3.generate_presigned_url({"get": "get_object", "put": "put_object"}[op], + Params={"Bucket": bucket, "Key": key}, ExpiresIn=int(ttl))) +PY +} + +wait_ssm() { + local id=$1 i + for i in $(seq 1 90); do + [ "$(aws ssm describe-instance-information --filters "Key=InstanceIds,Values=$id" \ + --query 'InstanceInformationList[0].PingStatus' --output text 2>/dev/null)" = Online ] && return 0 + sleep 10 + done + die "$id never came online in SSM" +} + +# ssm INSTANCE TIMEOUT_S SCRIPT [ARGS...]: run SCRIPT with ARGS under bash as root on the box; prints its stdout and +# stderr (SSM keeps ~24 KB of stdout and ~8 KB of stderr) and returns its exit code. The script and its arguments +# travel base64-encoded in the command, so presigned URLs passed as arguments sit in SSM's command history. +ssm() { + local id=$1 timeout=$2 file=$3; shift 3 + local wrapped b64 params cid status=Pending out + wrapped="$(printf 'set --'; printf ' %q' "$@"; printf '\n'; cat "$file")" + b64=$(printf '%s\n' "$wrapped" | base64 | tr -d '\n') + params=$(python3 -c 'import json,sys; print(json.dumps({"commands": [sys.argv[1]], "executionTimeout": [sys.argv[2]]}))' \ + "f=\$(mktemp); echo $b64 | base64 -d > \$f; bash \$f" "$timeout") + cid=$(aws ssm send-command --instance-ids "$id" --document-name AWS-RunShellScript --timeout-seconds 600 \ + --comment "airops $(basename "$file")" --parameters "$params" --query Command.CommandId --output text) + while :; do + sleep 5 + status=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$id" --query Status --output text \ + 2>/dev/null || echo Pending) + case $status in Pending|InProgress|Delayed|Cancelling) ;; *) break ;; esac + done + out=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$id" --output json) + python3 -c ' +import json, sys +r = json.loads(sys.stdin.read()) +sys.stdout.write(r.get("StandardOutputContent", "")) +err = r.get("StandardErrorContent", "") +if err.strip(): + sys.stderr.write("--- stderr ---\n" + err) +' <<<"$out" + [ "$status" = Success ] || { log "SSM command $cid ended $status"; return 1; } +} diff --git a/live/aws/launch-gpu.sh b/live/aws/launch-gpu.sh new file mode 100755 index 0000000..e06bb5f --- /dev/null +++ b/live/aws/launch-gpu.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# launch-gpu.sh NAME [MAX_HOURS] [ROOT_GB]: the on-air box, a GPU box on the AWS Deep Learning Base OSS NVIDIA driver +# AMI (Ubuntu 22.04). GPU capacity moves between zones: it tries each type of LIVE_GPU_TYPES in each subnet of +# LIVE_SUBNETS (default LIVE_SUBNET) until one comes up in SSM, and prints its instance id. Defaults: 48 h, 250 GB. +source "$(dirname "$0")/common.sh" +NAME=${1:?usage: launch-gpu.sh NAME [MAX_HOURS] [ROOT_GB]} +MAX_HOURS=${2:-48}; ROOT_GB=${3:-250} +TYPES=${LIVE_GPU_TYPES:-g6.4xlarge gr6.4xlarge g6.8xlarge}; SUBNETS=${LIVE_SUBNETS:-${LIVE_SUBNET:-}} +[ -n "$SUBNETS" ] || die "set LIVE_SUBNET (or LIVE_SUBNETS) in $LIVE_ENV" +AMI=$(aws ssm get-parameter --name /aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id \ + --query Parameter.Value --output text) +export AMI +for t in $TYPES; do + for s in $SUBNETS; do + out=$(SUBNET=$s bash "$HERE/launch.sh" "$t" "$NAME" "$MAX_HOURS" "$ROOT_GB" 2>&1 || true) + log "$t $s: $(echo "$out" | tail -1 | cut -c1-150)" + if echo "$out" | grep -q 'online in SSM'; then echo "$out" | tail -1; exit 0; fi + done +done +die "no $NAME came up: no capacity for $TYPES in $SUBNETS" diff --git a/live/aws/launch.sh b/live/aws/launch.sh new file mode 100755 index 0000000..edef347 --- /dev/null +++ b/live/aws/launch.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB] +# Launches (or finds, when one named NAME is already pending or running) an Ubuntu 24.04 box (AMI= another image), +# waits until SSM sees it and prints its instance id on stdout. The box powers itself off, and so terminates, MAX_HOURS +# (default 12) after boot. +source "$(dirname "$0")/common.sh" +need LIVE_SECURITY_GROUP LIVE_INSTANCE_PROFILE +[ -n "$SUBNET" ] || die "set LIVE_SUBNET in $LIVE_ENV" +TYPE=${1:?usage: launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB]} +NAME=${2:?usage: launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB]} +MAX_HOURS=${3:-12} +ROOT_GB=${4:-100} + +id=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=$NAME" "Name=tag:purpose,Values=$TAG_PURPOSE" \ + "Name=instance-state-name,Values=pending,running" \ + --query 'Reservations[].Instances[0].InstanceId' --output text) +if [ -n "$id" ] && [ "$id" != None ]; then + log "$NAME is already up as $id" +else + arch=amd64 + [[ $TYPE =~ ^[a-z]+[0-9]+[a-z]*g[a-z]*\. ]] && arch=arm64 + ami=${AMI:-$(aws ssm get-parameter --name "/aws/service/canonical/ubuntu/server/24.04/stable/current/$arch/hvm/ebs-gp3/ami-id" \ + --query Parameter.Value --output text)} + userdata=$(printf '#!/bin/bash\nshutdown -h +%d "airops: MAX_HOURS reached"\n' $((MAX_HOURS * 60)) | base64 | tr -d '\n') + tags="{Key=Name,Value=$NAME},{Key=owner,Value=$TAG_OWNER},{Key=purpose,Value=$TAG_PURPOSE}" + id=$(aws ec2 run-instances --image-id "$ami" --instance-type "$TYPE" --count 1 \ + --subnet-id "$SUBNET" --security-group-ids "$SG" --iam-instance-profile "Name=$PROFILE_NAME" \ + --instance-initiated-shutdown-behavior terminate \ + --metadata-options HttpTokens=required,HttpEndpoint=enabled \ + --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$ROOT_GB,VolumeType=gp3,DeleteOnTermination=true}" \ + --user-data "$userdata" \ + --tag-specifications "ResourceType=instance,Tags=[$tags]" "ResourceType=volume,Tags=[$tags]" \ + --query 'Instances[0].InstanceId' --output text) + log "launched $id ($TYPE, $arch, $ami, ${ROOT_GB} GB root, powers off after ${MAX_HOURS} h)" +fi +aws ec2 wait instance-running --instance-ids "$id" +log "$id running; waiting for SSM" +wait_ssm "$id" +log "$id online in SSM" +echo "$id" diff --git a/live/aws/put-secret.sh b/live/aws/put-secret.sh new file mode 100755 index 0000000..20456ed --- /dev/null +++ b/live/aws/put-secret.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# put-secret.sh litellm-key|x-server|x-key: reads the value without echoing it and stores it as a SecureString at the +# SSM parameter live.env names for it (LIVE_LITELLM_KEY_PARAM, LIVE_X_SERVER_PARAM, LIVE_X_KEY_PARAM). The value never +# reaches the terminal, the shell history or a file. +source "$(dirname "$0")/common.sh" +case ${1:-} in + litellm-key) NAME=${LIVE_LITELLM_KEY_PARAM:-/airops-live/litellm-key} ;; + x-server) NAME=${LIVE_X_SERVER_PARAM:-/airops-live/x-server} ;; + x-key) NAME=${LIVE_X_KEY_PARAM:-/airops-live/x-key} ;; + *) die "usage: put-secret.sh litellm-key|x-server|x-key" ;; +esac +read -rsp "value for $NAME: " v; echo >&2 +[ -n "$v" ] || die "empty value; nothing stored" +aws ssm put-parameter --name "$NAME" --type SecureString --overwrite --value "$v" --query Version --output text >/dev/null +unset v +log "stored $NAME ($(aws ssm get-parameter --name "$NAME" --with-decryption --query 'length(Parameter.Value)' --output text) chars)" diff --git a/live/aws/ssm.sh b/live/aws/ssm.sh new file mode 100755 index 0000000..acb02a6 --- /dev/null +++ b/live/aws/ssm.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# ssm.sh ID TIMEOUT_S SCRIPT [ARGS...]: runs a box script (one under a box/ folder) as root on box ID through SSM and +# prints its output, e.g. ssm.sh ID 300 live/onair/box/loudness.sh 16:00 17:00 +source "$(dirname "$0")/common.sh" +[ $# -ge 3 ] || die "usage: ssm.sh ID TIMEOUT_S SCRIPT [ARGS...]" +ssm "$@" diff --git a/live/aws/teardown.sh b/live/aws/teardown.sh new file mode 100755 index 0000000..0238f39 --- /dev/null +++ b/live/aws/teardown.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# teardown.sh INSTANCE_ID NAME +# Terminates INSTANCE_ID only if its Name tag is NAME and its purpose tag is LIVE_TAG_PURPOSE's, waits until it is +# terminated and prints the state. An instance already terminated is reported, not an error. +source "$(dirname "$0")/common.sh" +ID=${1:?usage: teardown.sh INSTANCE_ID NAME} +NAME=${2:?usage: teardown.sh INSTANCE_ID NAME} +read -r state name purpose < <(aws ec2 describe-instances --instance-ids "$ID" --query \ + 'Reservations[0].Instances[0].[State.Name, Tags[?Key==`Name`].Value|[0], Tags[?Key==`purpose`].Value|[0]]' \ + --output text) +[ "$name" = "$NAME" ] || die "$ID is named '$name', not '$NAME'; not terminating" +[ "$purpose" = "$TAG_PURPOSE" ] || die "$ID has purpose '$purpose', not '$TAG_PURPOSE'; not terminating" +if [ "$state" != terminated ]; then + log "terminating $ID ($NAME, $state)" + aws ec2 terminate-instances --instance-ids "$ID" --query 'TerminatingInstances[0].CurrentState.Name' --output text >&2 + aws ec2 wait instance-terminated --instance-ids "$ID" +fi +aws ec2 describe-instances --instance-ids "$ID" \ + --query 'Reservations[0].Instances[0].[InstanceId,InstanceType,State.Name,Tags[?Key==`Name`].Value|[0]]' --output text diff --git a/live/live.env.example b/live/live.env.example new file mode 100644 index 0000000..0a1164a --- /dev/null +++ b/live/live.env.example @@ -0,0 +1,43 @@ +# The live tooling's settings. Copy to live/live.env (git-ignored), fill it in; every laptop script sources it. +# Another file: LIVE_ENV=/path/to/file before the command. No secret goes here: keys live in SSM (aws/put-secret.sh). + +# AWS: the profile is forced for every call when set (a shell's own AWS_PROFILE is then ignored); empty uses the +# shell's credentials. With LIVE_AWS_ACCOUNT set, every script first checks the credentials are that account's. +LIVE_AWS_PROFILE= +LIVE_AWS_ACCOUNT= +LIVE_AWS_REGION=us-west-2 + +# Where boxes launch: a subnet with a route to the internet (apt, PyPI, GitHub, the model endpoint, X's ingest, the +# viewer's map tiles), a security group (no inbound rule is needed: everything goes through SSM) and an instance +# profile whose role has AmazonSSMManagedInstanceCore (it also reads the SecureString parameters below). +LIVE_SUBNET= +# launch-gpu.sh tries each of these subnets in turn (GPU capacity moves between zones); default LIVE_SUBNET. +LIVE_SUBNETS= +LIVE_SECURITY_GROUP= +LIVE_INSTANCE_PROFILE= +# launch-gpu.sh's instance types, in order. +LIVE_GPU_TYPES="g6.4xlarge gr6.4xlarge g6.8xlarge" +# Tags on every box; teardown.sh terminates only a box with this purpose tag. Owner default: $USER. +LIVE_TAG_OWNER= +LIVE_TAG_PURPOSE=airops-live + +# Transfers: code bundles, the server's data, results, clips and recordings go through this bucket (in LIVE_AWS_REGION) +# under LIVE_S3_PREFIX, with presigned URLs, so the boxes need no S3 rights. +LIVE_BUCKET= +LIVE_S3_PREFIX=airops-live + +# The code: the box clones LIVE_REPO_URL and checks out your branch, shipped as a git bundle over LIVE_BASE_REF (a tag +# of LIVE_REPO_URL that the branch contains), in a venv with LIVE_PINS. +LIVE_REPO_URL=https://github.com/earakely-scale/agentenv-airops-plugin.git +LIVE_BASE_REF=v0.5.0 +LIVE_PINS="agentenv-framework==0.9.1339 agentenv-framework-protocol==0.1.354 highspy==1.15.1 numpy==2.5.4 pandas==3.0.6 pyarrow==26.0.0 shapely==2.2.0 airportsdata==20260905" + +# The model: a LiteLLM-compatible endpoint (OpenAI-style chat), its key in SSM, and the model the desks play +# (scripts/live_runner.toml's model when empty). +LIVE_LITELLM_BASE_URL= +LIVE_LITELLM_KEY_PARAM=/airops-live/litellm-key +LIVE_MODEL=openai/gpt-6-astra + +# X (or any RTMP target): the server URL and the stream key, each a SecureString in SSM. +LIVE_X_SERVER_PARAM=/airops-live/x-server +LIVE_X_KEY_PARAM=/airops-live/x-key diff --git a/live/onair/box/clip.sh b/live/onair/box/clip.sh new file mode 100644 index 0000000..0bd9c5e --- /dev/null +++ b/live/onair/box/clip.sh @@ -0,0 +1,19 @@ +# clip.sh PUT_URL START=SECONDS...: cuts each [UTC start (HH:MM:SS today, or YYYY-MM-DDTHH:MM:SS), seconds] piece +# from the recording, joins them with 0.4 s fades (picture and sound), encodes H.264 + AAC and uploads the MP4. +set -euo pipefail +URL=$1; shift +REC=/var/lib/airops-stream/rec/airops; D=$(mktemp -d); i=0; : > $D/list.txt +for spec in "$@"; do + t=${spec%=*}; dur=${spec#*=} + if [ ${#t} -gt 8 ]; then want=$(date -u -d "$t" +%s); else want=$(date -u -d "${CLIPDAY:-$(date -u +%F)} $t" +%s); fi + seg=; for f in $(ls $REC | sort); do s=$(echo $f | sed -E 's/^([0-9-]+)_([0-9]+)-([0-9]+)-([0-9]+)-.*/\1 \2:\3:\4/'); st=$(date -u -d "$s" +%s); [ $st -le $want ] && { seg=$f; off=$((want-st)); }; done + i=$((i+1)); e=$(python3 -c "print($dur - 0.4)") + nice -n 10 ffmpeg -nostdin -loglevel error -ss $off -i $REC/$seg -t $dur \ + -vf "fade=t=in:st=0:d=0.4,fade=t=out:st=$e:d=0.4" -af "afade=t=in:st=0:d=0.4,afade=t=out:st=$e:d=0.4,aresample=48000" \ + -c:v h264_nvenc -preset p5 -rc vbr -cq 23 -b:v 8M -maxrate 10M -pix_fmt yuv420p -g 60 -c:a aac -b:a 192k -ar 48000 $D/p$i.mp4 + echo "file '$D/p$i.mp4'" >> $D/list.txt; echo "piece $i: $t +${dur}s from $seg at ${off}s" +done +nice -n 10 ffmpeg -nostdin -loglevel error -f concat -safe 0 -i $D/list.txt -c copy -movflags +faststart $D/clip.mp4 +ffprobe -v error -show_entries format=duration,size -of compact $D/clip.mp4 +ffmpeg -nostdin -hide_banner -i $D/clip.mp4 -vn -af ebur128=peak=true -f null - 2>&1 | grep -A14 "Summary:" | grep -E "I:|Peak:" +curl -fsS -X PUT --upload-file $D/clip.mp4 "$URL" && echo uploaded; rm -rf $D diff --git a/live/onair/box/docker-pools.sh b/live/onair/box/docker-pools.sh new file mode 100644 index 0000000..62e063d --- /dev/null +++ b/live/onair/box/docker-pools.sh @@ -0,0 +1,12 @@ +# docker-pools.sh: on the Deep Learning AMI (Docker preinstalled, with the NVIDIA runtime), adds Docker address pools +# of /24s to its daemon.json, keeping the rest, so the env's networks don't run out; turns off apt's timers. +set -euo pipefail +f=/etc/docker/daemon.json; [ -f $f ] || echo '{}' > $f +if ! jq -e '."default-address-pools"' $f >/dev/null; then + jq '. + {"default-address-pools":[{"base":"172.20.0.0/14","size":24}]}' $f > $f.new && mv $f.new $f + [ -z "$(docker ps -q)" ] || { echo "containers running: not restarting docker"; exit 1; } + systemctl restart docker +fi +cat $f; docker buildx version; docker compose version --short +systemctl stop unattended-upgrades apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true +systemctl disable unattended-upgrades apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true diff --git a/live/onair/box/fresh-install.sh b/live/onair/box/fresh-install.sh new file mode 100644 index 0000000..037a097 --- /dev/null +++ b/live/onair/box/fresh-install.sh @@ -0,0 +1,18 @@ +set -euo pipefail +mkdir -p /opt/airops-onair +echo "$1" | base64 -d > /opt/airops-onair/fresh.py +cat > /etc/systemd/system/airops-fresh.service <<'U' +[Unit] +Description=AirOps stream: the picture against the page; restarts a stale encoder +After=airops-capture.service +[Service] +ExecStart=/usr/bin/python3 -u /opt/airops-onair/fresh.py /var/log/airops-stream/fresh.jsonl +Restart=always +RestartSec=10 +CPUWeight=20 +Nice=10 +[Install] +WantedBy=multi-user.target +U +systemctl daemon-reload; systemctl enable -q --now airops-fresh; systemctl restart airops-fresh +sleep 75; systemctl is-active airops-fresh; tail -2 /var/log/airops-stream/fresh.jsonl diff --git a/live/onair/box/fresh.py b/live/onair/box/fresh.py new file mode 100644 index 0000000..568223e --- /dev/null +++ b/live/onair/box/fresh.py @@ -0,0 +1,70 @@ +"""The stream's picture against the page: each minute Chrome's screenshot of the page and the relay's next seconds of +frames, at 96x54 grey; the closest frame's mean difference stays a few levels while the encoder sees the page. Three +minutes in a row past FAR restart the encoder (the capture supervisor starts it again): x11grab can keep encoding a +stale screen at full rate, which no frame count catches.""" +import base64 +import json +import subprocess +import sys +import time +import urllib.request + +import websocket + +OUT, EVERY, FAR, TIMES = sys.argv[1], 60.0, 12.0, 3 +W, H = 96, 54 +RELAY = "rtmp://127.0.0.1:1935/airops" + + +def page() -> bytes: + pages = [x for x in json.load(urllib.request.urlopen("http://127.0.0.1:9222/json", timeout=10)) + if x.get("type") == "page"] + ws = websocket.create_connection(pages[0]["webSocketDebuggerUrl"], timeout=20, suppress_origin=True) + try: + ws.send(json.dumps({"id": 1, "method": "Page.captureScreenshot", "params": {"format": "png"}})) + while True: + m = json.loads(ws.recv()) + if m.get("id") == 1: + break + finally: + ws.close() + return grey(["-f", "png_pipe", "-i", "-"], base64.b64decode(m["result"]["data"])) + + +def grey(args: list[str], stdin: bytes | None = None) -> bytes: + return subprocess.run(["ffmpeg", "-nostdin" if stdin is None else "-hide_banner", "-loglevel", "error", *args, + "-vf", f"scale={W}:{H},format=gray", "-f", "rawvideo", "-"], + input=stdin, capture_output=True, timeout=60).stdout + + +def closest(shot: bytes, frames: bytes) -> float | None: + n = W * H + diffs = [sum(abs(a - b) for a, b in zip(shot, frames[i:i + n], strict=False)) / n + for i in range(0, len(frames) - n + 1, n)] + return round(min(diffs), 1) if diffs and len(shot) == n else None + + +def main() -> None: + far = 0 + while True: + t0 = time.time() + row = {"t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + try: + shot = page() + frames = grey(["-i", RELAY, "-t", "6", "-r", "10"]) + row |= {"diff": closest(shot, frames), "frames": len(frames) // (W * H)} + far = far + 1 if row["diff"] is None or row["diff"] > FAR else 0 + if far >= TIMES: + subprocess.run(["pkill", "-f", "x11grab -framerate"], timeout=10) + row["action"] = "restarted the encoder" + far = 0 + except Exception as e: + row["error"] = f"{type(e).__name__}: {e}"[:200] + row["far"] = far + with open(OUT, "a") as f: + f.write(json.dumps(row) + "\n") + time.sleep(max(1.0, EVERY - (time.time() - t0))) + + +if __name__ == "__main__": + main() diff --git a/live/onair/box/grab.sh b/live/onair/box/grab.sh new file mode 100644 index 0000000..aa8e9a4 --- /dev/null +++ b/live/onair/box/grab.sh @@ -0,0 +1,11 @@ +set -euo pipefail +URL=$1; shift +D=$(mktemp -d); REC=/var/lib/airops-stream/rec/airops +for spec in "$@"; do + t=${spec%%=*}; name=${spec#*=}; [ "$name" = "$spec" ] && name=$(echo $t | tr -d ':') + if [ ${#t} -gt 8 ]; then want=$(date -u -d "$t" +%s); else want=$(date -u -d "$(date -u +%F) $t" +%s); fi + seg=; for f in $(ls $REC | sort); do s=$(echo $f | sed -E 's/^([0-9-]+)_([0-9]+)-([0-9]+)-([0-9]+)-.*/\1 \2:\3:\4/'); st=$(date -u -d "$s" +%s); [ $st -le $want ] && { seg=$f; off=$((want-st)); }; done + [ -n "$seg" ] || { echo "no segment for $t"; continue; } + ffmpeg -nostdin -loglevel error -ss $off -i $REC/$seg -frames:v 1 -q:v 3 $D/$name.jpg && echo "$name: $seg +${off}s" +done +tar -cf /tmp/grab.tar -C $D . && curl -fsS -X PUT --upload-file /tmp/grab.tar "$URL"; rm -rf $D /tmp/grab.tar diff --git a/live/onair/box/health.sh b/live/onair/box/health.sh new file mode 100644 index 0000000..9c9de8e --- /dev/null +++ b/live/onair/box/health.sh @@ -0,0 +1,34 @@ +# health.sh: one JSON line: the stream's picture against Chrome's screenshot of the page (mean difference at 96x54 grey: +# about 1 while the encoder sees the page, tens on a stale screen), the runner, the live server, and the last capture +# and push metrics. +python3 - <<'PY' +import json, urllib.request, base64, subprocess, threading, websocket, time +def raw(args): + return subprocess.run(["ffmpeg", "-nostdin", "-loglevel", "error", *args, "-vf", "scale=96:54,format=gray", "-f", "rawvideo", "-"], capture_output=True, timeout=40).stdout +shot = {} +def page(): + t = [x for x in json.load(urllib.request.urlopen("http://127.0.0.1:9222/json")) if x.get("type") == "page"] + ws = websocket.create_connection(t[0]["webSocketDebuggerUrl"], timeout=20, suppress_origin=True) + ws.send(json.dumps({"id": 1, "method": "Page.captureScreenshot", "params": {"format": "png"}})) + while True: + m = json.loads(ws.recv()) + if m.get("id") == 1: break + open("/tmp/h-page.png", "wb").write(base64.b64decode(m["result"]["data"])) +th = threading.Thread(target=page); th.start() +relay = raw(["-i", "rtmp://127.0.0.1:1935/airops", "-frames:v", "1"]) +th.join() +pg = raw(["-i", "/tmp/h-page.png"]) +diff = round(sum(abs(a - b) for a, b in zip(relay, pg)) / max(1, len(pg)), 1) if relay and pg and len(relay) == len(pg) else None +cur = json.load(open("/home/ubuntu/live/current.json")) +st = json.loads(urllib.request.urlopen("http://127.0.0.1:8765/live/state.json", timeout=10).read())["live"] +def last(f): + try: return json.loads(open(f).read().strip().splitlines()[-1]) + except Exception: return {} +cap, push = last("/var/log/airops-stream/metrics-capture.jsonl"), last("/var/log/airops-stream/metrics-push.jsonl") +print(json.dumps({"utc": time.strftime("%H:%M:%S", time.gmtime()), "picture_vs_page": diff, + "runner": {k: cur.get(k) for k in ("status", "day", "date", "hub", "spent_usd", "live_usd", "final")}, + "next": (cur.get("next") or {}).get("starts_utc"), + "server": {k: st.get(k) for k in ("status", "watch", "watches", "lag", "exact", "mismatches", "error")}, + "capture": {k: cap.get(k) for k in ("t", "page_fps", "ff_fps", "ff_drop", "ff_speed", "chrome_pss_mb", "chrome_restarts", "ffmpeg", "cpu_pct", "gpu_c")}, + "push": {k: push.get(k) for k in ("t", "target", "bitrate", "fps", "speed", "reconnects", "restarts") if k in push} or push})) +PY diff --git a/live/onair/box/logs.sh b/live/onair/box/logs.sh new file mode 100644 index 0000000..60f1f62 --- /dev/null +++ b/live/onair/box/logs.sh @@ -0,0 +1,4 @@ +# logs.sh [N]: the last N lines (default 8) of the live server's and the runner's logs and the fresh watchdog's. +echo "--- server.log"; tail -${1:-8} /home/ubuntu/live/server.log +echo "--- runner.log"; tail -${1:-8} /home/ubuntu/live/runner.log +echo "--- fresh.jsonl"; tail -${1:-8} /var/log/airops-stream/fresh.jsonl diff --git a/live/onair/box/loudness.sh b/live/onair/box/loudness.sh new file mode 100644 index 0000000..0f05b59 --- /dev/null +++ b/live/onair/box/loudness.sh @@ -0,0 +1,21 @@ +# loudness.sh FROM TO [OUT.wav]: loudness (EBU R128 integrated, LRA, true peak) and silences over the recording between +# two UTC times (anything `date -d` reads, e.g. "2026-10-23 16:00"); OUT.wav keeps the audio on the box. +set -uo pipefail +REC=/var/lib/airops-stream/rec/airops; from=$(date -u -d "$1" +%s); to=$(date -u -d "$2" +%s) +D=$(mktemp -d); : > $D/list.txt; i=0 +segs=$(for f in $(ls $REC | sort); do s=$(echo $f | sed -E 's/^([0-9-]+)_([0-9]+)-([0-9]+)-([0-9]+)-.*/\1 \2:\3:\4/'); echo "$(date -u -d "$s" +%s) $f"; done) +prev=; echo "$segs" | while read st f; do echo "$st $f"; done > $D/segs +n=$(wc -l < $D/segs) +for k in $(seq 1 $n); do + read st f <<< "$(sed -n ${k}p $D/segs)"; nx=$(sed -n $((k+1))p $D/segs | cut -d' ' -f1); [ -n "$nx" ] || nx=9999999999 + [ $nx -le $from ] || [ $st -ge $to ] && continue + a=$(( from > st ? from - st : 0 )); b=$(( (to < nx ? to : nx) - st )) + nice -n 15 ffmpeg -nostdin -loglevel error -ss $a -i $REC/$f -t $((b-a)) -vn -ac 2 -ar 48000 -c:a pcm_s16le $D/p$k.wav && echo "file '$D/p$k.wav'" >> $D/list.txt + echo "segment $f: ${a}s to ${b}s" +done +nice -n 15 ffmpeg -nostdin -loglevel error -f concat -safe 0 -i $D/list.txt -c copy $D/all.wav +echo "audio: $(ffprobe -v error -show_entries format=duration -of csv=p=0 $D/all.wav) s" +nice -n 15 ffmpeg -nostdin -hide_banner -i $D/all.wav -af ebur128=peak=true -f null - 2>&1 | grep -A14 "Summary:" | grep -E "I:|LRA:|Peak:" +nice -n 15 ffmpeg -nostdin -hide_banner -i $D/all.wav -af silencedetect=n=-45dB:d=1 -f null - 2>&1 | grep -oE "silence_(start|end): [0-9.]+( \| silence_duration: [0-9.]+)?" | head -40 +[ -n "${3:-}" ] && cp $D/all.wav "$3" +rm -rf $D diff --git a/live/onair/box/metrics.sh b/live/onair/box/metrics.sh new file mode 100644 index 0000000..6d3c458 --- /dev/null +++ b/live/onair/box/metrics.sh @@ -0,0 +1,25 @@ +# metrics.sh HH:MM:SS HH:MM:SS: the capture's metrics between two UTC times of the day in its log: the encoder's runs +# with their dropped frames, the fps, speed, CPU, GPU, NVENC, Chrome's memory and frame rate, Chrome restarts. +python3 - "$@" <<'P' +import json, sys +a, b = sys.argv[1], sys.argv[2] +rows = [] +for x in open("/var/log/airops-stream/metrics-capture.jsonl"): + try: r = json.loads(x) + except ValueError: continue + if a <= r["t"][11:19] <= b: rows.append(r) +f = lambda k: [float(str(r[k]).strip().rstrip("x")) for r in rows if r.get(k) not in (None, "", "N/A")] +runs, prev = [], None +for r in rows: + d = int(r.get("ff_drop") or 0) + if prev is None or d < prev: runs.append([r["t"][11:19], d, d]) + else: runs[-1][2] = d + prev = d +print(f"{len(rows)} minutes {a}-{b}") +print("encoder runs [start, drop at first sample, drop at last]:", runs) +for k in ("ff_fps", "ff_speed", "cpu_pct", "gpu_pct", "nvenc_pct", "chrome_pss_mb", "page_fps", "load1", "gpu_c"): + v = f(k) + if v: print(f"{k}: min {min(v):.2f} mean {sum(v)/len(v):.2f} max {max(v):.2f}") +print("chrome_restarts", rows[-1].get("chrome_restarts") if rows else None) +print("pss by 5 min:", [(r["t"][11:16], r["chrome_pss_mb"]) for r in rows[::5]]) +P diff --git a/live/onair/box/server-setup.sh b/live/onair/box/server-setup.sh new file mode 100644 index 0000000..7c7cc44 --- /dev/null +++ b/live/onair/box/server-setup.sh @@ -0,0 +1,22 @@ +set -euo pipefail +PAYLOAD=$1; PREBUILT=$2; ASSETS=$3; INPUTS=$4 +T0=$(date +%s); log() { echo "[$(( $(date +%s) - T0 ))s] $*"; } +R=/home/ubuntu/agentenv-airops-plugin; L=/home/ubuntu/live +[ -x $R/.venv/bin/python ] || { echo "no run box checkout at $R: run the runner's setup.sh first"; exit 1; } +mkdir -p /opt/airops-onair && curl -fsSL "$PAYLOAD" | tar xz -C /opt/airops-onair +fetch() { sudo -u ubuntu mkdir -p "$2"; curl -fsSL "$1" | zstd -dc | sudo -u ubuntu tar -xf - -C "$2"; } +[ -d $L/prebuilt/news ] || fetch "$PREBUILT" $L/prebuilt +[ -d $L/assets/stage ] || fetch "$ASSETS" $L/assets +[ -d $R/cache/inputs ] || fetch "$INPUTS" $R/cache +log "prebuilt $(ls $L/prebuilt/news | wc -l) days, $(du -sh $L/prebuilt | cut -f1); assets $(du -sh $L/assets | cut -f1); inputs $(du -sh $R/cache/inputs | cut -f1)" +cp /opt/airops-onair/units/airops-live-server.service /etc/systemd/system/ +for u in airops-x airops-relay airops-capture airops-push; do + install -d /etc/systemd/system/$u.service.d; cp /opt/airops-onair/units/stream-priority.conf /etc/systemd/system/$u.service.d/priority.conf +done +systemctl daemon-reload +systemctl enable -q airops-live-server +systemctl restart airops-live-server +for i in $(seq 1 60); do curl -fsS -o /dev/null http://127.0.0.1:8765/live/state.json 2>/dev/null && break; sleep 1; done +log "server: $(systemctl is-active airops-live-server); state $(curl -fsS http://127.0.0.1:8765/live/state.json | head -c 300)" +for p in web/index.html web/live.js web/audio.js stage/JFK/stage.json.gz; do echo "$p $(curl -s -o /dev/null -w '%{http_code} %{size_download}' http://127.0.0.1:8765/$p)"; done +tail -5 $L/server.log diff --git a/live/onair/box/sync.sh b/live/onair/box/sync.sh new file mode 100644 index 0000000..ac3f748 --- /dev/null +++ b/live/onair/box/sync.sh @@ -0,0 +1,3 @@ +set -euo pipefail +curl -fsS -o /tmp/sync.bundle "$1"; chmod 644 /tmp/sync.bundle +sudo -iu ubuntu bash -c "cd agentenv-airops-plugin && git fetch -q /tmp/sync.bundle refs/heads/$2 && git checkout -q --detach FETCH_HEAD && git log --oneline -1 && git status --short | head" diff --git a/live/onair/box/units/airops-live-server.service b/live/onair/box/units/airops-live-server.service new file mode 100644 index 0000000..790e885 --- /dev/null +++ b/live/onair/box/units/airops-live-server.service @@ -0,0 +1,17 @@ +[Unit] +Description=AirOps live: the episode server (follows current.json, serves /live, the assets and the viewer) on 127.0.0.1:8765 +After=network.target docker.service +StartLimitIntervalSec=0 + +[Service] +User=ubuntu +WorkingDirectory=/home/ubuntu/agentenv-airops-plugin +Environment=PYTHONUNBUFFERED=1 PYTHONPATH=src +ExecStart=/home/ubuntu/agentenv-airops-plugin/.venv/bin/python scripts/live_server.py serve --current /home/ubuntu/live/current.json --live /home/ubuntu/live/prebuilt --assets /home/ubuntu/live/assets --web /home/ubuntu/agentenv-airops-plugin/web --host 127.0.0.1 --port 8765 --inputs /home/ubuntu/agentenv-airops-plugin/cache/inputs +Restart=always +RestartSec=2 +StandardOutput=append:/home/ubuntu/live/server.log +StandardError=append:/home/ubuntu/live/server.log + +[Install] +WantedBy=multi-user.target diff --git a/live/onair/box/units/stream-priority.conf b/live/onair/box/units/stream-priority.conf new file mode 100644 index 0000000..1fda796 --- /dev/null +++ b/live/onair/box/units/stream-priority.conf @@ -0,0 +1,4 @@ +[Service] +CPUWeight=1000 +IOWeight=1000 +Nice=-5 diff --git a/live/onair/clip.sh b/live/onair/clip.sh new file mode 100755 index 0000000..7a02dd1 --- /dev/null +++ b/live/onair/clip.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# clip.sh ID OUT.mp4 HH:MM:SS=SECONDS...: a clip of those pieces of the box's recording (UTC today, or a full +# YYYY-MM-DDTHH:MM:SS), to OUT.mp4. +set -euo pipefail +O="$(cd "$(dirname "$0")" && pwd)" +source "$O/../aws/common.sh" +[ $# -ge 3 ] || die "usage: clip.sh ID OUT.mp4 HH:MM:SS=SECONDS..." +ID=$1; OUT=$2; shift 2 +need LIVE_BUCKET +key="$PREFIX/onair/clip-$(date -u +%H%M%S).mp4" +ssm "$ID" 1800 "$O/box/clip.sh" "$(presign put "$key" 3600)" "$@" +aws s3 cp --only-show-errors "s3://$BUCKET/$key" "$OUT"; ls -la "$OUT" diff --git a/live/onair/deploy-server.sh b/live/onair/deploy-server.sh new file mode 100755 index 0000000..b5e5fe7 --- /dev/null +++ b/live/onair/deploy-server.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# deploy-server.sh ID: installs the live server on the on-air box as airops-live-server (enabled, Restart=always, +# 127.0.0.1:8765), with the prebuilt days, the viewer assets and cache/inputs (upload-data.sh puts them in +# s3://LIVE_BUCKET/LIVE_S3_PREFIX/onair/), and the CPU priority drop-ins for the stream units. Needs the runner's +# setup.sh done first. Safe to run again. +set -euo pipefail +O="$(cd "$(dirname "$0")" && pwd)" +source "$O/../aws/common.sh" +ID=${1:?usage: deploy-server.sh ID} +need LIVE_BUCKET +tmp=$(mktemp -d) +COPYFILE_DISABLE=1 tar --no-xattrs -czf "$tmp/onair.tgz" -C "$O/box" units +aws s3 cp --only-show-errors "$tmp/onair.tgz" "s3://$BUCKET/$PREFIX/onair/onair.tgz"; rm -rf "$tmp" +ssm "$ID" 1800 "$O/box/server-setup.sh" "$(presign get "$PREFIX/onair/onair.tgz" 3600)" \ + "$(presign get "$PREFIX/onair/prebuilt.tar.zst" 3600)" "$(presign get "$PREFIX/onair/assets.tar.zst" 3600)" \ + "$(presign get "$PREFIX/onair/inputs.tar.zst" 3600)" diff --git a/live/onair/fresh.sh b/live/onair/fresh.sh new file mode 100755 index 0000000..07a2478 --- /dev/null +++ b/live/onair/fresh.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# fresh.sh ID: installs box/fresh.py on box ID as airops-fresh (enabled, restarted on exit): each minute the stream's +# picture against the page, logged to /var/log/airops-stream/fresh.jsonl; three stale minutes in a row restart the +# encoder. Start the stream first. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: fresh.sh ID} +ssm "$ID" 300 "$H/box/fresh-install.sh" "$(base64 < "$H/box/fresh.py" | tr -d '\n')" diff --git a/live/onair/grab.sh b/live/onair/grab.sh new file mode 100755 index 0000000..ee3db1d --- /dev/null +++ b/live/onair/grab.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# grab.sh ID DEST_DIR HH:MM:SS[=name]...: frames from the box's recording at those UTC times (today, or a full +# YYYY-MM-DDTHH:MM:SS), into DEST_DIR. +set -euo pipefail +O="$(cd "$(dirname "$0")" && pwd)" +source "$O/../aws/common.sh" +[ $# -ge 3 ] || die "usage: grab.sh ID DEST_DIR HH:MM:SS[=name]..." +ID=$1; DEST=$2; shift 2 +need LIVE_BUCKET +key="$PREFIX/onair/grab-$(date -u +%H%M%S).tar" +ssm "$ID" 600 "$O/box/grab.sh" "$(presign put "$key" 3600)" "$@" +mkdir -p "$DEST"; aws s3 cp --only-show-errors "s3://$BUCKET/$key" - | tar -xf - -C "$DEST" diff --git a/live/onair/health.sh b/live/onair/health.sh new file mode 100755 index 0000000..d4bfb52 --- /dev/null +++ b/live/onair/health.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# health.sh ID: box/health.sh on box ID, one JSON line: the stream's picture against the page, the runner, the live +# server, capture and push. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: health.sh ID} +ssm "$ID" 120 "$H/box/health.sh" diff --git a/live/onair/prep.sh b/live/onair/prep.sh new file mode 100755 index 0000000..f5ffaf1 --- /dev/null +++ b/live/onair/prep.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# prep.sh ID: box/docker-pools.sh on box ID, before the runner's setup.sh: the Deep Learning AMI ships Docker, so +# setup.sh leaves its daemon.json alone; this adds the address pools the env's networks need, keeping the NVIDIA runtime. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: prep.sh ID} +wait_ssm "$ID" +ssm "$ID" 300 "$H/box/docker-pools.sh" diff --git a/live/onair/sync.sh b/live/onair/sync.sh new file mode 100755 index 0000000..aaf7e2f --- /dev/null +++ b/live/onair/sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# sync.sh ID [REPO_DIR] [BRANCH]: checks the box's plugin out at BRANCH's head (default: the checked-out branch of the +# checkout this script is in) without re-running setup: web/ changes are served at once; restart airops-live-server for +# server changes and the runner (between days) for runner changes. The env and desk images stay as registered (re-run +# the runner's setup.sh for those). +set -euo pipefail +O="$(cd "$(dirname "$0")" && pwd)" +source "$O/../aws/common.sh" +ID=${1:?usage: sync.sh ID [REPO_DIR] [BRANCH]}; DIR=${2:-$(git -C "$O" rev-parse --show-toplevel)} +BRANCH=${3:-$(git -C "$DIR" rev-parse --abbrev-ref HEAD)} +need LIVE_BUCKET +tmp=$(mktemp -d); key="$PREFIX/onair/sync-$(date -u +%Y%m%dT%H%M%SZ).bundle" +git -C "$DIR" bundle create "$tmp/live.bundle" "$BASE_REF..$BRANCH" 2>/dev/null +aws s3 cp --only-show-errors "$tmp/live.bundle" "s3://$BUCKET/$key" +ssm "$ID" 300 "$O/box/sync.sh" "$(presign get "$key" 3600)" "$BRANCH" diff --git a/live/onair/upload-data.sh b/live/onair/upload-data.sh new file mode 100755 index 0000000..b14e51f --- /dev/null +++ b/live/onair/upload-data.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# upload-data.sh PREBUILT_DIR ASSETS_DIR [REPO_DIR]: packs what the live server reads and uploads it to +# s3://LIVE_BUCKET/LIVE_S3_PREFIX/onair/, where deploy-server.sh takes it: PREBUILT_DIR (`live_server.py prebuild`'s +# news/, traffic/ and radar/), ASSETS_DIR (a viewer folder's stage/, radar/ and twin/, e.g. the dataset's viewer/) and +# REPO_DIR's cache/inputs (the raw inputs; default: the checkout this script is in). +set -euo pipefail +O="$(cd "$(dirname "$0")" && pwd)" +source "$O/../aws/common.sh" +[ $# -ge 2 ] || die "usage: upload-data.sh PREBUILT_DIR ASSETS_DIR [REPO_DIR]" +PRE=$1; ASSETS=$2 +REPO=${3:-$(git -C "$O" rev-parse --show-toplevel)} +need LIVE_BUCKET +for d in "$PRE/news" "$PRE/traffic" "$ASSETS/stage" "$ASSETS/twin" "$REPO/cache/inputs"; do [ -d "$d" ] || die "no $d"; done +tmp=$(mktemp -d) +pack() { local out=$1 dir=$2; shift 2; COPYFILE_DISABLE=1 tar --no-xattrs -cf - -C "$dir" "$@" | zstd -q -T0 -3 > "$tmp/$out"; } +pre=(news traffic); [ -d "$PRE/radar" ] && pre+=(radar) +assets=(stage twin); [ -d "$ASSETS/radar" ] && assets+=(radar) +pack prebuilt.tar.zst "$PRE" "${pre[@]}" +pack assets.tar.zst "$ASSETS" "${assets[@]}" +pack inputs.tar.zst "$REPO/cache" inputs +for f in prebuilt assets inputs; do + log "$f.tar.zst: $(du -h "$tmp/$f.tar.zst" | cut -f1)" + aws s3 cp --only-show-errors "$tmp/$f.tar.zst" "s3://$BUCKET/$PREFIX/onair/$f.tar.zst" +done +rm -rf "$tmp" diff --git a/live/runner/box/pack.sh b/live/runner/box/pack.sh new file mode 100644 index 0000000..b33fb35 --- /dev/null +++ b/live/runner/box/pack.sh @@ -0,0 +1,4 @@ +set -euo pipefail +URL=$1; NAME=$(python3 -c "import json; print(json.load(open('/home/ubuntu/live/current.json'))['pilot'])") +cd /home/ubuntu && tar -cf - live "agentenv-airops-plugin/results/runs/$NAME" | zstd -q -T0 -3 > /tmp/live-out.tar.zst +curl -s -o /dev/null -w "put $NAME http=%{http_code}\n" -X PUT --upload-file /tmp/live-out.tar.zst "$URL" diff --git a/live/runner/box/plan.sh b/live/runner/box/plan.sh new file mode 100644 index 0000000..9ffef13 --- /dev/null +++ b/live/runner/box/plan.sh @@ -0,0 +1,5 @@ +set -uo pipefail +cd /home/ubuntu/agentenv-airops-plugin || exit 1 +echo "runner: $(systemctl is-active airops-live)" +curl -s --max-time 5 http://127.0.0.1:8765/live/state.json | python3 -c 'import json,sys; l=json.load(sys.stdin).get("live",{}); print("server:", l.get("status"), l.get("watch"), l.get("exact"), l.get("final"))' +sudo -u ubuntu /home/ubuntu/no-model.sh .venv/bin/python scripts/live_runner.py run --current /tmp/dry-current.json --dry-run "$@" 2>&1 | tail -40 diff --git a/live/runner/box/setup.sh b/live/runner/box/setup.sh new file mode 100644 index 0000000..493fd77 --- /dev/null +++ b/live/runner/box/setup.sh @@ -0,0 +1,63 @@ +set -euo pipefail +REPO_URL=$1; BUNDLE_URL=$2; REF=$3; PINS=$4; PYVER=$5; LITELLM_URL=$6; KEY_PARAM=$7; REGION=$8 +T0=$(date +%s); log() { echo "[$(( $(date +%s) - T0 ))s] $*"; } +systemctl stop unattended-upgrades apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true +cloud-init status --wait >/dev/null 2>&1 || true +for i in $(seq 1 200); do fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock >/dev/null 2>&1 || break; sleep 3; done +if ! command -v docker >/dev/null; then + apt-get -o DPkg::Lock::Timeout=600 update -qq && DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=600 install -y -qq docker.io docker-buildx docker-compose-v2 git zstd jq >/dev/null + mkdir -p /etc/docker + printf '{"default-address-pools":[{"base":"172.20.0.0/14","size":24}]}\n' > /etc/docker/daemon.json + systemctl enable docker >/dev/null 2>&1; systemctl restart docker + usermod -aG docker ubuntu +fi +printf "fs.inotify.max_user_instances=8192\nfs.inotify.max_user_watches=1048576\n" > /etc/sysctl.d/90-inotify.conf; sysctl -q --system +docker compose version >/dev/null 2>&1 || DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=600 install -y -qq docker-compose-v2 >/dev/null +command -v aws >/dev/null || snap install aws-cli --classic >/dev/null +log "docker $(docker --version | cut -d' ' -f3) compose $(docker compose version --short) aws $(aws --version 2>&1 | cut -d' ' -f1)" +docker ps --format '{{.Names}}' | grep -q '^agentenv-registry$' || \ + docker run -d --restart unless-stopped -p 5000:5000 --name agentenv-registry public.ecr.aws/docker/library/registry:2 >/dev/null +log "registry up" +install -d -o ubuntu -g ubuntu /home/ubuntu/live +cat > /home/ubuntu/with-model.sh <&2; exit 1; } +cd /home/ubuntu/agentenv-airops-plugin +exec "\$@" +W +cat > /home/ubuntu/no-model.sh <<'W' +#!/bin/bash +export LITELLM_BASE_URL=http://127.0.0.1:9 LITELLM_API_KEY=sk-no-model +cd /home/ubuntu/agentenv-airops-plugin +exec "$@" +W +chmod 755 /home/ubuntu/with-model.sh /home/ubuntu/no-model.sh; chown ubuntu:ubuntu /home/ubuntu/with-model.sh /home/ubuntu/no-model.sh +curl -sf -o /tmp/live.bundle "$BUNDLE_URL"; chmod 644 /tmp/live.bundle +cat > /tmp/user-setup.sh </dev/null || curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1 +export PATH=\$HOME/.local/bin:\$PATH +[ -d agentenv-airops-plugin ] || git clone -q $REPO_URL agentenv-airops-plugin +cd agentenv-airops-plugin && git fetch -q --tags origin && git fetch -q /tmp/live.bundle "refs/heads/$REF" && git checkout -q --detach FETCH_HEAD +[ -x .venv/bin/python ] || uv venv -q -p $PYVER .venv +uv sync -q -p .venv/bin/python --all-extras +uv pip install -q -p .venv/bin/python $PINS +echo "repo at \$(git log --oneline -1)" +.venv/bin/python -c "import importlib.metadata as m, platform; print('python', platform.python_version(), 'framework', m.version('agentenv-framework'), 'protocol', m.version('agentenv-framework-protocol'))" +.venv/bin/agent-env airops-occ3 setup > /home/ubuntu/live/occ3-setup.log 2>&1 || { tail -30 /home/ubuntu/live/occ3-setup.log; exit 1; } +grep -E "^Registered" /home/ubuntu/live/occ3-setup.log +.venv/bin/python scripts/live_runner.py run --current /home/ubuntu/live/current.json --dry-run | tail -3 +U +chmod 644 /tmp/user-setup.sh +sudo -iu ubuntu bash /tmp/user-setup.sh +n=$(aws ssm get-parameter --region "$REGION" --name "$KEY_PARAM" --with-decryption --query 'length(Parameter.Value)' --output text 2>/dev/null || echo "none yet") +log "key at $KEY_PARAM: $n chars" +if [ -n "$LITELLM_URL" ]; then + log "litellm health: $(curl -s -o /dev/null -w '%{http_code}' --max-time 15 "$LITELLM_URL/health/liveliness")" +else + log "no LIVE_LITELLM_BASE_URL: scripted desks only" +fi +s=$(grep USEC /run/systemd/shutdown/scheduled 2>/dev/null | cut -d= -f2 | cut -c1-10); log "box powers off: $([ -n "$s" ] && date -u -d @$s +%FT%TZ || echo never)" +log "setup done" diff --git a/live/runner/box/start.sh b/live/runner/box/start.sh new file mode 100644 index 0000000..eb3e43a --- /dev/null +++ b/live/runner/box/start.sh @@ -0,0 +1,13 @@ +set -euo pipefail +WRAP=/home/ubuntu/with-model.sh; prev= +for a in "$@"; do [ "$prev" = --policy ] && [ "$a" = script ] && WRAP=/home/ubuntu/no-model.sh; prev=$a; done +if systemctl is-active -q airops-live; then echo "airops-live is already running"; exit 1; fi +systemctl reset-failed airops-live 2>/dev/null || true +rm -f /home/ubuntu/live/stop.flag +systemd-run --unit airops-live --uid ubuntu --gid ubuntu --property=WorkingDirectory=/home/ubuntu/agentenv-airops-plugin \ + --setenv=HOME=/home/ubuntu --setenv=PATH=/home/ubuntu/.local/bin:/usr/local/bin:/usr/bin:/bin:/snap/bin \ + --property=SupplementaryGroups=docker --property=KillMode=process --property=TimeoutStopSec=7200 \ + --property=Restart=on-failure --property=RestartSec=30 \ + --property=StandardOutput=append:/home/ubuntu/live/runner.log --property=StandardError=append:/home/ubuntu/live/runner.log \ + "$WRAP" .venv/bin/python scripts/live_runner.py run --current /home/ubuntu/live/current.json "$@" +sleep 5; echo "airops-live: $(systemctl is-active airops-live) ($(basename $WRAP))"; tail -5 /home/ubuntu/live/runner.log diff --git a/live/runner/box/status.sh b/live/runner/box/status.sh new file mode 100644 index 0000000..01926a4 --- /dev/null +++ b/live/runner/box/status.sh @@ -0,0 +1,24 @@ +set -uo pipefail +N=${1:-15} +echo "unit: $(systemctl is-active airops-live 2>/dev/null) restarts $(systemctl show airops-live -p NRestarts --value 2>/dev/null)$( [ -f /home/ubuntu/live/stop.flag ] && echo ', stop.flag set')" +python3 - <<'P' +import json, time +from pathlib import Path +c = Path("/home/ubuntu/live/current.json") +if not c.is_file(): + print("current.json: none yet") + raise SystemExit +d = json.loads(c.read_text()) +print(f"current: {d['status']} day {d.get('day')} {d.get('date')} {d.get('run_id')} seq {d['seq']} at {d['updated_utc']}" + f"{' FINAL: ' + str(d.get('reason')) if d.get('final') else ''}") +print(f" instance {d.get('instance_id')}; env {(d.get('env') or {}).get('base')}") +print(f" spent ${d['spent_usd']:.2f} + live ${d.get('live_usd') or 0:.2f} {d.get('desks') or ''} of ${d['budget_usd']:g}; next {d.get('next')}") +for h in d["history"][-4:]: + print(f" {h['date']} {h['outcome']} reward {h['reward']} ${h['cost_usd']} {h['wall_seconds']}s {h['attempts']} attempt(s)") +r = Path(f"/home/ubuntu/agentenv-airops-plugin/results/runs/{d['pilot']}/results.jsonl") +rows = [json.loads(x) for x in r.read_text().splitlines() if x.strip()] if r.is_file() else [] +print(f"results: {len(rows)} attempts, {sum(x['outcome'] == 'scored' for x in rows)} scored, {len(d['history'])} days settled") +P +echo "load $(cut -d' ' -f1-3 /proc/loadavg); mem used GB $(free -g | awk '/Mem/{print $3}'); disk $(df -h / | awk 'NR==2{print $5}'); containers $(docker ps -q | wc -l)" +s=$(grep USEC /run/systemd/shutdown/scheduled 2>/dev/null | cut -d= -f2 | cut -c1-10); echo "powers off: $([ -n "$s" ] && date -u -d @$s +%FT%TZ || echo never)" +tail -$N /home/ubuntu/live/runner.log 2>/dev/null | sed -E 's/sk-[A-Za-z0-9_-]{8,}//g' diff --git a/live/runner/box/stop.sh b/live/runner/box/stop.sh new file mode 100644 index 0000000..ab2a92a --- /dev/null +++ b/live/runner/box/stop.sh @@ -0,0 +1,8 @@ +set -uo pipefail +case ${1:-} in + now) systemctl kill --kill-who=main -s TERM airops-live; sleep 2; systemctl kill --kill-who=main -s TERM airops-live + echo "airops-live: stopping the day now, then exiting (agent-env tears the day down first)" ;; + cancel) rm -f /home/ubuntu/live/stop.flag; echo "stop cancelled: the runner goes on" ;; + *) sudo -u ubuntu touch /home/ubuntu/live/stop.flag; echo "airops-live finishes the day in progress, then exits" ;; +esac +sleep 3; tail -3 /home/ubuntu/live/runner.log diff --git a/live/runner/fetch.sh b/live/runner/fetch.sh new file mode 100755 index 0000000..f9823e2 --- /dev/null +++ b/live/runner/fetch.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# fetch.sh ID [DEST]: copy the box's /home/ubuntu/live and the pilot folder (results.jsonl, days/, transcripts/, logs/, +# bundle/, pilot.json) to DEST (default live/runner/fetched/), through S3. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: fetch.sh ID [DEST]}; DEST=${2:-$H/fetched/$(date -u +%Y%m%dT%H%M%SZ)} +need LIVE_BUCKET +key="$PREFIX/out-$(date -u +%Y%m%dT%H%M%SZ).tar.zst" +ssm "$ID" 1800 "$H/box/pack.sh" "$(presign put "$key" 3600)" +tmp=$(mktemp -d); aws s3 cp --only-show-errors "s3://$BUCKET/$key" "$tmp/o.tar.zst" +mkdir -p "$DEST"; zstd -dc "$tmp/o.tar.zst" | tar -xf - -C "$DEST"; rm -rf "$tmp"; log "fetched into $DEST" diff --git a/live/runner/plan.sh b/live/runner/plan.sh new file mode 100755 index 0000000..5428e7d --- /dev/null +++ b/live/runner/plan.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# plan.sh ID [runner args]: the schedule `live_runner.py run --dry-run` projects on box ID with these args (the same as +# start.sh's, e.g. --set start_utc=… --set budget_usd=… --set slot_minutes=…): when each day starts and the spend +# before it, against a throwaway current.json. Plays nothing and spends nothing. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: plan.sh ID [runner args]}; shift +ssm "$ID" 300 "$H/box/plan.sh" ${LIVE_MODEL:+--set "model=$LIVE_MODEL"} "$@" diff --git a/live/runner/setup.sh b/live/runner/setup.sh new file mode 100755 index 0000000..f46ee7a --- /dev/null +++ b/live/runner/setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# setup.sh ID [REPO_DIR] [BRANCH]: make box ID a run box. Ships BRANCH (default: REPO_DIR's checked-out branch; REPO_DIR +# defaults to the checkout this script is in) as a git bundle over LIVE_BASE_REF through S3, then on the box: Docker, +# compose v2, the inotify sysctl, aws-cli, a registry:2 on :5000, the plugin at BRANCH in a py3.12 venv with LIVE_PINS +# (the framework pinned), `agent-env airops-occ3 setup`, with-model.sh (LIVE_LITELLM_BASE_URL, the key read from the +# SSM parameter LIVE_LITELLM_KEY_PARAM at every start) and no-model.sh (the scripted desks), and a dry run of the +# schedule. Safe to run again. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: setup.sh ID [REPO_DIR] [BRANCH]}; DIR=${2:-$(git -C "$H" rev-parse --show-toplevel)} +BRANCH=${3:-$(git -C "$DIR" rev-parse --abbrev-ref HEAD)} +need LIVE_BUCKET +PINS=${LIVE_PINS:-agentenv-framework==0.9.1339 agentenv-framework-protocol==0.1.354 highspy==1.15.1 numpy==2.5.4 pandas==3.0.6 pyarrow==26.0.0 shapely==2.2.0 airportsdata==20260905} +tmp=$(mktemp -d); key="$PREFIX/runner-$(date -u +%Y%m%dT%H%M%SZ).bundle" +git -C "$DIR" bundle create "$tmp/live.bundle" "$BASE_REF..$BRANCH" 2>/dev/null +log "bundle $BRANCH at $(git -C "$DIR" rev-parse --short "$BRANCH"): $(du -h "$tmp/live.bundle" | cut -f1)" +code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --upload-file "$tmp/live.bundle" "$(presign put "$key" 3600)") +[ "$code" = 200 ] || die "bundle upload: http $code" +wait_ssm "$ID" +ssm "$ID" 3000 "$H/box/setup.sh" "$REPO_URL" "$(presign get "$key" 3600)" "$BRANCH" "$PINS" 3.12.12 \ + "${LIVE_LITELLM_BASE_URL:-}" "${LIVE_LITELLM_KEY_PARAM:-/airops-live/litellm-key}" "$AWS_REGION" diff --git a/live/runner/start.sh b/live/runner/start.sh new file mode 100755 index 0000000..765755b --- /dev/null +++ b/live/runner/start.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# start.sh ID [runner args]: start the runner on box ID as the systemd unit airops-live (restarted on a crash, which +# picks the running day up; `systemctl stop` finishes the day first). Args go to `live_runner.py run`, e.g. +# start.sh ID the stream: the config's days and budget +# start.sh ID --set start_utc=2026-10-23T16:00:00Z hold the first day until then +# start.sh ID --policy script --name live-script-box --set slot_minutes=6 --days 3 a $0 rehearsal +# start.sh ID --name occ3-live-rehearsal --days 1 one real Astra day (~$23) in its own pilot +# LIVE_MODEL, when set, comes first as --set model=…, so a later --set model= wins. current.json, stop.flag and +# runner.log are in /home/ubuntu/live/. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: start.sh ID [runner args]}; shift +ssm "$ID" 120 "$H/box/start.sh" ${LIVE_MODEL:+--set "model=$LIVE_MODEL"} "$@" diff --git a/live/runner/status.sh b/live/runner/status.sh new file mode 100755 index 0000000..c815c7a --- /dev/null +++ b/live/runner/status.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# status.sh ID [N]: the unit, current.json in a few lines (day, instance, env, spend, next, the last days), the results, +# the box's load and power-off time, and the last N lines of runner.log. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: status.sh ID [N]} +ssm "$ID" 60 "$H/box/status.sh" "${2:-15}" diff --git a/live/runner/stop.sh b/live/runner/stop.sh new file mode 100755 index 0000000..b51eb1b --- /dev/null +++ b/live/runner/stop.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# stop.sh ID [now|cancel]: the runner on box ID finishes the day in progress and exits (stop.flag); `now` stops the day +# at once (agent-env tears it down; the attempt is recorded and counts as one of the day's two); `cancel` takes +# stop.flag back. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: stop.sh ID [now|cancel]} +ssm "$ID" 60 "$H/box/stop.sh" "${2:-}" diff --git a/live/stream/box/airops-stream.env b/live/stream/box/airops-stream.env new file mode 100644 index 0000000..995dc20 --- /dev/null +++ b/live/stream/box/airops-stream.env @@ -0,0 +1,18 @@ +# /etc/airops-stream.env: read by the airops-* units; start.sh edits it. Restart a unit after editing by hand. +AIROPS_URL=http://127.0.0.1:8765/web/index.html?live=http://127.0.0.1:8765&stream +AIROPS_SIZE=1920x1080 +AIROPS_FPS=30 +AIROPS_PAGE_FPS=0 +AIROPS_VBITRATE=6000k +AIROPS_ABITRATE=128k +AIROPS_DISPLAY=:0 +AIROPS_RELAY=rtmp://127.0.0.1:1935/airops +AIROPS_TARGET=none +AIROPS_SSM_SERVER=/airops-live/x-server +AIROPS_SSM_KEY=/airops-live/x-key +AIROPS_SAFE_URL=http://127.0.0.1:8765/live/state.json +AIROPS_MEM_SOFT_MB=6000 +AIROPS_MEM_HARD_MB=9000 +AIROPS_RESTART_HOURS=0 +AIROPS_MIN_FREE_GB=8 +AWS_DEFAULT_REGION=us-west-2 diff --git a/live/stream/box/display-ready.sh b/live/stream/box/display-ready.sh new file mode 100755 index 0000000..ddc5465 --- /dev/null +++ b/live/stream/box/display-ready.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Waits for the display, then shows the slate over the whole root window, not just the output the driver sets up +# (1024x768), which is what the stream shows while Chrome restarts. +export DISPLAY=${AIROPS_DISPLAY:-:0} +for _ in $(seq 1 60); do xset q >/dev/null 2>&1 && break; sleep 0.5; done +xset s off s noblank -dpms || true +hsetroot -root -fill /opt/airops-stream/slate.png >/dev/null || true diff --git a/live/stream/box/display.sh b/live/stream/box/display.sh new file mode 100755 index 0000000..72ab0ba --- /dev/null +++ b/live/stream/box/display.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Xorg on the NVIDIA GPU with a virtual screen of AIROPS_SIZE and no monitor, no screen saver and no DPMS. +set -euo pipefail +SIZE=${AIROPS_SIZE:-1920x1080} +BUS=$(nvidia-smi --query-gpu=pci.bus_id --format=csv,noheader | head -1 \ + | awk -F'[:.]' '{printf "PCI:%d:%d:%d", strtonum("0x"$2), strtonum("0x"$3), strtonum("0x"$4)}') +cat > /etc/X11/xorg-airops.conf </dev/null || { echo "no NVIDIA GPU or driver: use the Deep Learning Base OSS NVIDIA driver AMI" >&2; exit 1; } +for _ in $(seq 1 60); do fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock >/dev/null 2>&1 || break; sleep 5; done +systemctl stop unattended-upgrades 2>/dev/null || true +apt-get -o Acquire::Retries=5 update -qq 2>/dev/null +apt-get -o Acquire::Retries=5 install -y -qq --no-install-recommends xserver-xorg-core x11-xserver-utils x11-utils hsetroot \ + pulseaudio pulseaudio-utils ffmpeg fonts-noto-core fonts-dejavu-core fonts-noto-color-emoji python3-websocket \ + python3-boto3 python3-pil jq curl mesa-utils >/dev/null +if ! command -v google-chrome >/dev/null; then + curl -fsSL -o /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + apt-get install -y -qq /tmp/chrome.deb >/dev/null +fi +id streamer >/dev/null 2>&1 || useradd -m -s /bin/bash streamer +usermod -aG video,audio,render streamer 2>/dev/null || true +mkdir -p /opt/airops-stream +curl -fsSL "$PAYLOAD_URL" | tar xz -C /opt/airops-stream +if [ ! -x /opt/airops-stream/mediamtx ] || ! /opt/airops-stream/mediamtx --version 2>&1 | grep -q "$MEDIAMTX"; then + curl -fsSL "https://github.com/bluenviron/mediamtx/releases/download/$MEDIAMTX/mediamtx_${MEDIAMTX}_linux_amd64.tar.gz" \ + | tar xz -C /opt/airops-stream mediamtx +fi +if [ ! -f /etc/airops-stream.env ]; then + cp /opt/airops-stream/airops-stream.env /etc/airops-stream.env + [ "$(nproc)" -le 4 ] && sed -i 's/^AIROPS_PAGE_FPS=.*/AIROPS_PAGE_FPS=30/' /etc/airops-stream.env +fi +for k in AIROPS_SSM_SERVER AIROPS_SSM_KEY AWS_DEFAULT_REGION; do + v=$(grep "^$k=" /opt/airops-stream/airops-stream.env | cut -d= -f2-) + if grep -q "^$k=" /etc/airops-stream.env; then sed -i "s|^$k=.*|$k=$v|" /etc/airops-stream.env; else echo "$k=$v" >> /etc/airops-stream.env; fi +done +python3 /opt/airops-stream/slate.py /opt/airops-stream/slate.png "$(grep ^AIROPS_SIZE= /etc/airops-stream.env | cut -d= -f2)" +install -d -o streamer -g streamer /var/lib/airops-stream /var/lib/airops-stream/rec /var/log/airops-stream +cp /opt/airops-stream/units/*.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable -q airops-x airops-relay airops-capture airops-push +if [ -n "$BUNDLE_URL" ]; then + rm -rf /opt/airops-view && mkdir -p /opt/airops-view && curl -fsSL "$BUNDLE_URL" | tar xz -C /opt/airops-view + systemctl enable -q --now airops-static +fi +echo "chrome: $(google-chrome --version)" +echo "mediamtx: $(/opt/airops-stream/mediamtx --version 2>&1 | head -1)" +echo "ffmpeg: $(ffmpeg -version | head -1)" +echo "gpu: $(nvidia-smi --query-gpu=name,driver_version --format=csv,noheader)" +echo "setup done; start.sh starts the stream" diff --git a/live/stream/box/slate.py b/live/stream/box/slate.py new file mode 100644 index 0000000..c9d3150 --- /dev/null +++ b/live/stream/box/slate.py @@ -0,0 +1,14 @@ +"""Draws the slate the stream shows while Chrome (re)starts: slate.py OUT.png WIDTHxHEIGHT.""" +import sys + +from PIL import Image, ImageDraw, ImageFont + +w, h = (int(x) for x in sys.argv[2].split("x")) +img = Image.new("RGB", (w, h), (11, 18, 32)) +d = ImageDraw.Draw(img) +big = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", h // 14) +small = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", h // 36) +for text, font, y in (("AirOpsEnv live", big, 0.42), ("Back in a moment", small, 0.56)): + box = d.textbbox((0, 0), text, font=font) + d.text(((w - box[2]) / 2, h * y), text, font=font, fill=(226, 232, 240)) +img.save(sys.argv[1]) diff --git a/live/stream/box/start.sh b/live/stream/box/start.sh new file mode 100755 index 0000000..562b778 --- /dev/null +++ b/live/stream/box/start.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# start.sh [--url URL] [--target none|ssm|rtmp(s)://...] [--size WxH] [--bitrate 6000k] [--safe-url URL]: sets what is +# given in /etc/airops-stream.env, starts what is stopped and restarts what a change concerns. --target ssm streams to +# X with the server and key in SSM (AIROPS_SSM_SERVER, AIROPS_SSM_KEY); none only records. Changing only the +# target restarts only the push. +set -euo pipefail +ENVF=/etc/airops-stream.env +changed=$(python3 - "$ENVF" "$@" <<'PY' +import sys +path, args = sys.argv[1], sys.argv[2:] +keys = {"--url": "AIROPS_URL", "--target": "AIROPS_TARGET", "--size": "AIROPS_SIZE", "--bitrate": "AIROPS_VBITRATE", + "--safe-url": "AIROPS_SAFE_URL"} +if len(args) % 2 or any(a not in keys for a in args[::2]): + sys.exit(f"usage: start.sh [{' X] ['.join(keys)} X]") +new = {keys[a]: v for a, v in zip(args[::2], args[1::2])} +lines = open(path).read().splitlines() +out = [f"{l.split('=', 1)[0]}={new.pop(l.split('=', 1)[0])}" if l.split("=", 1)[0] in new else l for l in lines] +out += [f"{k}={v}" for k, v in new.items()] +open(path, "w").write("\n".join(out) + "\n") +print(" ".join(keys[a] for a in args[::2])) +PY +) +if [[ " $changed " == *" AIROPS_SIZE "* ]]; then + python3 /opt/airops-stream/slate.py /opt/airops-stream/slate.png "$(grep ^AIROPS_SIZE= $ENVF | cut -d= -f2)" + systemctl restart airops-x +fi +systemctl start airops-x airops-relay +for unit in airops-capture airops-push; do + if ! systemctl is-active -q $unit; then + systemctl start $unit + elif [ -n "$changed" ] && { [ $unit = airops-push ] || [ "$changed" != AIROPS_TARGET ]; }; then + systemctl restart $unit + fi +done +sleep 3 +grep -E "^AIROPS_(URL|TARGET|SIZE|VBITRATE|SAFE_URL)=" $ENVF +echo "units: $(systemctl is-active airops-x airops-relay airops-capture airops-push | paste -sd' ') (x relay capture push)" diff --git a/live/stream/box/status.sh b/live/stream/box/status.sh new file mode 100755 index 0000000..87ce9a7 --- /dev/null +++ b/live/stream/box/status.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# status.sh [MINUTES]: the units, the relay's paths, the last metrics of capture and push, restarts and warnings in +# the last MINUTES (default 30), GPU, memory and disk. +M=${1:-30} +echo "units: $(systemctl is-active airops-x airops-relay airops-capture airops-push | paste -sd' ') (x relay capture push)" +grep -E "^AIROPS_(URL|TARGET|SIZE|VBITRATE)=" /etc/airops-stream.env +curl -fsS http://127.0.0.1:9997/v3/paths/list 2>/dev/null | jq -r '.items[] | "relay: \(.name) ready=\(.ready) in=\(.bytesReceived) out=\(.bytesSent) readers=\(.readers|length)"' \ + || echo "relay: not answering" +for m in capture push; do + f=/var/log/airops-stream/metrics-$m.jsonl + [ -f $f ] && echo "$m: $(tail -1 $f)" +done +echo "--- restarts and warnings, last $M min" +journalctl -u airops-capture -u airops-push -u airops-relay -u airops-x --since "-${M}min" --no-pager -o short-iso 2>/dev/null \ + | grep -Ei "restart|stall|error|fail|exited|warn|memory|disk" | grep -v metrics | tail -20 +echo "---" +nvidia-smi --query-gpu=utilization.gpu,utilization.encoder,memory.used,temperature.gpu --format=csv,noheader +free -m | awk 'NR==2{print "mem: used " $3 " MB, available " $7 " MB"}' +df -h /var/lib/airops-stream | awk 'NR==2{print "disk: " $4 " free"}' +ls -la /var/lib/airops-stream/rec/airops 2>/dev/null | tail -3 +cat /run/systemd/shutdown/scheduled 2>/dev/null | grep -E "USEC|MODE" | tr '\n' ' ' && echo "(a scheduled power-off is set: shutdown -c cancels it)" diff --git a/live/stream/box/stop.sh b/live/stream/box/stop.sh new file mode 100755 index 0000000..ffa4be6 --- /dev/null +++ b/live/stream/box/stop.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# stop.sh [--push-only]: stops pushing to the target (--push-only), or the whole stream: push, capture, relay and the +# display. The units stay enabled, so a reboot starts the stream again; stop.sh --disable keeps it stopped. +set -euo pipefail +case ${1:-} in + --push-only) systemctl stop airops-push ;; + --disable) systemctl disable -q --now airops-push airops-capture airops-relay airops-x ;; + *) systemctl stop airops-push airops-capture airops-relay airops-x ;; +esac +systemctl is-active airops-x airops-relay airops-capture airops-push | paste -sd' ' || true diff --git a/live/stream/box/streamer.py b/live/stream/box/streamer.py new file mode 100755 index 0000000..d9da68b --- /dev/null +++ b/live/stream/box/streamer.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""The AirOpsEnv live streamer on one box. `capture` keeps PulseAudio, Chrome on the viewer URL and an ffmpeg that +encodes the X display and the page's sound (over an airport bed, AIROPS_BED, ducked under the page and heard only +while the page is silent: a reload, a Chrome restart) into the local relay (mediamtx) running, and restarts whatever +dies, hangs or grows too big; `push` forwards the relay to the RTMP target (X's server and key from SSM, or any URL) +and reconnects with backoff. Both log a metrics line a minute to the journal and to AIROPS_LOG_DIR; neither prints +the stream key.""" + +import argparse +import json +import os +import shutil +import signal +import subprocess +import threading +import time +import urllib.request +from pathlib import Path + +import websocket + +E = os.environ +DISPLAY = E.get("AIROPS_DISPLAY", ":0") +SIZE = E.get("AIROPS_SIZE", "1920x1080") +FPS = int(E.get("AIROPS_FPS", "30")) +VBITRATE = E.get("AIROPS_VBITRATE", "6000k") +ABITRATE = E.get("AIROPS_ABITRATE", "128k") +RELAY = E.get("AIROPS_RELAY", "rtmp://127.0.0.1:1935/airops") +URL = E.get("AIROPS_URL", "") +TARGET = E.get("AIROPS_TARGET", "none") +SSM_SERVER = E.get("AIROPS_SSM_SERVER", "/airops-live/x-server") +SSM_KEY = E.get("AIROPS_SSM_KEY", "/airops-live/x-key") +SAFE_URL = E.get("AIROPS_SAFE_URL", "") +MEM_SOFT_MB = int(E.get("AIROPS_MEM_SOFT_MB", "6000")) +MEM_HARD_MB = int(E.get("AIROPS_MEM_HARD_MB", "9000")) +RESTART_HOURS = float(E.get("AIROPS_RESTART_HOURS", "0")) +CHROME_FLAGS = E.get("AIROPS_CHROME_FLAGS", "").split() +STATE = Path(E.get("AIROPS_STATE_DIR", "/var/lib/airops-stream")) +LOGS = Path(E.get("AIROPS_LOG_DIR", "/var/log/airops-stream")) +REC = Path(E.get("AIROPS_REC_DIR", str(STATE / "rec"))) +MIN_FREE_GB = float(E.get("AIROPS_MIN_FREE_GB", "8")) +CDP_PORT = int(E.get("AIROPS_CDP_PORT", "9222")) +PAGE_FPS = int(E.get("AIROPS_PAGE_FPS", "0")) +SINK = "broadcast" +BED = Path(E.get("AIROPS_BED", "/opt/airops-stream/bed.ogg")) +BED_DB = float(E.get("AIROPS_BED_DB", "-6")) +FMT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo" +FRAMES = """(() => { if (window.__airopsFrames !== undefined) return; + window.__airopsFrames = 0; window.__airopsLost = 0; + const f = () => { window.__airopsFrames++; requestAnimationFrame(f); }; requestAnimationFrame(f); + const get = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...rest) { + const ctx = get.call(this, type, ...rest); + if (ctx && /webgl/.test(type) && !this.__airopsWatched) { + this.__airopsWatched = true; + this.addEventListener("webglcontextlost", () => window.__airopsLost++); + } + return ctx; + }; })()""" +THROTTLE = """(() => { if (window.__airopsThrottle) return; window.__airopsThrottle = %d; + const raf = window.requestAnimationFrame.bind(window), gap = 1000 / %d - 4; + let queue = new Map(), id = 0, last = 0, pending = false; + const pump = (t) => { if (t - last < gap) { raf(pump); return; } last = t; pending = false; + const run = queue; queue = new Map(); for (const cb of run.values()) cb(t); }; + window.requestAnimationFrame = (cb) => { queue.set(++id, cb); + if (!pending) { pending = true; raf(pump); } return id; }; + window.cancelAnimationFrame = (i) => { queue.delete(i); }; })()""" +PROBE = """JSON.stringify({f: window.__airopsFrames || 0, err: !!document.querySelector('.error'), + lost: window.__airopsLost || 0, + ready: !!(window.airops && window.airops.state && window.airops.state().view), + heap: performance.memory ? performance.memory.usedJSHeapSize : 0})""" +SECRETS: dict[str, str] = {} + + +def log(*parts) -> None: + line = " ".join(str(p) for p in parts) + for secret, name in SECRETS.items(): + line = line.replace(secret, f"<{name}>") + print(line, flush=True) + + +def metrics(mode: str, row: dict) -> None: + row = {"t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), **row} + log("metrics", json.dumps(row)) + with open(LOGS / f"metrics-{mode}.jsonl", "a") as f: + f.write(json.dumps(row) + "\n") + + +class Backoff: + """Seconds to wait before the next restart: doubling from `lo` to `hi`, back to `lo` once a run lasted `healthy`.""" + + def __init__(self, lo: float = 1, hi: float = 60, healthy: float = 120): + self.lo, self.hi, self.healthy, self.cur, self.since = lo, hi, healthy, lo, None + + def started(self) -> None: + self.since = time.monotonic() + + def failed(self) -> float: + if self.since is not None and time.monotonic() - self.since >= self.healthy: + self.cur = self.lo + delay, self.cur, self.since = self.cur, min(self.hi, self.cur * 2), None + return delay + + +class Ffmpeg: + """An ffmpeg whose `-progress` blocks are parsed as they come and whose warnings go to the log, redacted. It counts + as progressing only while its frame count moves: x11grab on a display that went away keeps ffmpeg alive and + reporting, on the sound alone.""" + + def __init__(self, cmd: list[str], env: dict, name: str): + self.name, self.progress, self.last, self.mark = name, {}, time.monotonic(), None + self.proc = subprocess.Popen(cmd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + threading.Thread(target=self._read_progress, daemon=True).start() + threading.Thread(target=self._relay_stderr, daemon=True).start() + + def _read_progress(self) -> None: + block = {} + for raw in iter(self.proc.stdout.readline, b""): + k, _, v = raw.decode(errors="replace").strip().partition("=") + block[k] = v + if k == "progress": + mark = block.get("frame") or block.get("out_time_us") + if mark != self.mark: + self.mark, self.last = mark, time.monotonic() + self.progress, block = block, {} + + def _relay_stderr(self) -> None: + for raw in iter(self.proc.stderr.readline, b""): + log(f"[{self.name}]", raw.decode(errors="replace").rstrip()) + + def alive(self) -> bool: + return self.proc.poll() is None + + def stalled(self, seconds: float) -> bool: + return time.monotonic() - self.last > seconds + + def stop(self) -> None: + stop_group(self.proc) + + +def stop_group(proc: subprocess.Popen | None, grace: float = 10) -> None: + if proc is None or proc.poll() is not None: + return + try: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(grace) + except (ProcessLookupError, subprocess.TimeoutExpired): + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + +class Cdp: + def __init__(self, port: int, timeout: float = 10): + with urllib.request.urlopen(f"http://127.0.0.1:{port}/json", timeout=timeout) as r: + page = next(t for t in json.load(r) if t["type"] == "page" and not t["url"].startswith("devtools")) + self.ws = websocket.create_connection(page["webSocketDebuggerUrl"], suppress_origin=True, timeout=timeout) + self.n = 0 + + def call(self, method: str, **params): + self.n += 1 + self.ws.send(json.dumps({"id": self.n, "method": method, "params": params})) + while True: + m = json.loads(self.ws.recv()) + if m.get("id") == self.n: + if "error" in m: + raise RuntimeError(m["error"]) + return m["result"] + + def eval(self, expr: str): + r = self.call("Runtime.evaluate", expression=expr, returnByValue=True) + if "exceptionDetails" in r: + raise RuntimeError(r["exceptionDetails"].get("text")) + return r["result"].get("value") + + def close(self) -> None: + try: + self.ws.close() + except OSError: + pass + + +class Chrome: + """Chrome in kiosk mode on the viewer URL, a fresh profile each start, with a frame counter injected into every + document so a hung or crashed page shows as frames that stop advancing.""" + + def __init__(self, env: dict): + self.env, self.proc, self.cdp, self.backoff = env, None, None, Backoff(2, 60, 300) + self.restarts, self.frames, self.heap = 0, 0, 0 + self.started = self.progressed = self.reloaded = self.answered = 0.0 + self.lost_since = None + self.profile = STATE / "chrome" + + def start(self) -> None: + w, h = SIZE.split("x") + shutil.rmtree(self.profile, ignore_errors=True) + cmd = ["google-chrome", f"--user-data-dir={self.profile}", "--no-first-run", "--no-default-browser-check", + "--noerrdialogs", "--disable-infobars", "--hide-scrollbars", "--disable-session-crashed-bubble", + "--disable-component-update", "--password-store=basic", "--disable-features=Translate,MediaRouter", + "--kiosk", "--window-position=0,0", f"--window-size={w},{h}", + "--autoplay-policy=no-user-gesture-required", "--disable-background-timer-throttling", + "--disable-renderer-backgrounding", "--disable-backgrounding-occluded-windows", + "--ignore-gpu-blocklist", "--enable-gpu-rasterization", "--enable-zero-copy", + f"--remote-debugging-port={CDP_PORT}", *CHROME_FLAGS, "about:blank"] + self.proc = subprocess.Popen(cmd, env=self.env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + self.started = self.progressed = self.answered = time.monotonic() + self.frames = 0 + for _ in range(60): + time.sleep(0.5) + try: + self.attach() + break + except (OSError, RuntimeError, StopIteration, ValueError, websocket.WebSocketException): + self.cdp = None + if self.cdp is None: + log("chrome: DevTools never answered") + return + self.cdp.call("Page.navigate", url=URL) + self.backoff.started() + log(f"chrome: started (pid {self.proc.pid}) on {URL}") + + def attach(self) -> None: + """A DevTools session that injects the frame counter into this document and every later one (an injected + script lasts as long as the session that added it, and Chrome injects it only with the Page domain enabled).""" + self.cdp = Cdp(CDP_PORT, timeout=5) + self.cdp.call("Page.enable") + for script in ([THROTTLE % (PAGE_FPS, PAGE_FPS)] if PAGE_FPS else []) + [FRAMES]: + self.cdp.call("Page.addScriptToEvaluateOnNewDocument", source=script) + self.cdp.eval(script) + + def stop(self, why: str) -> None: + log(f"chrome: stopping ({why})") + if self.cdp: + self.cdp.close() + self.cdp = None + stop_group(self.proc) + subprocess.run(["pkill", "-KILL", "-f", f"--user-data-dir={self.profile}"], capture_output=True) + self.proc = None + + def restart(self, why: str) -> None: + self.stop(why) + self.restarts += 1 + delay = self.backoff.failed() + log(f"chrome: restart #{self.restarts} in {delay:.0f} s") + time.sleep(delay) + self.start() + + def pss_mb(self) -> int: + """The proportional set size of Chrome's processes, in MB (RSS would count shared pages once a process).""" + total = 0 + for pid in (p for p in os.listdir("/proc") if p.isdigit()): + try: + if str(self.profile).encode() not in Path(f"/proc/{pid}/cmdline").read_bytes(): + continue + for line in Path(f"/proc/{pid}/smaps_rollup").read_text().splitlines(): + if line.startswith("Pss:"): + total += int(line.split()[1]) + except OSError: + pass + return total // 1024 + + def check(self) -> None: + now = time.monotonic() + if self.proc is None or self.proc.poll() is not None: + self.restart(f"exited with {None if self.proc is None else self.proc.returncode}") + return + try: + if self.cdp is None: + self.attach() + p = json.loads(self.cdp.eval(PROBE)) + self.answered = now + except (OSError, RuntimeError, StopIteration, ValueError, TypeError, websocket.WebSocketException) as e: + if self.cdp: + self.cdp.close() + self.cdp = None + log(f"chrome: probe failed: {type(e).__name__} {e}") + if now - self.answered > 45 and now - max(self.started, self.reloaded) > 90: + self.restart("not answering for 45 s") + return + if p["f"] > self.frames or p["f"] < self.frames // 2: + self.progressed = now + self.frames, self.heap = p["f"], p["heap"] + self.lost_since = (self.lost_since or now) if p["lost"] > 0 else None + if self.lost_since and now - self.lost_since > 5: + log("chrome: a WebGL context was lost (the GPU process restarted?); reloading") + self.lost_since = None + self.reload() + elif p["err"] and now - self.reloaded > 30: + log("chrome: the viewer shows an error; reloading") + self.reload() + elif now - self.progressed > 20 and now - self.started > 60: + if now - self.reloaded > 90: + log("chrome: no frames for 20 s; reloading") + self.reload() + else: + self.restart("no frames after a reload") + + def reload(self) -> None: + self.reloaded = self.progressed = time.monotonic() + try: + self.cdp.call("Page.navigate", url=URL) + except (OSError, RuntimeError, AttributeError, websocket.WebSocketException) as e: + log(f"chrome: reload failed: {e}") + + +def safe_now() -> bool: + """Whether a restart would not cut a day short: SAFE_URL's status (or live.status) is intermission or done.""" + if not SAFE_URL: + return True + try: + with urllib.request.urlopen(SAFE_URL, timeout=5) as r: + d = json.load(r) + except (OSError, ValueError): + return False + return (d.get("status") or (d.get("live") or {}).get("status")) in ("intermission", "done") + + +def start_pulse(env: dict) -> subprocess.Popen: + """PulseAudio with one null sink as the default output: Chrome plays into it, ffmpeg reads its monitor.""" + Path(env["XDG_RUNTIME_DIR"]).mkdir(mode=0o700, parents=True, exist_ok=True) + subprocess.run(["pkill", "-u", str(os.getuid()), "-x", "pulseaudio"], capture_output=True) + proc = subprocess.Popen( + ["pulseaudio", "--daemonize=no", "--exit-idle-time=-1", "--disallow-exit", "-n", + "--load=module-native-protocol-unix", + f"--load=module-null-sink sink_name={SINK} rate=48000 sink_properties=device.description=Broadcast"], + env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) + for _ in range(50): + if not subprocess.run(["pactl", "set-default-sink", SINK], env=env, capture_output=True).returncode: + log(f"pulse: started (pid {proc.pid})") + return proc + time.sleep(0.2) + stop_group(proc) + raise RuntimeError("PulseAudio did not start") + + +def x_server() -> str: + """The X server's pid from its lock file: x11grab keeps reading the old server's shared memory after a restart.""" + try: + return Path(f"/tmp/.X{DISPLAY.lstrip(':').split('.')[0]}-lock").read_text().strip() + except OSError: + return "" + + +def encoder_cmd() -> list[str]: + rate = int(VBITRATE.rstrip("k")) + audio = ["-map", "1:a", "-af", "aresample=async=1000"] + if BED.is_file(): + audio = ["-filter_complex", + f"[1:a]aresample=48000:async=1000,{FMT},asplit=2[page][key];" + f"amovie=filename={BED}:loop=0,aresample=48000,{FMT},volume={BED_DB}dB[bed];" + "[bed][key]sidechaincompress=threshold=0.003:ratio=20:attack=20:release=1500[duck];" + f"[duck]{FMT}[bf];[page][bf]amerge=inputs=2,pan=stereo|c0=c0+c2|c1=c1+c3[a]", "-map", "[a]"] + return ["ffmpeg", "-hide_banner", "-loglevel", "warning", "-nostats", "-progress", "pipe:1", "-stats_period", "5", + "-thread_queue_size", "1024", "-f", "x11grab", "-framerate", str(FPS), "-video_size", SIZE, + "-draw_mouse", "0", "-i", f"{DISPLAY}.0", + "-thread_queue_size", "1024", "-f", "pulse", "-i", f"{SINK}.monitor", + "-map", "0:v", *audio, "-c:v", "h264_nvenc", "-preset", "p5", "-tune", "hq", "-profile:v", "high", + "-rc", "cbr", "-bf", "0", "-spatial-aq", "1", "-pix_fmt", "yuv420p", + "-b:v", VBITRATE, "-maxrate", VBITRATE, "-bufsize", f"{2 * rate}k", "-g", str(2 * FPS), + "-keyint_min", str(2 * FPS), + "-c:a", "aac", "-b:a", ABITRATE, "-ar", "44100", "-ac", "2", "-f", "flv", RELAY] + + +class Cpu: + def __init__(self): + self.prev = self._read() + + @staticmethod + def _read() -> tuple[int, int]: + f = [int(x) for x in Path("/proc/stat").read_text().split("\n", 1)[0].split()[1:]] + return sum(f), f[3] + f[4] + + def percent(self) -> float: + cur = self._read() + total, idle = cur[0] - self.prev[0], cur[1] - self.prev[1] + self.prev = cur + return round(100 * (1 - idle / total), 1) if total else 0.0 + + +def gpu() -> dict: + try: + out = subprocess.run(["nvidia-smi", + "--query-gpu=utilization.gpu,utilization.encoder,memory.used,temperature.gpu", + "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=10).stdout + u, enc, mem, temp = (x.strip() for x in out.split(",")) + return {"gpu_pct": int(u), "nvenc_pct": int(enc), "gpu_mem_mb": int(mem), "gpu_c": int(temp)} + except (OSError, ValueError, subprocess.TimeoutExpired): + return {} + + +def mem_available_mb() -> int: + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + return 0 + + +def guard_disk() -> None: + """Deletes the oldest recordings while the recording disk has less than AIROPS_MIN_FREE_GB free.""" + while REC.exists() and shutil.disk_usage(REC).free < MIN_FREE_GB * 2**30: + files = sorted((p for p in REC.rglob("*") if p.is_file()), key=lambda p: p.stat().st_mtime) + if len(files) < 2: + return + log(f"disk: under {MIN_FREE_GB} GB free; deleting {files[0]}") + files[0].unlink() + + +def capture() -> int: + if not URL: + log("AIROPS_URL is not set") + return 2 + LOGS.mkdir(parents=True, exist_ok=True) + env = {**E, "DISPLAY": DISPLAY, "XDG_RUNTIME_DIR": str(STATE / "pulse-runtime")} + pulse, chrome, ff = start_pulse(env), Chrome(env), None + ff_backoff, restarts, cpu = Backoff(1, 30, 120), {"pulse": 0, "ffmpeg": 0}, Cpu() + chrome.start() + up, next_metrics, next_mem, next_disk, last_frames = time.monotonic(), 0.0, 0.0, 0.0, (0, time.monotonic()) + stopping = [] + signal.signal(signal.SIGTERM, lambda *_: stopping.append(1)) + try: + while not stopping: + time.sleep(2) + now = time.monotonic() + if pulse.poll() is not None: + restarts["pulse"] += 1 + log("pulse: exited; restarting it and ffmpeg") + pulse = start_pulse(env) + if ff: + ff.stop() + if ff is None or not ff.alive() or ff.stalled(20) or ff.x != x_server(): + if ff is not None: + why = f"exited with {ff.proc.returncode}" if not ff.alive() else \ + "stalled" if ff.stalled(20) else "the X server changed" + ff.stop() + restarts["ffmpeg"] += 1 + delay = ff_backoff.failed() + log(f"ffmpeg: {why}; restart #{restarts['ffmpeg']} in {delay:.0f} s") + time.sleep(delay) + ff = Ffmpeg(encoder_cmd(), env, "encode") + ff.x = x_server() + ff_backoff.started() + log(f"ffmpeg: encoding {SIZE} at {FPS} fps, {VBITRATE} NVENC to {RELAY}") + chrome.check() + if now >= next_mem: + next_mem = now + 30 + pss = chrome.pss_mb() + aged = RESTART_HOURS and now - chrome.started > RESTART_HOURS * 3600 + if pss > MEM_HARD_MB: + chrome.restart(f"memory {pss} MB over the hard limit {MEM_HARD_MB} MB") + elif (pss > MEM_SOFT_MB or aged) and safe_now(): + chrome.restart(f"memory {pss} MB" if pss > MEM_SOFT_MB else f"up {RESTART_HOURS} h") + if now >= next_disk: + next_disk = now + 300 + guard_disk() + if now >= next_metrics: + next_metrics = now + 60 + f0, t0 = last_frames + page_fps = round((chrome.frames - f0) / (now - t0), 1) if chrome.frames >= f0 and now > t0 else None + last_frames = (chrome.frames, now) + p = ff.progress + metrics("capture", {"up_s": round(now - up), "page_fps": page_fps, "heap_mb": chrome.heap // 2**20, + "chrome_pss_mb": chrome.pss_mb(), "chrome_restarts": chrome.restarts, + "ff_frame": p.get("frame"), "ff_fps": p.get("fps"), "ff_drop": p.get("drop_frames"), + "ff_dup": p.get("dup_frames"), "ff_speed": p.get("speed"), + "ff_bitrate": p.get("bitrate"), **restarts, "cpu_pct": cpu.percent(), + "load1": round(os.getloadavg()[0], 2), "mem_avail_mb": mem_available_mb(), **gpu(), + "rec_free_gb": round(shutil.disk_usage(REC if REC.exists() else STATE).free + / 2**30, 1)}) + finally: + log("capture: stopping") + if ff: + ff.stop() + chrome.stop("service stopping") + stop_group(pulse) + return 0 + + +def target_url() -> str | None: + if TARGET == "none": + return None + if TARGET != "ssm": + return TARGET + import boto3 + ssm = boto3.client("ssm", region_name=E.get("AWS_DEFAULT_REGION", "us-west-2")) + server = ssm.get_parameter(Name=SSM_SERVER, WithDecryption=True)["Parameter"]["Value"].strip() + key = ssm.get_parameter(Name=SSM_KEY, WithDecryption=True)["Parameter"]["Value"].strip() + SECRETS[key] = "stream key" + return f"{server.rstrip('/')}/{key}" + + +def relay_ready() -> bool: + name = RELAY.rsplit("/", 1)[-1] + try: + with urllib.request.urlopen(f"http://127.0.0.1:9997/v3/paths/get/{name}", timeout=5) as r: + return bool(json.load(r).get("ready")) + except (OSError, ValueError): + return False + + +def push() -> int: + LOGS.mkdir(parents=True, exist_ok=True) + stopping = [] + signal.signal(signal.SIGTERM, lambda *_: stopping.append(1)) + backoff, restarts, ff, up, next_metrics, url = Backoff(1, 30, 60), 0, None, time.monotonic(), 0.0, None + while not stopping: + if url is None: + try: + url = target_url() + except Exception as e: + log(f"push: cannot read the target from SSM ({type(e).__name__}); retrying in 30 s") + time.sleep(30) + continue + if url is None: + log("push: AIROPS_TARGET is none; not pushing") + while not stopping: + time.sleep(5) + break + shown = TARGET if TARGET != "ssm" else f"X ({SSM_SERVER} + {SSM_KEY})" + SECRETS[url] = shown + if not relay_ready(): + log(f"push: waiting for {RELAY}") + while not stopping and not relay_ready(): + time.sleep(1) + continue + ff = Ffmpeg(["ffmpeg", "-hide_banner", "-loglevel", "warning", "-nostats", "-progress", "pipe:1", + "-stats_period", "5", "-rw_timeout", "15000000", "-i", RELAY, "-map", "0", "-c", "copy", + "-f", "flv", "-flvflags", "no_duration_filesize", url], dict(E), "push") + backoff.started() + log(f"push: forwarding {RELAY} to {shown}") + while not stopping and ff.alive() and not ff.stalled(30): + time.sleep(2) + now = time.monotonic() + if now >= next_metrics: + next_metrics = now + 60 + p = ff.progress + metrics("push", {"up_s": round(now - up), "restarts": restarts, "bitrate": p.get("bitrate"), + "speed": p.get("speed"), "out_time": p.get("out_time"), "frame": p.get("frame")}) + ended = not ff.alive() and ff.proc.returncode == 0 + why = "stopping" if stopping else (f"exited with {ff.proc.returncode}" if not ff.alive() else "stalled") + ff.stop() + if stopping: + break + restarts += 1 + delay = 1 if ended else backoff.failed() + log(f"push: {why}; reconnect #{restarts} in {delay:.0f} s") + time.sleep(delay) + log("push: stopped") + return 0 + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("mode", choices=["capture", "push"]) + return capture() if p.parse_args().mode == "capture" else push() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/live/stream/box/units/airops-capture.service b/live/stream/box/units/airops-capture.service new file mode 100644 index 0000000..285bd6f --- /dev/null +++ b/live/stream/box/units/airops-capture.service @@ -0,0 +1,17 @@ +[Unit] +Description=AirOps stream: Chrome on the viewer, PulseAudio and the ffmpeg encoder, supervised +Wants=airops-x.service airops-relay.service +After=airops-x.service airops-relay.service +StartLimitIntervalSec=0 + +[Service] +User=streamer +EnvironmentFile=/etc/airops-stream.env +ExecStart=/usr/bin/python3 -u /opt/airops-stream/streamer.py capture +Restart=always +RestartSec=3 +TimeoutStopSec=30 +KillMode=mixed + +[Install] +WantedBy=multi-user.target diff --git a/live/stream/box/units/airops-push.service b/live/stream/box/units/airops-push.service new file mode 100644 index 0000000..41d4fd3 --- /dev/null +++ b/live/stream/box/units/airops-push.service @@ -0,0 +1,17 @@ +[Unit] +Description=AirOps stream: forwards the relay to the RTMP target (X) +Wants=airops-relay.service +After=airops-relay.service +StartLimitIntervalSec=0 + +[Service] +User=streamer +EnvironmentFile=/etc/airops-stream.env +ExecStart=/usr/bin/python3 -u /opt/airops-stream/streamer.py push +Restart=always +RestartSec=3 +TimeoutStopSec=20 +KillMode=mixed + +[Install] +WantedBy=multi-user.target diff --git a/live/stream/box/units/airops-relay.service b/live/stream/box/units/airops-relay.service new file mode 100644 index 0000000..e75e2ed --- /dev/null +++ b/live/stream/box/units/airops-relay.service @@ -0,0 +1,14 @@ +[Unit] +Description=AirOps stream: the local RTMP relay and recorder (mediamtx) +After=network.target +StartLimitIntervalSec=0 + +[Service] +User=streamer +WorkingDirectory=/var/lib/airops-stream +ExecStart=/opt/airops-stream/mediamtx /opt/airops-stream/mediamtx.yml +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/live/stream/box/units/airops-static.service b/live/stream/box/units/airops-static.service new file mode 100644 index 0000000..806124c --- /dev/null +++ b/live/stream/box/units/airops-static.service @@ -0,0 +1,13 @@ +[Unit] +Description=AirOps stream: a static server for the recorded test bundle in /opt/airops-view (tests only) +After=network.target + +[Service] +User=streamer +WorkingDirectory=/opt/airops-view +ExecStart=/usr/bin/python3 -m http.server 8080 --bind 127.0.0.1 +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/live/stream/box/units/airops-x.service b/live/stream/box/units/airops-x.service new file mode 100644 index 0000000..4bfdb68 --- /dev/null +++ b/live/stream/box/units/airops-x.service @@ -0,0 +1,14 @@ +[Unit] +Description=AirOps stream: the X display on the NVIDIA GPU +After=network.target nvidia-persistenced.service +StartLimitIntervalSec=0 + +[Service] +EnvironmentFile=/etc/airops-stream.env +ExecStart=/opt/airops-stream/display.sh +ExecStartPost=/opt/airops-stream/display-ready.sh +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/live/stream/deploy.sh b/live/stream/deploy.sh new file mode 100755 index 0000000..deae044 --- /dev/null +++ b/live/stream/deploy.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# deploy.sh INSTANCE_ID [--test-bundle TGZ]: uploads box/ (with the viewer's web/audio/apron.ogg as bed.ogg, and +# live.env's region and X parameter names in airops-stream.env) to S3 and runs box/setup.sh on the box through SSM; +# --test-bundle also installs TGZ, a static viewer folder (viewer/ and data/), behind airops-static on :8080, for tests +# without the live server. +set -euo pipefail +STREAM="$(cd "$(dirname "$0")" && pwd)" +source "$STREAM/../aws/common.sh" +ID=${1:?usage: deploy.sh INSTANCE_ID [--test-bundle TGZ]} +need LIVE_BUCKET +tmp=$(mktemp -d); mkdir "$tmp/box" +cp -R "$STREAM/box/." "$tmp/box/"; rm -rf "$tmp/box/__pycache__" +cp "$STREAM/../../web/audio/apron.ogg" "$tmp/box/bed.ogg" +sed -e "s|^AIROPS_SSM_SERVER=.*|AIROPS_SSM_SERVER=${LIVE_X_SERVER_PARAM:-/airops-live/x-server}|" \ + -e "s|^AIROPS_SSM_KEY=.*|AIROPS_SSM_KEY=${LIVE_X_KEY_PARAM:-/airops-live/x-key}|" \ + -e "s|^AWS_DEFAULT_REGION=.*|AWS_DEFAULT_REGION=$AWS_REGION|" "$STREAM/box/airops-stream.env" > "$tmp/box/airops-stream.env" +COPYFILE_DISABLE=1 tar --no-xattrs -czf "$tmp/box.tgz" -C "$tmp/box" . +aws s3 cp "$tmp/box.tgz" "s3://$BUCKET/$PREFIX/stream/box.tgz" --only-show-errors +BUNDLE= +if [ "${2:-}" = --test-bundle ]; then + aws s3 cp "${3:?usage: deploy.sh INSTANCE_ID [--test-bundle TGZ]}" "s3://$BUCKET/$PREFIX/stream/test-bundle.tgz" --only-show-errors + BUNDLE=$(presign get "$PREFIX/stream/test-bundle.tgz" 3600) +fi +ssm "$ID" 1200 "$STREAM/box/setup.sh" "$(presign get "$PREFIX/stream/box.tgz" 3600)" $BUNDLE +rm -rf "$tmp" diff --git a/live/stream/save-recordings.sh b/live/stream/save-recordings.sh new file mode 100755 index 0000000..e844a74 --- /dev/null +++ b/live/stream/save-recordings.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# save-recordings.sh INSTANCE_ID [PATTERN]: copies the box's recordings (all, or the names matching PATTERN, e.g. +# 2026-10-23) to s3://LIVE_BUCKET/LIVE_S3_PREFIX/rec/ with a presigned PUT each (the box's role needs no S3 rights). +# Skips the segment being written and those already in S3 at the same size. +set -euo pipefail +STREAM="$(cd "$(dirname "$0")" && pwd)" +source "$STREAM/../aws/common.sh" +ID=${1:?usage: save-recordings.sh INSTANCE_ID [PATTERN]} +PAT=${2:-} +need LIVE_BUCKET +tmp=$(mktemp -d); LIST="$tmp/list.sh"; PUT="$tmp/put.sh" +printf 'cd /var/lib/airops-stream/rec/airops && ls -t | tail -n +2 | grep -F -- "%s" | while read f; do echo "$f $(stat -c %%s "$f")"; done\n' "$PAT" > "$LIST" +args=() +while read -r f size; do + [ -n "$f" ] || continue + have=$(aws s3api head-object --bucket "$BUCKET" --key "$PREFIX/rec/$f" --query ContentLength --output text 2>/dev/null || echo 0) + [ "$have" = "$size" ] && { log "$f already saved"; continue; } + args+=("$f" "$(presign put "$PREFIX/rec/$f" 21600)") +done < <(ssm "$ID" 120 "$LIST") +[ ${#args[@]} -gt 0 ] || { log "nothing to save"; exit 0; } +cat > "$PUT" <<'BOX' +cd /var/lib/airops-stream/rec/airops +while [ $# -gt 1 ]; do curl -fsS -X PUT -T "$1" "$2" && echo "saved $1"; shift 2; done +BOX +ssm "$ID" 7200 "$PUT" "${args[@]}" +aws s3 ls "s3://$BUCKET/$PREFIX/rec/"; rm -rf "$tmp" diff --git a/live/stream/start.sh b/live/stream/start.sh new file mode 100755 index 0000000..dafe841 --- /dev/null +++ b/live/stream/start.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# start.sh ID [--url URL] [--target none|ssm|rtmp(s)://...] [--size WxH] [--bitrate 6000k] [--safe-url URL]: +# box/start.sh on box ID: sets what is given in /etc/airops-stream.env, starts what is stopped and restarts what a +# change concerns. +# --target ssm streams to X with the server and key in SSM (LIVE_X_SERVER_PARAM, LIVE_X_KEY_PARAM); none only records. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: start.sh ID [--url URL] [--target T] [--size WxH] [--bitrate B] [--safe-url URL]}; shift +ssm "$ID" 120 "$H/box/start.sh" "$@" diff --git a/live/stream/status.sh b/live/stream/status.sh new file mode 100755 index 0000000..831c6cd --- /dev/null +++ b/live/stream/status.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# status.sh ID [MINUTES]: box/status.sh on box ID: the units, the relay, the last metrics of capture and push, restarts +# and warnings in the last MINUTES (default 30), GPU, memory, disk and a scheduled power-off. +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: status.sh ID [MINUTES]} +ssm "$ID" 60 "$H/box/status.sh" "${2:-30}" diff --git a/live/stream/stop.sh b/live/stream/stop.sh new file mode 100755 index 0000000..533b5cc --- /dev/null +++ b/live/stream/stop.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# stop.sh ID [--push-only|--disable]: box/stop.sh on box ID: stops pushing to the target (--push-only), or the whole +# stream (a reboot starts it again; --disable keeps it stopped). +source "$(dirname "$0")/../aws/common.sh" +H="$(cd "$(dirname "$0")" && pwd)" +ID=${1:?usage: stop.sh ID [--push-only|--disable]}; shift +ssm "$ID" 60 "$H/box/stop.sh" "$@" diff --git a/scripts/live_fake.py b/scripts/live_fake.py new file mode 100644 index 0000000..52be08d --- /dev/null +++ b/scripts/live_fake.py @@ -0,0 +1,345 @@ +"""A fake live server for the viewer's live mode (web/index.html?live=BASE): recorded control-center days from a viewer +folder played back as if live, each truncated at the watches its desks have decided so far, one every --watch-seconds +until they stand down, then the day done with its grade, an intermission naming the next day, and the next day. It +serves what CONTRACT.md's live server does: /live/episode.json (the episode so far with its `live` object), +/live/state.json ({seq, live}) and /live/current.json (the runner's), the traffic sidecar under /live/ with a new URL +at each watch, the folder's stage/, radar/ and twin/ beside them, and the viewer itself under /web/. +/live/control?seek=SECONDS&rate=X&pause=0|1 moves its clock, for tests. + +usage: .venv/bin/python scripts/live_fake.py FOLDER EPISODE.json [EPISODE.json ...] [--port 8791] [--watch-seconds 20] + then open http://127.0.0.1:8791/web/index.html?live=http://127.0.0.1:8791&stream +""" + +import argparse +import copy +import functools +import json +import re +import sys +import threading +import time +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlparse + +ROOT = Path(__file__).resolve().parents[1] + + +class Fake: + """The schedule of the fake days on a clock that --rate speeds up and /live/control moves.""" + + def __init__(self, folder: Path, names: list[str], watch_seconds: float, start_seconds: float, + done_seconds: float, gap_seconds: float, spent0: float): + self.folder, self.names = folder, names + self.W, self.S, self.D, self.G = watch_seconds, start_seconds, done_seconds, gap_seconds + self.spent0 = spent0 + self.lock = threading.Lock() + self.cache: dict[str, dict] = {} + self.t0, self.rate, self.paused, self.frozen = time.monotonic(), 1.0, False, 0.0 + self.epoch = datetime.now(UTC) + self.history: list[dict] = [] + self.memo: tuple | None = None + + def ep(self, i: int) -> dict: + name = self.names[i % len(self.names)] + if name not in self.cache: + self.cache[name] = json.loads((self.folder / "episodes" / name).read_text()) + return self.cache[name] + + def clock(self) -> float: + return self.frozen if self.paused else self.frozen + (time.monotonic() - self.t0) * self.rate + + def control(self, q: dict) -> dict: + with self.lock: + now = self.clock() + if "seek" in q: + now = float(q["seek"][0]) + if "rate" in q: + self.rate = float(q["rate"][0]) + if "pause" in q: + self.paused = q["pause"][0] == "1" + self.frozen, self.t0 = now, time.monotonic() + self.memo = None + return {"clock": now, "rate": self.rate, "paused": self.paused, **self.where(now)} + + def length(self, i: int) -> tuple[float, int]: + """A day's wall seconds from its start to done, and the watch its desks stand down at.""" + ep = self.ep(i) + stand = max((a.get("watch", 0) for a in (ep["occ"].get("absent") or {}).values()), default=len(ep["watches"])) + return self.S + (stand + 1) * self.W, stand + + def where(self, now: float) -> dict: + """Which day the clock is in and how far: {day, phase: starting|playing|done|intermission, k (watches + decided), into (seconds into the day), frac (of the watch in progress)}.""" + i, at = 0, 0.0 + while True: + n, stand = self.length(i) + if now < at + n + self.D + self.G: + into = now - at + if into < self.S: + return {"day": i, "phase": "starting", "k": 0, "into": into, "frac": 0.0} + if into < n: + k = int((into - self.S) // self.W) + return {"day": i, "phase": "playing", "k": k, "into": into, + "frac": (into - self.S - k * self.W) / self.W} + return {"day": i, "phase": "done" if into < n + self.D else "intermission", "k": None, "into": into, + "frac": 1.0} + at += n + self.D + self.G + i += 1 + + def day_start(self, i: int) -> float: + return sum(self.length(j)[0] + self.D + self.G for j in range(i)) + + def state(self) -> tuple[dict, dict]: + """The episode so far and its `live` object, as CONTRACT.md's live server serves them.""" + with self.lock: + now = self.clock() + w = self.where(now) + key = (w["day"], w["phase"], w["k"], round(w["frac"] * 20)) + if self.memo and self.memo[0] == key: + return self.memo[1], self.memo[2] + ep = self.ep(w["day"]) + ws = sorted(ep["watches"], key=lambda x: x["t"]) + done = w["phase"] in ("done", "intermission") + k = len(ws) if done else w["k"] + out = truncate(ep, k, w["frac"]) + stem = ep["stage"]["traffic"].removesuffix(".stage.json.gz") + out["stage"] = {**ep["stage"], "traffic": ep["stage"]["traffic"] if done else f"{stem}.stage-{k}.json.gz"} + seq = int(self.day_start(w["day"]) * 10) + (k + 1) * 100 + round(w["frac"] * 20) + started = self.epoch + timedelta(seconds=self.day_start(w["day"]) / self.rate) + run = lambda j: f"{self.ep(j)['task']['task_id']}-joint-a1-d{j}" # noqa: E731 + history = [{"date": self.ep(j)["task"]["date"], "task_id": self.ep(j)["task"]["task_id"], "run_id": run(j), + "instance_id": f"fake-{j}", "outcome": "scored", "reward": self.ep(j)["reward"], + "reward_L": self.ep(j)["occ"]["grade"].get("reward_L"), "attempts": 1, + "cost_usd": self.ep(j)["record"]["cost_usd"], + "wall_seconds": self.ep(j)["record"]["wall_seconds"]} + for j in range(w["day"] + (1 if done else 0))] + nxt = self.ep(w["day"] + 1) + starts = self.epoch + timedelta(seconds=self.day_start(w["day"] + 1) / self.rate) + span = max(1.0, self.length(w["day"])[0] - self.S) + share = 1.0 if done else min(1.0, max(0.0, w["into"] - self.S) / span) + t_open = ws[min(k, len(ws) - 1)]["t"] + last = max(e[x] for e in ep["executed"] for x in ("dep", "arr") if e.get(x) is not None) + cost = ws[k - 1]["est_cost"] if 0 < k <= len(ws) else ws[0]["est_cost"] + g = ep["occ"]["grade"] + live = {"status": w["phase"], "seq": seq, "run_id": run(w["day"]), "task_id": ep["task"]["task_id"], + "date": ep["task"]["date"], "instance_id": f"fake-{w['day']}", "model": ep["model"], + "model_label": ep.get("model_label"), "mode": "joint", "pilot": "fake-live", + "watch": min(k, len(ws) - 1), "watches": len(ws), "t": t_open, + "decided_until": last if done else t_open, + "turn": None if done else turn(ep, k, w["frac"]), + "desks": desks(ep, ws, k, w["frac"], datetime.now(UTC), done), + "feed": feed(ep, ws, k, w["frac"], datetime.now(UTC)), + "projected": {"total": cost}, + "next_watch_at": None if done else (datetime.now(UTC) + timedelta( + seconds=(1 - w["frac"]) * self.W / self.rate)).isoformat(timespec="seconds"), + "watch_wall_s": self.W / self.rate, + "started_utc": started.isoformat(timespec="seconds"), + "updated_utc": datetime.now(UTC).isoformat(timespec="seconds"), + "wall_seconds": round(ep["record"]["wall_seconds"] if done else w["into"] / self.rate, 1), + "spent_usd": round(share * ep["record"]["cost_usd"], 2), + "result": {"reward": ep["reward"], "reward_L": g.get("reward_L"), "cost": g["cost"], + "best": g["best"], "bound": g["bound"], "B0": g["B0"], + "cost_usd": ep["record"]["cost_usd"], + "wall_seconds": ep["record"]["wall_seconds"]} if done else None, + "exact": True, "mismatches": 0, + "next": {"date": nxt["task"]["date"], "task_id": nxt["task"]["task_id"], + "starts_utc": starts.isoformat(timespec="seconds")} + if w["phase"] == "intermission" else None, + "history": history, "error": None} + out["live"] = live + self.memo = (key, out, {"seq": seq, "live": live}) + return out, self.memo[2] + + +def truncate(ep: dict, k: int, frac: float = 0.0) -> dict: + """The episode as the live server builds it with watch ``k`` open and ``frac`` of it gone: the watches closed so + far and the open one, the room's events so far (the open watch's held back by its later `shown`), the flights as + the latest projection has them; the grade once every watch is closed.""" + ws = sorted(ep["watches"], key=lambda x: x["t"]) + if k >= len(ws): + return copy.copy(ep) + until = ws[k]["t"] + out = copy.copy(ep) + out["watches"] = ws[:k] + [{key: v for key, v in ws[k].items() if key != "est_cost"} | {"changes": []}] + out["steps"] = so_far([s for s in ep["steps"] if s.get("watch", 0) <= k], lambda s: s.get("watch", 0) == k, frac) + out["news"] = [n for n in ep.get("news") or [] if n["at"] <= until] + proj = {r["id"]: r for r in ws[k - 1]["projected"]} if k else {} + + def final(e): + return (e["status"] == "flown" and e.get("dep") is not None and e["dep"] < until) or \ + (e["status"] != "flown" and e.get("decided") is not None and e["decided"] < until) + out["executed"] = [e if final(e) or e["id"] not in proj else {**e, **proj[e["id"]]} for e in ep["executed"]] + occ = copy.copy(ep["occ"]) + occ["turns"] = so_far([u for u in occ["turns"] if u["watch"] <= k], lambda u: u["watch"] == k, frac) + occ["pages"] = [p for p in occ["pages"] if p["watch"] < k] + occ["decisions"] = so_far([d for d in occ["decisions"] if d["t"] <= until], lambda d: d["t"] == until, frac) + occ["notes"] = [n for n in occ.get("notes") or [] if n["t"] < until] + reqs = [] + for r in so_far([r for r in occ["requests"] if r["sent"] <= until], lambda r: r["sent"] == until, frac): + if r.get("replied") is None or r["replied"] >= until: + r = {**r, "status": "open", "reply": None, "replied": None, "reply_text": None, "answered": None, + "actions": []} + reqs.append(r) + occ["requests"] = reqs + occ["grade"] = None + occ["absent"] = {d: a for d, a in (occ.get("absent") or {}).items() if a["t"] < until} + out["occ"] = occ + out["reward"] = out["reward_L"] = None + out["end_reason"] = None + out["audit"] = None + out["record"] = {**ep["record"], "cost_usd": None, "wall_seconds": None, "desks": None} + out["cost"] = {**ep["cost"], "total": ws[k - 1]["est_cost"] if k else ws[0]["est_cost"]} + return out + + +def so_far(items: list, open_: Callable[[dict], bool], frac: float) -> list: + """``items`` with those of the open watch cut to the first ``frac`` of them.""" + opened = [x for x in items if open_(x)] + keep = {id(x) for x in opened[: round(len(opened) * frac)]} + return [x for x in items if not open_(x) or id(x) in keep] + + +def turn(ep: dict, k: int, frac: float) -> dict | None: + turns = [u for u in ep["occ"]["turns"] if u["watch"] == k] + if not turns: + return None + u = turns[min(len(turns) - 1, int(frac * len(turns)))] + return {"n": u["n"], "desk": u["desk"], "kind": u["kind"], "calls": u["calls"]} + + +def events(ep: dict, t: float) -> list[tuple]: + """The room's events of the watch at ``t``: (minute shown, desk, kind, text, request, flight).""" + out = [] + for r in ep["occ"]["requests"]: + if r["sent"] == t: + fl = f" f{r['flight']}" if r["flight"] is not None else "" + text = f"{r['id']} {r['type']}{fl} to {r['receiver']}: {(r.get('text') or '')[:80]}" + out.append((r["shown"], r["sender"], "request", text, r["id"], r["flight"])) + if r.get("replied") == t and r.get("answered") is not None: + text = f"{r['id']} {r['reply']}: {(r.get('reply_text') or '')[:80]}" + out.append((r["answered"], r["receiver"], "reply", text, r["id"], r["flight"])) + for d in ep["occ"]["decisions"]: + if d["t"] == t: + text = f"{d['action']} f{d['flight']}{' ' + str(d['detail']) if d.get('detail') else ''}" + out.append((d["shown"], d["desk"], "decision", text, d.get("ref"), d["flight"])) + return sorted(out, key=lambda e: e[0]) + + +def feed(ep: dict, ws: list[dict], k: int, frac: float, now: datetime) -> list[dict]: + out = [] + for j in range(max(0, k - 3), min(k, len(ws) - 1) + 1): + ev = events(ep, ws[j]["t"]) + if j == k: + ev = ev[: round(len(ev) * frac)] + for n, (_, desk, kind, text, req, flight) in enumerate(ev): + at = now - timedelta(seconds=(k - j) * 60 + (len(ev) - n) * 4) + out.append({"seq": j * 1000 + n, "at": at.isoformat(timespec="seconds"), "watch": j, "t": ws[j]["t"], + "desk": desk, "kind": kind, "text": text, "request": req, "flight": flight}) + return out[-60:] + + +def desks(ep: dict, ws: list[dict], k: int, frac: float, now: datetime, done: bool) -> list[dict]: + """Each desk's state and latest call of the watch in progress, as far as ``frac`` of it has gone.""" + order = [d for d in ("crew", "customer", "ops") if d in ep["occ"]["desks"]] + if done or k >= len(ws): + return [{"role": d, "state": "off", "off": "stand_down", "last": None, "at": None} for d in order] + steps = [s for s in ep["steps"] if s.get("watch") == k] + steps = steps[: round(len(steps) * frac)] + u = turn(ep, k, frac) + out = [] + for d in order: + mine = [s for s in steps if s.get("desk") == d] + last = mine[-1] if mine else None + out.append({"role": d, "state": "turn" if u and u["desk"] == d else "waiting", "off": None, + "last": {"seq": len(mine), "tool": last["tool"], "t": last["t"], + "text": f"{last['tool']} {json.dumps(last.get('args') or {})[:60]}"} if last else None, + "at": (now - timedelta(seconds=5)).isoformat(timespec="seconds") if last else None}) + return out + + +class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, fake: Fake, web: Path, **kw): + self.fake, self.web = fake, web + super().__init__(*args, **kw) + + def do_GET(self) -> None: + url = urlparse(self.path) + p = unquote(url.path) + if p == "/live/episode.json": + return self.send_json(self.fake.state()[0]) + if p in ("/live/state.json", "/live/state"): + return self.send_json(self.fake.state()[1]) + if p == "/live/current.json": + live = self.fake.state()[1]["live"] + return self.send_json({k: live.get(k) for k in ("instance_id", "run_id", "date", "task_id", "model", "mode", + "started_utc", "status", "next", "history")}) + if p == "/live/control": + return self.send_json(self.fake.control(parse_qs(url.query))) + return super().do_GET() + + def send_json(self, doc: dict) -> None: + body = json.dumps(doc, ensure_ascii=False, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def translate_path(self, path: str) -> str: + p = unquote(path.split("?", 1)[0].split("#", 1)[0]) + for prefix in ("/viewer/", "/web/"): + if p.startswith(prefix): + return str(self.web / p[len(prefix):]) + if m := re.match(r"^/live/([^/]+)\.stage(-\d+)?\.json\.gz$", p): + return str(self.fake.folder / "episodes" / f"{m[1]}.stage.json.gz") + for prefix in ("stage", "radar", "twin", "thumbs", "episodes"): + if p.startswith(f"/{prefix}/"): + return str(self.fake.folder / p[1:]) + return str(self.fake.folder / "__missing__") + + def end_headers(self) -> None: + self.send_header("Access-Control-Allow-Origin", "*") + super().end_headers() + + def log_message(self, *args) -> None: + pass + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("folder", type=Path, help="A viewer folder: episodes/, stage/, radar/.") + p.add_argument("episodes", nargs="+", help="Episode files under FOLDER/episodes, played one after another.") + p.add_argument("--port", type=int, default=8791) + p.add_argument("--viewer-port", type=int, help="Also serve on this port, another origin, for a cross-origin test.") + p.add_argument("--web", type=Path, default=ROOT / "web") + p.add_argument("--watch-seconds", type=float, default=20) + p.add_argument("--start-seconds", type=float, default=10, help="Seconds a day starts with nothing decided.") + p.add_argument("--done-seconds", type=float, default=60) + p.add_argument("--gap-seconds", type=float, default=120) + p.add_argument("--seek", type=float, default=0, help="Start the clock this many seconds in.") + p.add_argument("--rate", type=float, default=1) + p.add_argument("--spent", type=float, default=0, help="Model spend before the first day, in dollars.") + args = p.parse_args(argv) + fake = Fake(args.folder.expanduser().resolve(), args.episodes, args.watch_seconds, args.start_seconds, + args.done_seconds, args.gap_seconds, args.spent) + fake.control({"seek": [str(args.seek)], "rate": [str(args.rate)]}) + handler = functools.partial(Handler, fake=fake, web=args.web.resolve()) + servers = [ThreadingHTTPServer(("127.0.0.1", port), handler) for port in (args.port, args.viewer_port) if port] + for s in servers[1:]: + threading.Thread(target=s.serve_forever, daemon=True).start() + print(f"live at http://127.0.0.1:{args.port}/live/episode.json; open " + f"http://127.0.0.1:{servers[-1].server_port}/web/index.html?live=http://127.0.0.1:{args.port}&stream", + flush=True) + try: + servers[0].serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live_runner.py b/scripts/live_runner.py new file mode 100644 index 0000000..9caf3dd --- /dev/null +++ b/scripts/live_runner.py @@ -0,0 +1,671 @@ +"""A model's three-desk storm days on airops-occ3 back to back, for the live stream. Each day is one `agent-env run` of +its task, played as the board's pilot plays one (occ.sweep: the same bundle, caps, rows, days and transcripts, under +results/runs/), and starts on a grid of slots, after an intermission until the last day's slot ends, only while +the spend so far plus a worst day stays within the budget. current.json tells the live server the running day, its +instance as soon as agent-env records it and its env once deployed. A day that fails is tried once more, then skipped; +one whose audit refuses what the desks did is final, as on the board. stop.flag (beside current.json) or SIGTERM +finishes the day and exits; a second SIGTERM stops the day now. The settings and the days' order are +scripts/live_runner.toml, read again before every day. + +usage: .venv/bin/python scripts/live_runner.py run [--config TOML] [--current PATH] [--policy script] [--name NAME] + [--days N] [--dry-run] + .venv/bin/python scripts/live_runner.py order [--write scripts/live_runner.toml] +""" + +import argparse +import gzip +import json +import os +import re +import signal +import subprocess +import sys +import time +import tomllib +from dataclasses import asdict, dataclass, fields, replace +from datetime import UTC, date, datetime +from pathlib import Path + +from agent_env.bundle.parse import BundleKind, parse_bundle +from agent_env.store.routing import namespace_routing +from agent_env.task.store import get_task_instance_store + +from agentenv_airops.occ import sweep +from agentenv_airops.occ.pack import Pack3 + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = Path(__file__).with_name("live_runner.toml") +HUBS = ("JFK", "BOS") +WINTER = (11, 12, 1, 2, 3) +ATTEMPTS = 2 +POLL_SECONDS = 5.0 +IDLE_SECONDS = 10.0 +KILL_AFTER_SECONDS = 300.0 +SPENT = re.compile(r"request \d+: [^\n]*?\$([0-9.]+) spent") +STAMP = "%Y-%m-%dT%H:%M:%SZ" + + +@dataclass(frozen=True) +class Settings: + """live_runner.toml: the pilot, the money, the pacing and the days in play order.""" + + name: str = "occ3-live-astra" + model: str = "openai/gpt-6-astra" + policy: str | None = None + pack: str | None = None + cap_joint_usd: float = 23.5 + request_margin_usd: float = 4.0 + budget_usd: float = 430.0 + worst_day_usd: float = 42.0 + typical_day_usd: float = 22.68 + slot_minutes: float = 75.0 + min_intermission_minutes: float = 5.0 + card_seconds: float = 60.0 + window_hours: float = 24.0 + start_utc: str | None = None + longest_day_minutes: float = 150.0 + days: tuple[str, ...] = () + + @classmethod + def read(cls, path: Path, **overrides) -> "Settings": + raw = tomllib.loads(path.read_text()) + if unknown := set(raw) - {f.name for f in fields(cls)}: + raise ValueError(f"{path}: unknown settings {sorted(unknown)}") + given = {k: v for k, v in overrides.items() if v is not None} + s = replace(cls(**raw), days=tuple(raw.get("days", ())), **given) + if s.policy == "llm": + s = replace(s, policy=None) + known = Pack3(s.pack).tasks + if bad := [t for t in s.days if t not in known or known[t]["split"] == "control"]: + raise ValueError(f"{path}: not a train or eval day of the pack: {', '.join(bad)}") + if len(set(s.days)) != len(s.days): + raise ValueError(f"{path}: a day is listed twice") + if s.start_utc is not None: + stamp(s.start_utc) + return s + + def pilot(self, runs: list[str]) -> sweep.Pilot3: + return sweep.Pilot3(self.name, self.model, runs, {"joint": self.cap_joint_usd, "one": self.cap_joint_usd}, + self.budget_usd, self.pack, self.policy, self.request_margin_usd) + + +def utc(t: float | None = None) -> str: + return time.strftime(STAMP, time.gmtime(time.time() if t is None else t)) + + +def stamp(text: str) -> float: + return datetime.strptime(text, STAMP).replace(tzinfo=UTC).timestamp() + + +# ---------------------------------------------------------------------------------------------- the days' order + + +def hours(spans) -> float: + """The hours the minute spans cover, overlaps counted once.""" + total, end = 0, float("-inf") + for a, b in sorted(spans): + if b > end: + total += b - max(a, end) + end = b + return round(total / 60, 1) + + +def storm(task_id: str, meta: dict, task: dict, best_from: str | None = None) -> dict: + """A day's storm from its task: per hub the hours its GDPs governed departures (their segments) and its ground + stops (count and hours), the flights JetBlue cancelled in reality, and the score: GDP hours plus twice the + ground-stop hours at both hubs, plus one per ten flights cancelled; and where its best plan found came from.""" + hubs = {} + for hub in HUBS: + p = task["programs"].get(hub) or {"segments": [], "stops": []} + hubs[hub] = {"gdp_h": hours((s["since"], s["until"]) for s in p["segments"]), "gs": len(p["stops"]), + "gs_h": hours((s["start"], s["end"]) for s in p["stops"])} + cancelled = sum(f["actual"]["cancelled"] for f in task["flights"]) + score = sum(h["gdp_h"] + 2 * h["gs_h"] for h in hubs.values()) + cancelled / 10 + return {"task_id": task_id, "date": meta["date"], "split": meta["split"], + "season": "winter" if int(meta["date"][5:7]) in WINTER else "summer", "hubs": hubs, + "cancelled": cancelled, "score": round(score, 1), "best_from": best_from} + + +def storms(pack: Pack3) -> list[dict]: + out = [] + for task_id, meta in pack.tasks.items(): + if meta["split"] in ("train", "eval"): + doc = json.loads(gzip.decompress((pack.path / "dates" / f"{meta['date']}.json.gz").read_bytes())) + out.append(storm(task_id, meta, doc["task"], (doc.get("refs_p") or {}).get("best_from"))) + return out + + +def order(days: list[dict]) -> list[dict]: + """The play order: the training days, which no model played on the board, then the eval days as the fallback. In + each, a season's days by score, a day a calendar day from a higher one of the season (the same storm) after the + others, and the seasons interleaved in proportion, the top day first; the days whose best plan found is the + standard desks' own (B0, so any plan as good scores 1) after the rest.""" + return [d for split in ("train", "eval") for tail in (False, True) + for d in interleave([ranked([x for x in days if x["split"] == split and x["season"] == season + and (x.get("best_from") == "B0") == tail]) + for season in ("summer", "winter")])] + + +def ranked(days: list[dict]) -> list[dict]: + lead, later = [], [] + for d in sorted(days, key=lambda d: (-d["score"], d["date"])): + near = any(abs((date.fromisoformat(d["date"]) - date.fromisoformat(x["date"])).days) <= 1 for x in lead) + (later if near else lead).append(d) + return lead + later + + +def interleave(lists: list[list[dict]]) -> list[dict]: + """Each next day from the list furthest behind its share of the whole, ties to the higher score.""" + taken, out = [0] * len(lists), [] + while left := [i for i, x in enumerate(lists) if taken[i] < len(x)]: + i = min(left, key=lambda i: ((taken[i] + 0.5) / len(lists[i]), -lists[i][taken[i]]["score"])) + out.append(lists[i][taken[i]]) + taken[i] += 1 + return out + + +def describe(d: dict) -> str: + parts = [] + for hub, h in d["hubs"].items(): + bits = ([f"GDP {h['gdp_h']:g} h"] if h["gdp_h"] else []) + \ + ([f"{h['gs']} ground stop{'s' if h['gs'] > 1 else ''} {h['gs_h']:g} h"] if h["gs"] else []) + parts.append(f"{hub} {', '.join(bits) or 'no program'}") + return f"{d['season']}; {'; '.join(parts)}; {d['cancelled']} cancelled" + + +HEAD = """\ +# The live runner's settings (scripts/live_runner.py), read again before every day: edits apply from the next day on. +name = "occ3-live-astra" # results/runs/: pilot.json, results.jsonl, days/, transcripts/, logs/ +model = "openai/gpt-6-astra" +# policy = "script" # the env's scripted desks, $0 +cap_joint_usd = 23.5 # each desk's cap, as registered for the board +request_margin_usd = 4.0 # one worst request a desk may make past its cap +budget_usd = 430.0 # model spend may not pass it +worst_day_usd = 42.0 # a day starts only if the spend so far plus this fits (the board's worst: $41.46) +typical_day_usd = 22.68 # the board's mean Astra day: --dry-run's projection only +slot_minutes = 80.0 # a day that ends early is followed by an intermission until its slot ends +min_intermission_minutes = 5.0 # after a day past its slot, or a failed attempt +card_seconds = 60.0 # status "done" for this long after a day, then "intermission" +window_hours = 24.0 # a day starts only if its slot ends within the window +# start_utc = "2026-10-23T16:00:00Z" # the first day's start; default: when the runner first runs +longest_day_minutes = 150.0 # a day still running then is stopped (the board's longest: 84) + +# The days in play order (`live_runner.py order`): the training days by storm (GDP hours, twice the ground-stop hours, +# at JFK and BOS, plus one per ten flights cancelled), each season by score with a storm's second day after the rest, +# the seasons interleaved; then the eval days the board played, the same way. Reorder or delete freely. +""" + + +def config_text(days: list[dict]) -> str: + lines, split = [HEAD + "days = ["], None + for i, d in enumerate(days, 1): + if d["split"] != split: + split = d["split"] + lines.append(" # training days" if split == "train" else " # eval days: Astra played these on the board") + lines.append(f' "{d["task_id"]}", # {i:2} {d["date"]} score {d["score"]:g}: {describe(d)}') + return "\n".join(lines + ["]", ""]) + + +# ---------------------------------------------------------------------------------------------- the money and pacing + + +def spend(row: dict, margin: float) -> float: + """An attempt's model spend: each desk's own record, else its last live log line and one request past it, else + nothing (it never made a request).""" + live = row.get("live_usd") or {} + return round(sum(d["cost_usd"] if d.get("cost_usd") is not None else live[r] + margin if r in live else 0.0 + for r, d in row["desks"].items()), 6) + + +def spent(rows: list[dict]) -> float: + return round(sum(r["spend_usd"] if "spend_usd" in r else r["cost_usd"] for r in rows), 6) + + +def affordable(so_far: float, s: Settings) -> bool: + """The guard: a day starts only if the spend so far plus a worst day stays within the budget.""" + return so_far + s.worst_day_usd <= s.budget_usd + 1e-9 + + +def over(so_far: float, live: float, s: Settings) -> bool: + """The backstop while a day runs: the spend so far and the day's live estimate reach the budget.""" + return so_far + live >= s.budget_usd + + +def grouped(rows: list[dict]) -> dict[str, list[dict]]: + out = {} + for r in rows: + out.setdefault(r["run"], []).append(r) + return out + + +def settled(attempts: list[dict]) -> str | None: + """scored or unscored (the audit refused what the desks did), skipped after ATTEMPTS failed, else None.""" + for r in attempts: + if r["outcome"] in ("scored", "unscored"): + return r["outcome"] + return "skipped" if len(attempts) >= ATTEMPTS else None + + +def pending(s: Settings, rows: list[dict]) -> list[str]: + groups = grouped(rows) + return [t for t in s.days if settled(groups.get(f"{t}-joint", [])) is None] + + +def history(rows: list[dict]) -> list[dict]: + out = [] + for run, attempts in grouped(rows).items(): + if (state := settled(attempts)) is None: + continue + last = next((r for r in attempts if r["outcome"] == state), attempts[-1]) + out.append({"date": last.get("date"), "task_id": run.rsplit("-", 1)[0], "run_id": f"{run}-a{last['attempt']}", + "hub": last.get("hub"), "instance_id": last["instance"], "outcome": state, + "reward": last["reward"], "reward_L": last["reward_L"], "cost_usd": round(spent(attempts), 4), + "wall_seconds": last["wall_seconds"], "desks": last.get("desks"), "attempts": len(attempts)}) + return out + + +def next_start(s: Settings, last: dict | None, window_start: float, now: float) -> float: + """The next attempt's start: the window's; after a day played, its slot's end, or the least intermission after + it if it ran past; after a failed attempt, the least intermission.""" + if last is None: + return max(now, window_start) + after = stamp(last["ended_utc"]) + 60 * s.min_intermission_minutes + if last["outcome"] == "failed": + return max(now, after) + return max(now, after, stamp(last["started_utc"]) + 60 * s.slot_minutes) + + +def projection(s: Settings, rows: list[dict], window_start: float, now: float) -> tuple[list[dict], str]: + """The days ahead as they would start if each took its slot and cost the typical day, and why the last one is the + last.""" + start, so_far, end = next_start(s, rows[-1] if rows else None, window_start, now), spent(rows), \ + window_start + 3600 * s.window_hours + plan = [] + for task_id in pending(s, rows): + if not affordable(so_far, s): + return plan, f"the budget: ${so_far:.2f} spent + a worst day ${s.worst_day_usd:g} > ${s.budget_usd:g}" + if start + 60 * s.slot_minutes > end + 1e-6: + return plan, f"the window: the next slot would end after {utc(end)}" + plan.append({"task_id": task_id, "starts": start, "spent_before": round(so_far, 2)}) + start, so_far = start + 60 * s.slot_minutes, so_far + s.typical_day_usd + return plan, "the order: no days left" + + +# ---------------------------------------------------------------------------------------------- the running day + + +def bundle_task(bundle: Path, run: str) -> str: + """The id agent-env gives the bundle's task `run`, which its instances carry.""" + return next(e.id for e in parse_bundle(bundle.absolute()).entries if e.kind is BundleKind.TASK and e.name == run) + + +def instances(task: str) -> list[str]: + with namespace_routing(): + return [i.instance_id for i in get_task_instance_store().find(task, limit=10)] + + +def endpoints(ctx: dict) -> dict | None: + for e in ctx.get("deployed_envs") or []: + if e.get("mcp_url"): + return {"base": e["mcp_url"].removesuffix("/mcp"), "mcp_url": e["mcp_url"], + "gateway_url": e.get("gateway_url")} + return None + + +def desk_spend(ctx: dict) -> dict[str, float]: + """Each deployed desk's spend so far: the last "request N: … $X spent" line of its container's log.""" + out = {} + for a in ctx.get("deployed_agents") or []: + if not (a.get("role") and a.get("sandbox_id")) or a.get("sandbox_type") not in (None, "local"): + continue + try: + logs = subprocess.run(["docker", "logs", "--tail", "400", f"agent-{a['sandbox_id']}"], capture_output=True, + text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + continue + if found := SPENT.findall(logs.stdout + logs.stderr): + out[a["role"]] = float(found[-1]) + return out + + +def kill(pid: int, sig: int) -> None: + """SIGTERM to `agent-env run` alone, which cancels the day and tears its sandboxes down itself; SIGKILL to its + group.""" + try: + (os.killpg if sig == signal.SIGKILL else os.kill)(pid, sig) + except ProcessLookupError: + pass + + +def alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + ps = subprocess.run(["ps", "-o", "command=", "-p", str(pid)], capture_output=True, text=True) + return "agent_env.cli" in ps.stdout + + +class Runner: + def __init__(self, config: Path, current: Path | None, overrides: dict, limit: int | None): + self.config, self.overrides, self.limit = config, overrides, limit + self.s = Settings.read(config, **overrides) + self.out = sweep.RUNS / self.s.name + self.current = (current or self.out / "current.json").absolute() + self.flag = self.current.with_name("stop.flag") + self.signals, self.started, self.hubs = 0, set(), {} + old = json.loads(self.current.read_text()) if self.current.is_file() else {} + self.doc = {k: old.get(k) for k in ("instance_id", "run_id", "date", "hub", "task_id", "started_utc", "attempt", + "day", "env", "day_file", "result")} + self.seq = old.get("seq") or 0 + + # --------------------------------------------------------------------------------------- bookkeeping + + def log(self, text: str) -> None: + print(f"[{utc()}] {text}", flush=True) + + def on_signal(self, signum, _frame) -> None: + self.signals += 1 + self.log(f"signal {signum}: " + ("finishing the day, then stopping (again to stop the day now)" + if self.signals == 1 else "stopping the day now")) + + def stopping(self) -> bool: + return self.signals > 0 or self.flag.exists() + + def reload(self) -> None: + try: + s = Settings.read(self.config, **self.overrides) + except (OSError, ValueError, TypeError, tomllib.TOMLDecodeError) as e: + self.log(f"keeping the settings: {self.config} doesn't read: {e}") + return + if s.name != self.s.name: + self.log(f"keeping the pilot {self.s.name!r}: the name can't change while it runs") + s = replace(s, name=self.s.name) + self.s = s + + def state(self) -> dict: + path = self.out / "live_state.json" + return json.loads(path.read_text()) if path.is_file() else {} + + def save(self, **changes) -> None: + state = self.state() | changes + self.out.mkdir(parents=True, exist_ok=True) + tmp = self.out / "live_state.json.tmp" + tmp.write_text(json.dumps(state, indent=1) + "\n") + os.replace(tmp, self.out / "live_state.json") + + def publish(self, rows: list[dict], **changes) -> None: + self.seq += 1 + self.doc.update(changes) + self.doc.update(model=self.s.model, mode="joint", pilot=self.s.name, policy=self.s.policy or "llm", + budget_usd=self.s.budget_usd, spent_usd=round(spent(rows), 4), + history=history(rows), seq=self.seq, updated_utc=utc()) + self.doc.setdefault("final", False) + self.current.parent.mkdir(parents=True, exist_ok=True) + tmp = self.current.with_name(self.current.name + ".tmp") + tmp.write_text(json.dumps(self.doc, indent=1) + "\n") + os.replace(tmp, self.current) + + def instances(self, task: str) -> list[str]: + try: + return instances(task) + except Exception as e: + self.log(f"the store didn't read: {e!r}") + return [] + + def dates(self) -> dict[str, str]: + return {t: v["date"] for t, v in Pack3(self.s.pack).tasks.items()} + + def hub(self, task_id: str) -> str: + if task_id not in self.hubs: + pack = Pack3(self.s.pack) + raw = gzip.decompress((pack.path / "dates" / f"{pack.tasks[task_id]['date']}.json.gz").read_bytes()) + self.hubs[task_id] = json.loads(raw)["task"]["hub"] + return self.hubs[task_id] + + # --------------------------------------------------------------------------------------- the loop + + def run(self) -> int: + window = stamp(self.save_window()) + if running := self.state().get("running"): + if any(r["run"] == running["run"] and r["attempt"] == running["attempt"] + for r in sweep.results(self.out)): + self.save(running=None) + else: + self.log(f"picking up {running['run']} a{running['attempt']}, pid {running['pid']}") + self.started.add(running["task_id"]) + self.watch(running, None) + self.record(running, None) + while True: + self.reload() + rows = sweep.results(self.out) + left, now = pending(self.s, rows), time.time() + last = rows[-1] if rows and "ended_utc" in rows[-1] else None + starts = next_start(self.s, last, window, now) + reason = None + if not left: + reason = "dates: no day left in the order" + elif not affordable(spent(rows), self.s): + reason = f"budget: ${spent(rows):.2f} spent + a worst day ${self.s.worst_day_usd:g} > " \ + f"${self.s.budget_usd:g}" + elif starts + 60 * self.s.slot_minutes > window + 3600 * self.s.window_hours + 1e-6: + reason = f"window: the next slot would end after {utc(window + 3600 * self.s.window_hours)}" + elif self.limit is not None and len(self.started) >= self.limit and left[0] not in self.started: + reason = f"days: {self.limit} played" + elif self.stopping(): + reason = "stopped" + if reason: + self.end(rows, reason) + return 0 + task_id = left[0] + upcoming = {"date": self.dates()[task_id], "hub": self.hub(task_id), "task_id": task_id, + "starts_utc": utc(starts)} + if now < starts: + fresh = last is not None and last["outcome"] != "failed" and \ + now < stamp(last["ended_utc"]) + self.s.card_seconds + self.publish(rows, status="done" if fresh else "intermission", next=upcoming) + self.sleep(min(starts - now, IDLE_SECONDS)) + continue + attempts = grouped(rows).get(f"{task_id}-joint", []) + self.started.add(task_id) + self.attempt(task_id, len(attempts) + 1, upcoming["date"], upcoming["hub"]) + + def save_window(self) -> str: + state = self.state() + window = self.s.start_utc or state.get("window_start_utc") or utc() + if window != state.get("window_start_utc"): + self.save(window_start_utc=window) + return window + + def sleep(self, seconds: float) -> None: + until = time.time() + seconds + while time.time() < until and not self.stopping(): + time.sleep(min(1.0, max(0.0, until - time.time()))) + + def end(self, rows: list[dict], reason: str) -> None: + last = rows[-1] if rows and "ended_utc" in rows[-1] else None + fresh = last is not None and last["outcome"] != "failed" and \ + time.time() < stamp(last["ended_utc"]) + self.s.card_seconds + self.publish(rows, status="done" if fresh else "intermission", next=None, final=True, reason=reason) + self.flag.unlink(missing_ok=True) + self.log(f"finished, {reason}; spent ${spent(rows):.2f} on {len(history(rows))} days") + + def attempt(self, task_id: str, number: int, day: str, hub: str) -> dict: + s, run = self.s, f"{task_id}-joint" + rows = sweep.results(self.out) + self.out.mkdir(parents=True, exist_ok=True) + pilot = s.pilot(list(dict.fromkeys([r["run"] for r in rows] + [run]))) + (self.out / "pilot.json").write_text(json.dumps(asdict(pilot), indent=2) + "\n") + bundle = self.out / "bundle" + sweep.KIT3.generate(bundle, task_ids=[task_id], modes=("joint",), + pack=Path(s.pack).expanduser() if s.pack else None, cap=s.cap_joint_usd, policy=s.policy) + task = bundle_task(bundle, run) + log = self.out / "logs" / f"{run}-a{number}.log" + log.parent.mkdir(parents=True, exist_ok=True) + running = {"task_id": task_id, "date": day, "hub": hub, "run": run, "attempt": number, "log": str(log), + "task": task, "before": self.instances(task), "started_utc": utc(), "spent_before": spent(rows), + "live": {}, + "day": sum(h["outcome"] != "skipped" for h in history(rows)) + 1} + args = [*sweep.AGENT_ENV, "run", str(bundle.absolute()), "--task", run, "--model", s.model] + with log.open("w") as f: + proc = subprocess.Popen(args, stdout=f, stderr=subprocess.STDOUT, start_new_session=True) + running["pid"] = proc.pid + self.save(running=running) + self.log(f"start {run} a{number} ({day}), spent ${running['spent_before']:.2f} of ${s.budget_usd:g}") + self.publish(rows, status="starting", instance_id=None, run_id=f"{run}-a{number}", date=day, hub=hub, + task_id=task_id, + started_utc=running["started_utc"], attempt=number, day=running["day"], env=None, day_file=None, + result=None, live_usd=0.0, desks={}, next=self.following(rows, running)) + self.watch(running, proc) + return self.record(running, proc.returncode) + + def following(self, rows: list[dict], running: dict) -> dict | None: + """The day after the running one, were it to take its slot and cost the typical day.""" + s, window = self.s, stamp(self.state()["window_start_utc"]) + start = stamp(running["started_utc"]) + 60 * s.slot_minutes + left = [t for t in pending(s, rows) if t != running["task_id"]] + if not left or not affordable(running["spent_before"] + s.typical_day_usd, s) or \ + start + 60 * s.slot_minutes > window + 3600 * s.window_hours + 1e-6: + return None + return {"date": self.dates()[left[0]], "hub": self.hub(left[0]), "task_id": left[0], "starts_utc": utc(start)} + + def watch(self, running: dict, proc: subprocess.Popen | None) -> None: + """Until the day's run ends: its instance (the store's newest of its task, new since the launch), its env, each + desk's live spend, the backstops (the budget, the longest day, a second signal), and current.json.""" + rows, began, env = sweep.results(self.out), stamp(running["started_utc"]), None + upcoming = self.following(rows, running) + while proc.poll() is None if proc else alive(running["pid"]): + try: + if not running.get("instance"): + running["instance"] = next((i for i in self.instances(running["task"]) + if i not in running["before"]), None) + if running["instance"]: + self.log(f"instance {running['instance']}") + ctx = sweep.context(running["instance"]) if running.get("instance") else {} + except Exception as e: + self.log(f"the store didn't read: {e!r}") + ctx = {} + env = endpoints(ctx) or env + for role, usd in desk_spend(ctx).items(): + running["live"][role] = max(running["live"].get(role, 0.0), usd) + live = round(sum(running["live"].values()), 4) + why = "the budget" if over(running["spent_before"], live, self.s) else \ + "the longest day" if time.time() - began > 60 * self.s.longest_day_minutes else \ + "a second signal" if self.signals > 1 else None + if why and not running.get("stopped"): + running["stopped"], running["stopped_at"] = why, time.time() + self.log(f"stopping {running['run']} a{running['attempt']} now: {why} (live ${live:.2f})") + kill(running["pid"], signal.SIGTERM) + elif running.get("stopped") and time.time() - running["stopped_at"] > KILL_AFTER_SECONDS: + kill(running["pid"], signal.SIGKILL) + self.save(running=running) + self.publish(rows, status="playing" if env else "starting", instance_id=running.get("instance"), + env=env, live_usd=live, desks=dict(running["live"]), next=upcoming) + time.sleep(POLL_SECONDS) + + def record(self, running: dict, code: int | None) -> dict: + log, instance = Path(running["log"]), running.get("instance") + if instance and not sweep.INSTANCE.findall(log.read_text(errors="replace")): + with log.open("a") as f: + f.write(f"live_runner: {time.time() - stamp(running['started_utc']):.1f}s, instance {instance}\n") + rows = sweep.results(self.out) + pilot = self.s.pilot(list(dict.fromkeys([r["run"] for r in rows] + [running["run"]]))) + try: + row = sweep.outcome(pilot, running["run"], running["attempt"], log, code, running["started_utc"]) + except Exception as e: + self.log(f"the attempt's record didn't read: {e!r}") + row = {"pilot": pilot.name, "run": running["run"], "attempt": running["attempt"], + "started_utc": running["started_utc"], "wall_seconds": None, "exit": code, "instance": instance, + "outcome": "failed", "failed_step": None, "error": f"live_runner: {e!r}"[:500], + "cost_usd": None, "desks": {r: {"cost_usd": None} for r in running["live"]}, "absent": {}, + "day": None, "reward": None, "reward_L": None} + if instance and row["instance"] != instance: + self.log(f"agent-env reported instance {row['instance']}, the store had {instance}") + row.update(date=running["date"], hub=running.get("hub"), ended_utc=utc(), live_usd=running["live"], + stopped=running.get("stopped")) + row["spend_usd"] = spend(row, self.s.request_margin_usd) + if row["cost_usd"] is None: + row["cost_usd"] = row["spend_usd"] + with (self.out / "results.jsonl").open("a") as f: + f.write(json.dumps(row) + "\n") + self.save(running=None) + rows.append(row) + self.log(f"{row['outcome']} {row['run']} a{row['attempt']}: reward {row['reward']}, ${row['spend_usd']:.2f}, " + f"{row['wall_seconds'] or 0:.0f}s{', stopped: ' + row['stopped'] if row['stopped'] else ''}" + f"{', ' + row['error'][:200] if row['error'] else ''}") + done = settled(grouped(rows)[row["run"]]) + self.publish(rows, status="done" if done in ("scored", "unscored") else "intermission", + instance_id=row["instance"] or instance, live_usd=0.0, + day_file=str((self.out / row["day"]).absolute()) if row["day"] else None, + result=next((h for h in history(rows) + if h["run_id"] == f"{row['run']}-a{row['attempt']}"), None)) + return row + + +def dry_run(runner: Runner) -> int: + s, rows = runner.s, sweep.results(runner.out) + window = stamp(s.start_utc or runner.state().get("window_start_utc") or utc()) + plan, why = projection(s, rows, window, time.time()) + by = {d["task_id"]: d for d in storms(Pack3(s.pack)) if d["task_id"] in s.days} + print(f"pilot {s.name}: {s.model}{' (' + s.policy + ' desks)' if s.policy else ''}, caps ${s.cap_joint_usd:g} a " + f"desk; budget ${s.budget_usd:g}, a day starts only if spent + ${s.worst_day_usd:g} fits; " + f"{s.slot_minutes:g}-minute slots in {s.window_hours:g} h from {utc(window)}") + print(f"spent ${spent(rows):.2f} on {len(history(rows))} days; current.json at {runner.current}") + for i, p in enumerate(plan, 1): + d = by[p["task_id"]] + print(f"{i:3} {utc(p['starts'])} {d['date']} {d['split']:5} ${p['spent_before']:7.2f} {describe(d)}") + print(f"{len(plan)} days at the board's typical ${s.typical_day_usd:g}; the last is the last because of {why}") + return 0 + + +def setting(pairs: list[str]) -> dict: + kinds = {f.name: f.type for f in fields(Settings) if f.name != "days"} + out = {} + for pair in pairs: + key, _, value = pair.partition("=") + if key not in kinds: + raise SystemExit(f"--set {pair}: settings are {', '.join(kinds)}") + out[key] = float(value) if kinds[key] is float else value + return out + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = p.add_subparsers(dest="command", required=True) + r = sub.add_parser("run", help="Play the days.") + r.add_argument("--config", type=Path, default=CONFIG) + r.add_argument("--current", type=Path, help="Where current.json goes. Default: results/runs//current.json.") + r.add_argument("--policy", choices=("llm", "script"), help="The desks' policy. Default: the config's.") + r.add_argument("--name", help="The pilot's name. Default: the config's.") + r.add_argument("--days", type=int, help="Play at most this many days, then stop.") + r.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", + help="Override a setting of the config (not days), e.g. slot_minutes=15. Repeatable.") + r.add_argument("--dry-run", action="store_true", help="Print the schedule and play nothing.") + o = sub.add_parser("order", help="Rank the pack's days by storm and print them.") + o.add_argument("--pack", type=Path, help="Default: the packaged b6-v3.") + o.add_argument("--write", type=Path, help="Write a config with the default settings and this order.") + args = p.parse_args(argv) + for key in ("config", "current", "pack", "write"): + if getattr(args, key, None) is not None: + setattr(args, key, getattr(args, key).expanduser().absolute()) + os.chdir(ROOT) + if args.command == "order": + days = order(storms(Pack3(args.pack))) + for i, d in enumerate(days, 1): + print(f"{i:3} {d['task_id']} {d['split']:5} {d['score']:6.1f} {describe(d)}") + if args.write: + args.write.write_text(config_text(days)) + return 0 + runner = Runner(args.config, args.current, {"policy": args.policy, "name": args.name, **setting(args.set)}, + args.days) + if args.dry_run: + return dry_run(runner) + signal.signal(signal.SIGTERM, runner.on_signal) + signal.signal(signal.SIGINT, runner.on_signal) + return runner.run() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live_runner.toml b/scripts/live_runner.toml new file mode 100644 index 0000000..f8fd91c --- /dev/null +++ b/scripts/live_runner.toml @@ -0,0 +1,91 @@ +# The live runner's settings (scripts/live_runner.py), read again before every day: edits apply from the next day on. +name = "occ3-live-astra" # results/runs/: pilot.json, results.jsonl, days/, transcripts/, logs/ +model = "openai/gpt-6-astra" +# policy = "script" # the env's scripted desks, $0 +cap_joint_usd = 23.5 # each desk's cap, as registered for the board +request_margin_usd = 4.0 # one worst request a desk may make past its cap +budget_usd = 430.0 # model spend may not pass it +worst_day_usd = 42.0 # a day starts only if the spend so far plus this fits (the board's worst: $41.46) +typical_day_usd = 22.68 # the board's mean Astra day: --dry-run's projection only +slot_minutes = 80.0 # a day that ends early is followed by an intermission until its slot ends +min_intermission_minutes = 5.0 # after a day past its slot, or a failed attempt +card_seconds = 60.0 # status "done" for this long after a day, then "intermission" +window_hours = 24.0 # a day starts only if its slot ends within the window +# start_utc = "2026-10-23T16:00:00Z" # the first day's start; default: when the runner first runs +longest_day_minutes = 150.0 # a day still running then is stopped (the board's longest: 84) + +# The days in play order (`live_runner.py order`): the training days by storm (GDP hours, twice the ground-stop hours, +# at JFK and BOS, plus one per ten flights cancelled), each season by score with a storm's second day after the rest, +# the seasons interleaved; then the eval days the board played, the same way. Reorder or delete freely. +days = [ + # training days + "b6-20240803-0400", # 1 2024-08-03 score 81.6: summer; JFK GDP 11.2 h, 15 ground stops 15.6 h; BOS GDP 16.6 h, 9 ground stops 6.9 h; 88 cancelled + "b6-20230627-0400", # 2 2023-06-27 score 79.1: summer; JFK GDP 7.2 h, 10 ground stops 10.2 h; BOS GDP 19.1 h, 7 ground stops 7.5 h; 174 cancelled + "b6-20250119-0400", # 3 2025-01-19 score 33.9: winter; JFK GDP 9.8 h, 4 ground stops 4.5 h; BOS no program; 151 cancelled + "b6-20240818-0400", # 4 2024-08-18 score 68.3: summer; JFK GDP 19.4 h, 7 ground stops 8.6 h; BOS GDP 21.1 h, 2 ground stops 2.3 h; 60 cancelled + "b6-20260727-0400", # 5 2026-07-27 score 66.5: summer; JFK GDP 11 h, 18 ground stops 16.7 h; BOS GDP 13.9 h, 2 ground stops 1.6 h; 50 cancelled + "b6-20240630-0400", # 6 2024-06-30 score 54: summer; JFK GDP 14 h, 8 ground stops 9.6 h; BOS 6 ground stops 5.1 h; 106 cancelled + "b6-20240106-0400", # 7 2024-01-06 score 24.2: winter; JFK GDP 5.2 h; BOS GDP 10.8 h, 4 ground stops 3.1 h; 20 cancelled + "b6-20260729-0400", # 8 2026-07-29 score 51.7: summer; JFK 8 ground stops 7.6 h; BOS GDP 27.7 h, 2 ground stops 1.6 h; 56 cancelled + "b6-20250813-0400", # 9 2025-08-13 score 51.3: summer; JFK GDP 15.4 h, 9 ground stops 9.5 h; BOS GDP 8.8 h, 2 ground stops 1.6 h; 49 cancelled + "b6-20240809-0400", # 10 2024-08-09 score 34.5: summer; JFK GDP 12.2 h; BOS GDP 14.3 h, 1 ground stop 1.3 h; 54 cancelled + "b6-20250206-0400", # 11 2025-02-06 score 18.9: winter; JFK GDP 6.7 h; BOS GDP 10.7 h; 15 cancelled + "b6-20230701-0400", # 12 2023-07-01 score 33.9: summer; JFK GDP 16.9 h; BOS GDP 9.8 h, 4 ground stops 3.5 h; 2 cancelled + "b6-20240614-0400", # 13 2024-06-14 score 32.9: summer; JFK GDP 11.3 h, 3 ground stops 3.2 h; BOS GDP 11.4 h, 2 ground stops 0.7 h; 24 cancelled + "b6-20230817-0400", # 14 2023-08-17 score 32.4: summer; JFK GDP 4.3 h; BOS GDP 19.4 h, 5 ground stops 3.5 h; 17 cancelled + "b6-20240116-0400", # 15 2024-01-16 score 13.7: winter; JFK no program; BOS GDP 2.5 h, 3 ground stops 3.8 h; 36 cancelled + "b6-20230624-0400", # 16 2023-06-24 score 29.5: summer; JFK GDP 4.1 h, 7 ground stops 6.7 h; BOS GDP 4.5 h, 1 ground stop 1.2 h; 51 cancelled + "b6-20230807-0400", # 17 2023-08-07 score 29.3: summer; JFK 6 ground stops 5.6 h; BOS GDP 3.7 h, 5 ground stops 3.9 h; 66 cancelled + "b6-20240626-0400", # 18 2024-06-26 score 24.7: summer; JFK GDP 9.4 h, 4 ground stops 4.7 h; BOS 3 ground stops 2.2 h; 15 cancelled + "b6-20250218-0400", # 19 2025-02-18 score 13.2: winter; JFK GDP 9.2 h; BOS GDP 3.8 h; 2 cancelled + "b6-20240731-0400", # 20 2024-07-31 score 22.6: summer; JFK GDP 7 h, 3 ground stops 1.9 h; BOS GDP 9 h, 1 ground stop 1.3 h; 2 cancelled + "b6-20240729-0400", # 21 2024-07-29 score 22.2: summer; JFK GDP 7 h; BOS GDP 15.2 h; 0 cancelled + "b6-20230814-0400", # 22 2023-08-14 score 20.6: summer; JFK no program; BOS GDP 15.4 h, 1 ground stop 1.1 h; 30 cancelled + "b6-20230829-0400", # 23 2023-08-29 score 16.6: summer; JFK 2 ground stops 0.4 h; BOS GDP 8.7 h, 3 ground stops 2 h; 31 cancelled + "b6-20240228-0400", # 24 2024-02-28 score 11.9: winter; JFK 1 ground stop 1.1 h; BOS GDP 7 h, 1 ground stop 1.2 h; 3 cancelled + "b6-20230825-0400", # 25 2023-08-25 score 14.6: summer; JFK 4 ground stops 2.3 h; BOS GDP 7.9 h; 21 cancelled + "b6-20230812-0400", # 26 2023-08-12 score 14: summer; JFK GDP 6 h, 1 ground stop 1.3 h; BOS 1 ground stop 1.1 h; 32 cancelled + "b6-20240607-0400", # 27 2024-06-07 score 11.4: summer; JFK no program; BOS GDP 6.2 h, 2 ground stops 2.6 h; 0 cancelled + "b6-20231226-0400", # 28 2023-12-26 score 8.3: winter; JFK GDP 2.8 h, 1 ground stop 1.5 h; BOS 1 ground stop 1.1 h; 3 cancelled + "b6-20230820-0400", # 29 2023-08-20 score 11.1: summer; JFK no program; BOS GDP 9.5 h; 16 cancelled + "b6-20240609-0400", # 30 2024-06-09 score 8.3: summer; JFK no program; BOS GDP 8.1 h; 2 cancelled + "b6-20230810-0400", # 31 2023-08-10 score 7.1: summer; JFK GDP 4.3 h; BOS no program; 28 cancelled + "b6-20251211-0400", # 32 2025-12-11 score 7.7: winter; JFK GDP 7.7 h; BOS no program; 0 cancelled + "b6-20240817-0400", # 33 2024-08-17 score 47.7: summer; JFK GDP 19.8 h, 7 ground stops 8.6 h; BOS GDP 10.6 h; 1 cancelled + "b6-20240629-0400", # 34 2024-06-29 score 47.6: summer; JFK GDP 14 h, 9 ground stops 10.6 h; BOS 6 ground stops 5.1 h; 22 cancelled + "b6-20230818-0400", # 35 2023-08-18 score 24.7: summer; JFK GDP 4.3 h; BOS GDP 11.2 h, 5 ground stops 3.5 h; 22 cancelled + "b6-20260220-0400", # 36 2026-02-20 score 6.9: winter; JFK GDP 6.2 h; BOS no program; 7 cancelled + "b6-20230808-0400", # 37 2023-08-08 score 24.2: summer; JFK 6 ground stops 5.6 h; BOS GDP 3.7 h, 5 ground stops 3.9 h; 15 cancelled + "b6-20240730-0400", # 38 2024-07-30 score 21.8: summer; JFK GDP 7 h, 2 ground stops 1.6 h; BOS GDP 9 h, 1 ground stop 1.3 h; 0 cancelled + "b6-20240728-0400", # 39 2024-07-28 score 15.2: summer; JFK no program; BOS GDP 15.2 h; 0 cancelled + "b6-20260119-0400", # 40 2026-01-19 score 6.5: winter; JFK 1 ground stop 1 h; BOS GDP 3.5 h; 10 cancelled + "b6-20260730-0400", # 41 2026-07-30 score 11.5: summer; JFK no program; BOS GDP 8.6 h; 29 cancelled + "b6-20230815-0400", # 42 2023-08-15 score 8: summer; JFK no program; BOS GDP 7 h; 10 cancelled + "b6-20230702-0400", # 43 2023-07-02 score 35.9: summer; JFK GDP 14.7 h; BOS GDP 9.8 h, 4 ground stops 3.5 h; 44 cancelled + "b6-20240816-0400", # 44 2024-08-16 score 27.7: summer; JFK GDP 8.6 h; BOS GDP 12.2 h, 4 ground stops 3.3 h; 3 cancelled + # eval days: Astra played these on the board + "b6-20230709-0400", # 45 2023-07-09 score 79.9: summer; JFK GDP 16.7 h, 8 ground stops 7.8 h; BOS GDP 15.5 h, 7 ground stops 5.5 h; 211 cancelled + "b6-20231218-0400", # 46 2023-12-18 score 34.3: winter; JFK GDP 9.1 h; BOS GDP 10.3 h, 6 ground stops 5.4 h; 41 cancelled + "b6-20250708-0400", # 47 2025-07-08 score 74.4: summer; JFK GDP 32.5 h, 5 ground stops 4.8 h; BOS GDP 23 h, 2 ground stops 2.5 h; 43 cancelled + "b6-20230713-0400", # 48 2023-07-13 score 58.1: summer; JFK GDP 10.1 h, 12 ground stops 11.3 h; BOS GDP 8.5 h, 9 ground stops 7.7 h; 15 cancelled + "b6-20251219-0400", # 49 2025-12-19 score 28.1: winter; JFK GDP 14.2 h; BOS GDP 8.8 h; 51 cancelled + "b6-20260709-0400", # 50 2026-07-09 score 48.8: summer; JFK GDP 21.5 h, 10 ground stops 7.7 h; BOS GDP 6.2 h; 57 cancelled + "b6-20250712-0400", # 51 2025-07-12 score 45.1: summer; JFK GDP 14.8 h, 9 ground stops 8.7 h; BOS GDP 5.4 h, 2 ground stops 2.6 h; 23 cancelled + "b6-20251214-0400", # 52 2025-12-14 score 22.4: winter; JFK GDP 10.5 h, 1 ground stop 1.5 h; BOS no program; 89 cancelled + "b6-20230729-0400", # 53 2023-07-29 score 40.2: summer; JFK GDP 10 h, 6 ground stops 3.6 h; BOS GDP 8.8 h, 5 ground stops 3.6 h; 70 cancelled + "b6-20240715-0400", # 54 2024-07-15 score 34.5: summer; JFK GDP 13.3 h, 4 ground stops 4.3 h; BOS GDP 2.4 h, 5 ground stops 4.5 h; 12 cancelled + "b6-20251230-0400", # 55 2025-12-30 score 20.5: winter; JFK GDP 11.2 h; BOS GDP 5.3 h, 1 ground stop 1.5 h; 10 cancelled + "b6-20240713-0400", # 56 2024-07-13 score 22.3: summer; JFK GDP 9.7 h, 4 ground stops 3.7 h; BOS GDP 1.4 h, 1 ground stop 1.4 h; 10 cancelled + "b6-20231210-0400", # 57 2023-12-10 score 15.8: winter; JFK GDP 10.6 h; BOS GDP 5.2 h; 0 cancelled + "b6-20260701-0400", # 58 2026-07-01 score 18.5: summer; JFK GDP 4.9 h, 1 ground stop 1 h; BOS GDP 7.2 h, 2 ground stops 1.8 h; 8 cancelled + "b6-20230616-0400", # 59 2023-06-16 score 18: summer; JFK GDP 7.3 h, 1 ground stop 0.9 h; BOS 2 ground stops 2.1 h; 47 cancelled + "b6-20221207-0400", # 60 2022-12-07 score 11.8: winter; JFK GDP 1.2 h, 2 ground stops 1.8 h; BOS GDP 6.4 h; 6 cancelled + "b6-20260614-0400", # 61 2026-06-14 score 16.8: summer; JFK GDP 8.2 h; BOS GDP 3.5 h; 51 cancelled + "b6-20260715-0400", # 62 2026-07-15 score 15.3: summer; JFK GDP 3.7 h, 3 ground stops 3.6 h; BOS 2 ground stops 1.7 h; 10 cancelled + "b6-20250111-0400", # 63 2025-01-11 score 10.2: winter; JFK no program; BOS GDP 5.6 h, 2 ground stops 2 h; 6 cancelled + "b6-20240722-0400", # 64 2024-07-22 score 8.4: summer; JFK GDP 6.3 h; BOS no program; 21 cancelled + "b6-20230606-0400", # 65 2023-06-06 score 7.4: summer; JFK no program; BOS 5 ground stops 3.6 h; 2 cancelled + "b6-20251202-0400", # 66 2025-12-02 score 8.6: winter; JFK no program; BOS GDP 6 h, 1 ground stop 1.3 h; 0 cancelled + "b6-20230708-0400", # 67 2023-07-08 score 78.3: summer; JFK GDP 20.7 h, 13 ground stops 12.4 h; BOS GDP 11.8 h, 8 ground stops 9.2 h; 26 cancelled + "b6-20240623-0400", # 68 2024-06-23 score 58.9: summer; JFK GDP 19.5 h, 5 ground stops 4 h; BOS GDP 17.5 h, 5 ground stops 4.1 h; 57 cancelled +] diff --git a/scripts/live_server.py b/scripts/live_server.py new file mode 100644 index 0000000..12db184 --- /dev/null +++ b/scripts/live_server.py @@ -0,0 +1,838 @@ +"""The live stream's episode server: it follows the airops-occ3 day the runner plays (current.json: its instance, the +env's data/get base, the day file once graded), plays the env's calls again in its own day (agentenv_airops.occ.onair) +and serves the viewer the day so far as Episode JSON, with its traffic sidecars, the static assets and the viewer. +It only reads data/get, every --poll seconds; the env never sees it. + + prebuild TASK_ID... --live LIVE --inputs cache/inputs each day's news and hub traffic into LIVE (seconds a day) + serve --current current.json --live LIVE --assets A HTTP on --port: /live/state.json, /live/episode.json, the + sidecars, /stage /radar /twin from A, /web the viewer + feed DAY.json... --current current.json a fake runner and env: recorded days revealed a watch + every --watch-seconds behind a data/get of their own + attach BUNDLE TASK --current current.json current.json for a local `agent-env run` of BUNDLE TASK + +The viewer: http://HOST:PORT/web/index.html?live=http://HOST:PORT (CONTRACT.md, "Live server → viewer"). + +serve checkpoints the day's replay in LIVE/ckpt/.pkl at each watch and at most every CKPT seconds between (a +few MB, a tenth of a second); started again on the same run (attempt, instance) and code, it goes on from there in +seconds instead of playing the day again from 04:00. + + .venv/bin/python scripts/live_server.py prebuild b6-20260715-0400 --live /tmp/live --inputs cache/inputs + .venv/bin/python scripts/live_server.py serve --current /tmp/current.json --live /tmp/live --assets VIEWER_ASSETS +""" + +import argparse +import collections +import functools +import gzip +import hashlib +import json +import mimetypes +import os +import pickle +import random +import sys +import threading +import time +import traceback +from datetime import UTC, datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import unquote, urlsplit + +import httpx + +from agentenv_airops import export +from agentenv_airops.occ import onair +from agentenv_airops.occ.pack import Pack3 + +ROOT = Path(__file__).resolve().parents[1] +RPC = "/agentenv" +STALE = 30.0 +REBUILD = 15.0 +CATCH_UP = 2 +KEEP = 8 +GET_TIMEOUT = 60.0 +CKPT = 30.0 +RADAR = (240, 1900, 20, 1600) + + +def utc(t: float | None = None) -> str: + return datetime.fromtimestamp(time.time() if t is None else t, UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def read_json(path: Path) -> dict | None: + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return None + + +def write_atomic(path: Path, body: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + part = path.with_name(path.name + ".part") + part.write_bytes(body) + os.replace(part, path) + + +def data_get(base: str, timeout: float = GET_TIMEOUT) -> dict: + """The env's data/get record (the verifier's call: JSON-RPC at the env's base).""" + r = httpx.post(base.rstrip("/") + RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/get", "params": {}}, + timeout=timeout) + r.raise_for_status() + body = r.json() + if "error" in body: + raise RuntimeError(f"data/get: {body['error'].get('message')}") + return body["result"]["parts"][0]["data"] + + +# ---------------------------------------------------------------------------------------------------------- prebuild + + +def prebuild(args) -> int: + """Each day's news and traffic (seconds), and its radar frames where neither LIVE nor --assets has them: IEM's + NEXRAD composites every 20 minutes from 04:00 to 07:40 the next day, as the v3 episodes have them (minutes).""" + pack = Pack3(args.pack) + for task_id in args.tasks: + started = time.monotonic() + world = pack.world(task_id) + out = onair.prebuild(world, args.live, args.inputs.expanduser()) + line = f"{task_id}: {out['news']} news, {out['aircraft']} other aircraft at the hub in " \ + f"{time.monotonic() - started:.1f}s" + have = [r for r in (args.live, args.assets) if r and (r / "radar" / task_id / "index.json").is_file()] + if have: + line += f"; radar in {have[0]}" + elif args.radar: + started = time.monotonic() + line += f"; radar {bake_radar(world, args.live / 'radar' / task_id)} in {time.monotonic() - started:.0f}s" + else: + line += "; NO RADAR" + print(line, flush=True) + return 0 + + +def bake_radar(world, out: Path) -> str: + sys.path.insert(0, str(ROOT / "scripts")) + import prebake_radar + + task = world.task + spec = out.parent / f"{task.task_id}.task.json" + write_atomic(spec, json.dumps({"task_id": task.task_id, "date": task.date, "tz": task.tz, + "window": list(task.window), "airports": {c: a.to_dict() for c, a in + task.airports.items()}}).encode()) + code = prebake_radar.main(["--task", str(spec), "--from", str(RADAR[0]), "--to", str(RADAR[1]), "--step", + str(RADAR[2]), "--width", str(RADAR[3]), "--out", str(out)]) + spec.unlink() + return "baked" if code == 0 else "FAILED" + + +# ---------------------------------------------------------------------------------------------------------- serve + + +@functools.cache +def code() -> str: + """The replay's code and packs (the package's files): a checkpoint is good only on what wrote it.""" + root = Path(onair.__file__).resolve().parents[1] + h = hashlib.sha256() + for p in sorted(root.rglob("*")): + if p.is_file() and "__pycache__" not in p.parts and not p.name.startswith("."): + h.update(f"{p.relative_to(root)}\0{p.stat().st_size}\0".encode() + p.read_bytes()) + return h.hexdigest() + + +class Day: + """One run of one day: its world, its replay and what the stream shows of it.""" + + def __init__(self, cur: dict, live: Path, assets: Path, pack: Pack3, inputs: Path | None): + self.run_id, self.instance_id = cur.get("run_id"), cur.get("instance_id") + self.task_id, self.mode, self.attempt = cur["task_id"], cur.get("mode") or "joint", cur.get("attempt") + self.model, self.pilot = cur.get("model") or "", cur.get("pilot") or "live" + self.run_label = f"{self.pilot}-{self.mode}" + self.world = pack.world(self.task_id) + if not (live / "news" / f"{self.task_id}.json").is_file(): + if inputs is None: + print(f"{self.task_id} isn't prebuilt in {live}: no news and no sidecar (prebuild it, or --inputs)", + flush=True) + else: + onair.prebuild(self.world, live, inputs) + self.statics = onair.Statics(self.world, live, assets) + self.follower = onair.Follower(self.world, self.mode) + self.stem = onair.stem(self.run_label, self.model, self.task_id) + self.view: dict | None = None + self.viewed = 0.0 + self.feed: collections.deque = collections.deque(maxlen=onair.FEED) + self.last: dict[str, dict] = {} + self.opened: dict[int, float] = {} + self.started = time.time() + self.ep: dict | None = None + self.sidecar: str | None = None + self.final = self.sealed = False + self.failing: float | None = None + self.error: str | None = None + self.data_grade: dict | None = None + self.lag = 0 + self.ckpt = live / "ckpt" / f"{self.run_id}.pkl" + self.saved: tuple[float, int] = (0.0, -1) + self.resumed = self.resume() + + def resume(self) -> bool: + """The replay from this run's checkpoint, when it is this run's (its instance's) on this code.""" + if not self.ckpt.is_file(): + return False + started = time.monotonic() + try: + with self.ckpt.open("rb") as fh: + head = pickle.load(fh) + mine = {k: head.get(k) for k in self.key()} == self.key() and ( + None in (self.instance_id, head.get("instance_id")) or self.instance_id == head["instance_id"]) + if not mine: + print(f"{self.run_id}: the checkpoint is another run's ({head}); playing the day from its start", + flush=True) + return False + state = onair.unpickled(fh.read(), self.world) + except Exception as e: + print(f"{self.run_id}: the checkpoint doesn't load ({type(e).__name__}: {e}); playing the day from its " + "start", flush=True) + return False + self.follower, self.feed, self.last = state["follower"], state["feed"], state["last"] + self.opened, self.started, self.data_grade = state["opened"], state["started"], state["data_grade"] + day = self.follower.day + self.saved = (time.monotonic(), day.k) + print(f"{utc()} {self.run_id}: resumed at watch {day.k}, {self.follower.played} calls" + f"{', done' if day.done else ''} in {time.monotonic() - started:.2f}s", flush=True) + return True + + def save(self) -> None: + """The replay so far into its checkpoint (tmp + rename), at a new watch, once done, and at most every CKPT + seconds between.""" + day = self.follower.day + if self.saved[1] == day.k and not day.done and time.monotonic() - self.saved[0] < CKPT: + return + started = time.monotonic() + state = {"follower": self.follower, "feed": self.feed, "last": self.last, "opened": self.opened, + "started": self.started, "data_grade": self.data_grade} + body = pickle.dumps(self.key() | {"instance_id": self.instance_id}) + onair.pickled(state, self.world) + write_atomic(self.ckpt, body) + self.saved = (time.monotonic(), day.k) + print(f"{utc()} {self.run_id}: checkpoint at watch {day.k}, {self.follower.played} calls: " + f"{len(body) / 1e6:.1f} MB in {self.saved[0] - started:.2f}s", flush=True) + + def key(self) -> dict: + return {"run_id": self.run_id, "task_id": self.task_id, "mode": self.mode, "attempt": self.attempt, + "code": code()} + + def restart(self) -> None: + self.follower = onair.Follower(self.world, self.mode) + self.view, self.feed, self.last, self.opened = None, collections.deque(maxlen=onair.FEED), {}, {} + + def follow(self, data: dict) -> bool: + """The record's news played, at most CATCH_UP watches past the replay's at a time so that a server behind the + day shows it watch by watch, and after the day's end the desks' calls the env logs after it (the done day's + episode built again); whether anything changed.""" + if data.get("task_id") is None: + return False + self.lag = max(0, data.get("watch", 0) - self.follower.day.k) + limit = self.follower.day.k + CATCH_UP + if any(c["watch"] > limit for c in data["calls"]): + data = {**data, "calls": [c for c in data["calls"] if c["watch"] <= limit], "done": False, "digest": None, + "notices": [n for n in data["notices"] if n["watch"] <= limit]} + try: + new = self.follower.follow(data) + except onair.Diverged as e: + print(f"{self.run_id}: {e}; playing the record again from its start", flush=True) + self.restart() + new = self.follower.follow(data) + if new: + self.final = False + now = time.time() + day = self.follower.day + self.opened.setdefault(day.k, now) + for x in new: + if x["tool"] != "notice": + self.last[x["desk"]] = {"last": {"seq": x["seq"], "tool": x["tool"], "t": x["t"], + "text": onair.summary(x, self.world.task)}, "at": utc(now)} + by_id = {n["id"]: n for n in self.statics.news} + self.feed.extend(row | {"at": utc(now)} for row in onair.feed(self.follower, new, by_id)) + graded = self.data_grade is None and data.get("done") and data.get("grade") is not None + if graded: + self.data_grade = data["grade"] + return bool(new) or (day.done and not self.final) or bool(graded) + + def build(self, live_dir: Path, cur: dict, force: bool = False) -> None: + """The episode now: the final one once done, else the day so far, its projection and sidecar renewed at a new + watch and at most every REBUILD seconds after a decision.""" + day = self.follower.day + record = {"folder": self.pilot, "run": f"{self.task_id}-{self.mode}", "attempt": cur.get("attempt"), + "cost_usd": (cur.get("result") or {}).get("cost_usd"), + "wall_seconds": (cur.get("result") or {}).get("wall_seconds"), + "desks": (cur.get("result") or {}).get("desks")} + if day.done: + if not self.final: + self.ep = onair.final(self.follower, self.statics, model=self.model, run_label=self.run_label, + record=record) + body = self.statics.sidecar(self.ep, self.ep["watches"][-1]["t"]) + self.sidecar = None if body is None else f"{self.stem}.stage.json.gz" + if body is not None: + write_atomic(live_dir / self.sidecar, body) + self.final = True + else: + self.ep["record"] = record + return + now = time.time() + stale = self.view is None or self.view["watch"] != day.k or ( + self.view["version"] != day.version and now - self.viewed >= REBUILD) + if stale or force: + self.view, self.viewed = onair.projection(self.follower), now + self.ep = onair.now(self.follower, self.statics, model=self.model, run_label=self.run_label, record=record, + view=self.view) + if stale or force or self.sidecar is None: + body = self.statics.sidecar(self.ep, day.t) + if body is not None: + self.sidecar = f"{self.stem}.stage-{int(now * 1000)}.json.gz" + write_atomic(live_dir / self.sidecar, body) + old = sorted(live_dir.glob(f"{self.stem}.stage-*.json.gz"), key=lambda p: p.stat().st_mtime) + for p in old[:-KEEP]: + p.unlink(missing_ok=True) + self.ep["stage"] = None if self.sidecar is None else { + "hub": self.statics.hub, "url": f"../stage/{self.statics.hub}/stage.json.gz", "traffic": self.sidecar} + + def live(self, cur: dict, status: str, seq: int) -> dict: + f, day = self.follower, self.follower.day + task = self.world.task + seen = sorted(self.opened) + mean = (self.opened[seen[-1]] - self.opened[seen[0]]) / (seen[-1] - seen[0]) if len(seen) > 1 else None + g = day.grade if day.done else None + result = None + if g is not None: + res = cur.get("result") or {} + result = {"reward": g["reward"], "reward_L": g["reward_L"], "cost": g["cost"], "best": g["best"], + "bound": g["bound"], "B0": g["B0"], "end_reason": g["end_reason"], + "audit_ok": ((self.ep or {}).get("audit") or {}).get("ok"), + "cost_usd": res.get("cost_usd"), "wall_seconds": res.get("wall_seconds")} + exact = not f.mismatches and (self.data_grade is None or g is None or self.data_grade == onair.jsonable(g)) + return { + "status": status, "seq": seq, "run_id": self.run_id, "task_id": self.task_id, "date": task.date, + "hub": task.hub, "instance_id": self.instance_id, "model": self.model, + "model_label": export.NAMES.get(self.model, self.model), + "mode": self.mode, "pilot": self.pilot, "watch": day.k, "watches": len(day.times), "t": day.t, + "decided_until": onair.decided_until(day, (self.ep or {}).get("executed") or []), + "turn": None if day.turn is None else {k: day.turn[k] for k in ("n", "desk", "kind", "calls")}, + "desks": onair.desks(f, self.last), "feed": list(self.feed), + "projected": None if self.view is None or day.done else self.view["costs"], + "next_watch_at": utc(self.opened[day.k] + mean) if mean and day.k in self.opened and not day.done + else None, + "watch_wall_s": None if mean is None else round(mean, 1), + "started_utc": cur.get("started_utc") or utc(self.started), "updated_utc": utc(), + "wall_seconds": round(time.time() - self.started, 1), + "spent_usd": cur.get("live_usd"), + "total_usd": None if cur.get("spent_usd") is None and cur.get("live_usd") is None else + round((cur.get("spent_usd") or 0) + (cur.get("live_usd") or 0), 2), "budget_usd": cur.get("budget_usd"), + "result": result, "exact": exact, "mismatches": len(f.mismatches), "lag": self.lag, + "next": cur.get("next"), "history": cur.get("history") or [], "final": bool(cur.get("final")), + "error": self.error, + } + + +class Server: + def __init__(self, args): + self.args = args + self.live_dir = args.live / "live" + self.live_dir.mkdir(parents=True, exist_ok=True) + self.pack = Pack3(args.pack) + self.seq = int(time.time()) + self.day: Day | None = None + self.days: dict[str, dict] = {} + self.lock = threading.Lock() + self.state = b"{}" + self.episode = (b"null", gzip.compress(b"null"), "0") + self.status = "intermission" + self.last_cur: dict = {} + + def loop(self) -> None: + while True: + started = time.monotonic() + try: + self.step() + except Exception as e: + traceback.print_exc() + if self.day is not None: + self.day.error = f"{type(e).__name__}: {e}" + self.publish() + behind = self.day is not None and self.day.lag > CATCH_UP and not self.day.final + time.sleep(0.05 if behind else max(0.2, self.args.poll - (time.monotonic() - started))) + + def current(self) -> dict: + cur = read_json(self.args.current) if self.args.current else None + if cur is None: + cur = self.last_cur + if self.args.env: + cur = {**cur, "status": cur.get("status") or "playing", "env": {"base": self.args.env}, + "task_id": cur.get("task_id") or self.args.task, "run_id": cur.get("run_id") or self.args.task, + "mode": cur.get("mode") or "joint"} + self.last_cur = cur + return cur + + def step(self) -> None: + cur = self.current() + status = cur.get("status") or "intermission" + moved = self.day is not None and None not in (self.day.instance_id, cur.get("instance_id")) and \ + self.day.instance_id != cur["instance_id"] + known = status in ("starting", "playing", "done") or bool(cur.get("day_file")) + if cur.get("task_id") and cur.get("run_id") and known and ( + self.day is None or self.day.run_id != cur["run_id"] or moved): + print(f"{utc()} day {cur['run_id']} ({cur['task_id']}, {status})", flush=True) + for p in (self.args.live / "ckpt").glob("*"): + if p.name != f"{cur['run_id']}.pkl": + p.unlink(missing_ok=True) + self.day = Day(cur, self.args.live, self.args.assets, self.pack, self.args.inputs) + self.day.instance_id = cur.get("instance_id") + self.day.build(self.live_dir, cur, force=True) + if self.day.final: + self.archive(self.day, cur) + self.status = self.standing(self.day, cur, status) if self.day.resumed else "starting" + self.publish(cur) + day = self.day + if day is None: + self.status = status + self.publish(cur) + return + day.instance_id = cur.get("instance_id") or day.instance_id + changed = False + if not day.sealed and status in ("playing", "done", "intermission"): + data, base = None, (cur.get("env") or {}).get("base") if status == "playing" else None + if base: + try: + data = data_get(base) + day.failing, day.error = None, None + except Exception as e: + day.failing = day.failing or time.time() + day.error = f"data/get: {type(e).__name__}: {e}"[:300] + filed = (data is None or not data.get("done")) and status != "playing" and bool(cur.get("day_file")) + if filed: + data = read_json(Path(cur["day_file"])) + if data is not None: + changed = day.follow(data) + day.sealed = filed and day.final + if changed: + day.build(self.live_dir, cur) + if day.final: + self.archive(day, cur) + try: + day.save() + except Exception as e: + print(f"{utc()} {day.run_id}: no checkpoint ({type(e).__name__}: {e})", flush=True) + elif day.final: + day.build(self.live_dir, cur) + if day.final: + day.error, day.lag = None, 0 + self.status = self.standing(day, cur, status) + self.publish(cur, changed) + + @staticmethod + def standing(day: Day, cur: dict, status: str) -> str: + if day.final: + return "intermission" if status == "intermission" else "done" + if status == "starting": + return "starting" + if day.failing and time.time() - day.failing > STALE: + return "stale" + if status == "playing": + return "playing" + if status == "done" or cur.get("day_file"): + return "ending" + return status + + def archive(self, day: Day, cur: dict) -> None: + """The done day's episode kept beside the live one (its sidecar is there already), in /live/index.json.""" + ep = {**day.ep, "live": day.live(cur, "done", self.seq)} + name = f"{day.run_id}.json" + write_atomic(self.live_dir / name, json.dumps(ep, ensure_ascii=False, separators=(",", ":")).encode()) + self.days[day.run_id] = export.entry(ep, name, self.args.assets) | {"live": True} + index = {"version": 1, "episodes": list(self.days.values())} + write_atomic(self.live_dir / "index.json", (json.dumps(index, indent=1) + "\n").encode()) + print(f"{utc()} {day.run_id} done: reward {ep['reward']}, exact {ep['live']['exact']}", flush=True) + + def publish(self, cur: dict | None = None, changed: bool = True) -> None: + cur = cur if cur is not None else self.last_cur + day = self.day + with self.lock: + if day is None: + self.seq += 1 + state = {"seq": self.seq, "live": {"status": self.status, "seq": self.seq, "next": cur.get("next"), + "history": cur.get("history") or [], "final": bool(cur.get("final")), + "updated_utc": utc()}} + self.state = json.dumps(state).encode() + return + previous = json.loads(self.state).get("live", {}) if self.state else {} + live = day.live(cur, self.status, self.seq) + keys = ("seq", "updated_utc", "wall_seconds") + if not changed and {k: v for k, v in live.items() if k not in keys} == \ + {k: v for k, v in previous.items() if k not in keys}: + return + self.seq += 1 + live["seq"] = self.seq + ep = {**day.ep, "live": live} + body = json.dumps(ep, ensure_ascii=False, separators=(",", ":")).encode() + self.episode = (body, gzip.compress(body, compresslevel=5), str(self.seq)) + self.state = json.dumps({"seq": self.seq, "live": live}, ensure_ascii=False).encode() + write_atomic(self.live_dir / "episode.json", body) + + def bind(self) -> ThreadingHTTPServer: + server = self + assets, web = self.args.assets, self.args.web + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *a): + pass + + def cors(self): + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Headers", "*") + self.send_header("Access-Control-Expose-Headers", "ETag") + + def do_OPTIONS(self): + self.send_response(204) + self.cors() + self.send_header("Content-Length", "0") + self.end_headers() + + def send(self, code, body, kind="application/json", headers=None): + self.send_response(code) + self.cors() + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def file(self, root: Path, rel: str, cache: str): + path = (root / rel).resolve() + if not root or not path.is_relative_to(root.resolve()) or not path.is_file(): + return self.send(404, b'{"error":"not found"}') + kind = "application/octet-stream" if path.suffix == ".gz" else \ + mimetypes.guess_type(path.name)[0] or "application/octet-stream" + if kind.startswith("text/") or kind in ("application/javascript", "application/json"): + kind += "; charset=utf-8" + self.send(200, path.read_bytes(), kind, {"Cache-Control": cache}) + + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + path = unquote(urlsplit(self.path).path) + if path in ("/", "/index.html"): + host = self.headers.get("Host", f"localhost:{server.args.port}") + self.send_response(302) + self.cors() + self.send_header("Location", f"/web/index.html?live=http://{host}") + self.send_header("Content-Length", "0") + return self.end_headers() + if path == "/live/state.json": + with server.lock: + body = server.state + return self.send(200, body, "application/json; charset=utf-8", {"Cache-Control": "no-store"}) + if path == "/live/episode.json": + with server.lock: + raw, packed, etag = server.episode + headers = {"Cache-Control": "no-store", "ETag": f'"{etag}"'} + if self.headers.get("If-None-Match") == f'"{etag}"': + self.send_response(304) + self.cors() + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("Content-Length", "0") + return self.end_headers() + if "gzip" in (self.headers.get("Accept-Encoding") or ""): + return self.send(200, packed, "application/json; charset=utf-8", + headers | {"Content-Encoding": "gzip", "Vary": "Accept-Encoding"}) + return self.send(200, raw, "application/json; charset=utf-8", headers) + if path.startswith("/live/"): + cache = "no-store" if path.endswith("index.json") else "public, max-age=86400, immutable" + return self.file(server.live_dir, path[len("/live/"):], cache) + if path.startswith("/radar/") and (server.args.live / path[1:]).is_file(): + return self.file(server.args.live, path[1:], "public, max-age=3600") + for prefix in ("stage", "radar", "twin"): + if path.startswith(f"/{prefix}/"): + return self.file(assets, path[1:], "public, max-age=3600") + if path.startswith("/web/"): + return self.file(web, path[len("/web/"):], "no-cache") + return self.send(404, b'{"error":"not found"}') + + httpd = ThreadingHTTPServer((self.args.host, self.args.port), Handler) + httpd.daemon_threads = True + print(f"{utc()} serving on http://{self.args.host}:{self.args.port}/web/index.html?live=http://" + f"{'localhost' if self.args.host in ('0.0.0.0', '127.0.0.1') else self.args.host}:{self.args.port}", + flush=True) + return httpd + + +def serve(args) -> int: + server = Server(args) + threading.Thread(target=server.bind().serve_forever, daemon=True).start() + server.loop() + return 0 + + +# ---------------------------------------------------------------------------------------------------------- feed + + +class Feed: + """A fake runner and env: each recorded day (a days/.json, the env's final data/get) played back as a run, + starting, its calls and advisories released a watch every `watch` seconds behind a data/get of its own, done once + the last is out (the full record), torn down after `linger`, then an intermission of `gap` before the next.""" + + def __init__(self, args): + self.args = args + self.days = [json.loads(Path(p).read_text()) for p in args.days] + self.files = [str(Path(p).resolve()) for p in args.days] + self.lock = threading.Lock() + self.instance: str | None = None + self.data: dict | None = None + self.history: list[dict] = [] + self.seq = 0 + self.cur: dict = {} + + def write(self, **cur) -> None: + self.seq += 1 + self.cur = {"pilot": self.args.pilot, "model": self.args.model, "mode": "joint", "policy": "fake", + "budget_usd": None, "spent_usd": round(sum(h.get("cost_usd") or 0 for h in self.history), 4), + "history": self.history, "final": False, "seq": self.seq, "updated_utc": utc(), **cur} + write_atomic(self.args.current, (json.dumps(self.cur, indent=1) + "\n").encode()) + + @staticmethod + def prefix(day: dict, s: int, watch: int) -> dict: + """The record as data/get had it when seq s was the last event: what has a seq cut by it, the rest by watch.""" + turns = [{**u, "ended": u["ended"] if u["ended"] is not None and u["ended"] <= s else None} + for u in day["turns"] if u["granted"] is not None and u["granted"] <= s] + last = max([c for c in day["calls"] if c["seq"] <= s], key=lambda c: c["seq"], default=None) + k = watch if last is None else last["watch"] + return {**day, "watch": k, "t": day["estimates"][k]["t"] if k < len(day["estimates"]) else day["t"], + "done": False, "turns": turns, "calls": [c for c in day["calls"] if c["seq"] <= s], + "refusals": [r for r in day["refusals"] if r["seq"] <= s], + "changes": [c for c in day["changes"] if c["seq"] <= s], + "notices": [n for n in day["notices"] if n["seq"] <= s], + "absent": {d: a for d, a in day["absent"].items() if a["seq"] <= s}, + "requests": [r for r in day["requests"] if r["watch"] < k], "pages": [p for p in day["pages"] + if p["watch"] <= k], + "flags": [f for f in day["flags"] if f["watch"] <= k], + "estimates": [e for e in day["estimates"] if e["watch"] < k], "plan": {}, "committed": [], + "acausal": [], "crews": None, "status": None, "audit": None, "grade": None, "reward": None, + "digest": None, "passengers": None} + + def timeline(self, day: dict) -> list[tuple[float, int]]: + """(seconds after the start, seq) for each event: a watch's spread evenly over its `watch` seconds.""" + events = sorted([(c["seq"], c["watch"]) for c in day["calls"]] + [(n["seq"], n["watch"]) for n in + day["notices"]]) + by: dict[int, list[int]] = collections.defaultdict(list) + for seq, k in events: + by[k].append(seq) + out = [] + for i, k in enumerate(sorted(by)): + for j, seq in enumerate(by[k]): + out.append(((i + (j + 1) / (len(by[k]) + 1)) * self.args.watch_seconds, seq)) + return out + + def play(self) -> None: + a = self.args + for n, (day, file) in enumerate(zip(self.days, self.files, strict=True)): + task_id, mode = day["task_id"], day["mode"] + date = f"{task_id[3:7]}-{task_id[7:9]}-{task_id[9:11]}" + nxt = None if n + 1 >= len(self.days) else {"date": None, "task_id": self.days[n + 1]["task_id"], + "starts_utc": None} + attempt, restart = 1, a.restart_at if n == 0 else None + while True: + run_id = f"{task_id}-{mode}-a{attempt}" + self.instance = f"@local/fake/{task_id}-{mode}-{random.randrange(16 ** 8):08x}" + base = f"http://127.0.0.1:{a.port}/{self.instance.rsplit('/', 1)[1]}" + started = utc() + common = {"instance_id": self.instance, "run_id": run_id, "date": date, "task_id": task_id, + "mode": mode, "started_utc": started, "next": nxt, "attempt": attempt, "day": n + 1} + with self.lock: + self.data = {"task_id": None, "mode": None, "done": False, "calls": [], "turns": [], + "requests": [], "audit": None, "grade": None, "reward": None} + self.write(status="starting", env=None, day_file=None, result=None, **common) + time.sleep(a.start_delay) + self.write(status="playing", env={"base": base, "mcp_url": base + "/mcp"}, day_file=None, result=None, + **common) + t0, broke = time.monotonic(), False + for at, seq in self.timeline(day): + time.sleep(max(0.0, t0 + at - time.monotonic())) + k = next(c["watch"] for c in day["calls"] + day["notices"] if c["seq"] == seq) + if restart is not None and k >= restart: + broke = True + break + with self.lock: + self.data = self.prefix(day, seq, k) + if broke: + print(f"{utc()} {run_id}: the instance dies at watch {restart}", flush=True) + with self.lock: + self.instance = None + attempt, restart = attempt + 1, None + continue + with self.lock: + self.data = day + time.sleep(a.linger) + with self.lock: + self.instance = None + result = {"date": date, "task_id": task_id, "run_id": run_id, "instance_id": common["instance_id"], + "outcome": "scored", "reward": day["grade"]["reward"], "reward_L": day["grade"]["reward_L"], + "cost_usd": a.cost_usd, "wall_seconds": round(time.monotonic() - t0, 1), + "attempts": attempt} + self.history.append(result) + self.write(status="done", env=None, day_file=file, result=result, **common) + print(f"{utc()} {run_id}: done, reward {day['grade']['reward']}", flush=True) + time.sleep(a.gap / 2) + self.write(status="intermission", env=None, day_file=file, result=result, **common) + time.sleep(a.gap / 2) + break + self.write(**{k: v for k, v in self.cur.items() if k not in ("seq", "updated_utc", "final", "history")} + | {"final": True, "reason": "dates", "next": None}) + + def serve(self) -> None: + feed = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + with feed.lock: + instance, data = feed.instance, feed.data + parts = self.path.strip("/").split("/") + if instance is None or parts[0] != instance.rsplit("/", 1)[1] or "/".join(parts[1:]) != RPC[1:]: + self.send_response(404) + self.send_header("Content-Length", "0") + return self.end_headers() + if random.random() < feed.args.hiccups: + if random.random() < 0.5: + time.sleep(feed.args.hang) + self.send_response(503) + self.send_header("Content-Length", "0") + return self.end_headers() + rid = json.loads(body or b"{}").get("id") + out = json.dumps({"jsonrpc": "2.0", "id": rid, "result": {"parts": [{"kind": "data", "data": data}]}} + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + httpd = ThreadingHTTPServer(("127.0.0.1", self.args.port), Handler) + httpd.daemon_threads = True + httpd.serve_forever() + + +def feed(args) -> int: + f = Feed(args) + threading.Thread(target=f.serve, daemon=True).start() + f.play() + time.sleep(args.linger) + return 0 + + +# ---------------------------------------------------------------------------------------------------------- attach + + +def attach(args) -> int: + """current.json for the newest run of a bundle task, rewritten until it ends: the instance, the env's base once + deployed, done with the graded day.""" + from agent_env.store.ids import LOCAL_PREFIX + from agent_env.store.routing import namespace_routing + from agent_env.task.store import task_instances + + root = args.bundle.expanduser().resolve() + home = Path.home() + task = LOCAL_PREFIX + "/".join(["~", *root.relative_to(home).parts] if root.is_relative_to(home) else + root.parts[1:]) + f"/{args.task}" + task_id, mode = args.task.rsplit("-", 1) + date = f"{task_id[3:7]}-{task_id[7:9]}-{task_id[9:11]}" + seen, seq = None, 0 + while True: + with namespace_routing(): + found = task_instances(task, limit=1) + if found and (args.after is None or (found[0].created_at_utc or "") >= args.after): + x = found[0] + ctx = x.context or {} + env = next((e for e in ctx.get("deployed_envs") or [] if e.get("env_id") == "airops-occ3"), None) + v = ((ctx.get("metadata") or {}).get("verifications") or {}).get("airops-occ3") or {} + day = (v.get("results") or [{}])[0].get("episode") + file = None + if day: + file = args.current.with_name(f"{args.task}-{x.instance_id.rsplit('-', 1)[1]}.json") + write_atomic(file, json.dumps(day).encode()) + status = "done" if day or x.status in ("completed", "failed", "cancelled") else \ + "playing" if env else "starting" + cur = {"instance_id": x.instance_id, "run_id": f"{args.task}-a1", "date": date, "task_id": task_id, + "model": args.model, "mode": mode, "started_utc": x.created_at_utc, "status": status, "next": None, + "history": [], "attempt": 1, "day": 1, "pilot": args.pilot, "policy": "script", + "env": {"base": env["mcp_url"].removesuffix("/mcp"), "mcp_url": env["mcp_url"]} if env else None, + "day_file": str(file) if file else None, "result": None} + if cur != seen: + seq += 1 + write_atomic(args.current, (json.dumps(cur | {"seq": seq, "updated_utc": utc()}, indent=1) + "\n" + ).encode()) + print(f"{utc()} {x.instance_id}: {status}", flush=True) + seen = cur + if status == "done": + return 0 + time.sleep(2) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = p.add_subparsers(dest="command", required=True) + b = sub.add_parser("prebuild", help="Each day's news and hub traffic.") + b.add_argument("tasks", nargs="+", metavar="TASK_ID") + b.add_argument("--live", type=Path, required=True) + b.add_argument("--inputs", type=Path, default=ROOT / "cache" / "inputs") + b.add_argument("--assets", type=Path, help="A viewer folder whose radar frames count as there.") + b.add_argument("--no-radar", dest="radar", action="store_false", help="Don't bake missing radar frames.") + b.add_argument("--pack", type=Path) + s = sub.add_parser("serve", help="Follow the runner's day and serve the viewer.") + s.add_argument("--current", type=Path, help="The runner's current.json.") + s.add_argument("--env", help="Follow this env's data/get instead (with --task).") + s.add_argument("--task", help="The task id the env plays, with --env.") + s.add_argument("--live", type=Path, required=True, help="Prebuilt days; the episodes are written under live/.") + s.add_argument("--assets", type=Path, required=True, help="stage/, radar/ and twin/ (a viewer folder).") + s.add_argument("--inputs", type=Path, help="Prebuild a day missing from --live from these raw inputs.") + s.add_argument("--web", type=Path, default=ROOT / "web") + s.add_argument("--pack", type=Path) + s.add_argument("--host", default="0.0.0.0") + s.add_argument("--port", type=int, default=8765) + s.add_argument("--poll", type=float, default=3.0) + f = sub.add_parser("feed", help="A fake runner and env playing recorded days back.") + f.add_argument("days", nargs="+", type=Path, metavar="DAY_JSON") + f.add_argument("--current", type=Path, required=True) + f.add_argument("--port", type=int, default=8766) + f.add_argument("--watch-seconds", type=float, default=75.0) + f.add_argument("--start-delay", type=float, default=5.0) + f.add_argument("--linger", type=float, default=10.0, help="Seconds the env stays up once done.") + f.add_argument("--gap", type=float, default=30.0, help="Seconds between days.") + f.add_argument("--hiccups", type=float, default=0.0, help="The share of data/get calls that fail.") + f.add_argument("--hang", type=float, default=8.0, help="Seconds a failing data/get hangs, half the time.") + f.add_argument("--restart-at", type=int, help="The first day's instance dies at this watch and starts again.") + f.add_argument("--pilot", default="occ3-live-fake") + f.add_argument("--model", default="openai/gpt-6-astra") + f.add_argument("--cost-usd", type=float) + a = sub.add_parser("attach", help="current.json for a local agent-env run of a bundle task.") + a.add_argument("bundle", type=Path) + a.add_argument("task", help="e.g. b6-20260715-0400-joint") + a.add_argument("--current", type=Path, required=True) + a.add_argument("--after", help="Only a run created at or after this (created_at_utc, 'YYYY-MM-DD HH:MM UTC').") + a.add_argument("--model", default="script") + a.add_argument("--pilot", default="occ3-live-local") + args = p.parse_args(argv) + if getattr(args, "live", None): + args.live = args.live.expanduser().resolve() + if args.command == "serve" and args.env and not args.task: + p.error("--env needs --task") + return {"prebuild": prebuild, "serve": serve, "feed": feed, "attach": attach}[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/occ_qa.py b/scripts/occ_qa.py index c4cefd3..8889a93 100644 --- a/scripts/occ_qa.py +++ b/scripts/occ_qa.py @@ -3,10 +3,13 @@ viewer's (its third-party code is vendored under web/vendor/), check that the 3D airport, the control-center room and the US map each come to the front and draw (window.airops.state() and a still that isn't blank) at a few minutes of the day, with a day's passengers at the airport, inside its terminal and its customer desk in the room when it has -them, and no terminal without them, and write the stills, a contact sheet of them and a JSON report. +them, and no terminal without them, and write the stills, a contact sheet of them and a JSON report. --live also plays +the folder's first two passenger days as a live server would serve them (scripts/live_fake.py) in the viewer's live +mode (?live=…&stream): the clock never past what is decided and the day growing as more is, the director cutting, the +live panel up, then the day's end through its score card, highlights and next-storm card onto the next day. usage: .venv/bin/python scripts/occ_qa.py cache/occ/viewer --out cache/qa [--minutes 840,1075] [--no-room] \ - [--ignore REGEX] + [--ignore REGEX] [--live] """ import argparse @@ -17,6 +20,7 @@ import re import sys import threading +import time from contextlib import contextmanager from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -25,6 +29,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageStat sys.path.insert(0, str(Path(__file__).resolve().parent)) +import live_fake # noqa: E402 from record import clock, find_chrome, session # noqa: E402 ROOT = Path(__file__).resolve().parents[1] @@ -34,6 +39,8 @@ TILE = (480, 270) QUIET = re.compile(r"favicon\.ico") BLOCKED = re.compile(r"Access to \w+ at '([^']+)' from origin") +CHROME = re.compile(r"^Unchecked runtime\.lastError") +"""What headless Chrome itself logs to a page it reloads (an extension's message channel), not the page's error.""" LOCAL = (None, "127.0.0.1", "localhost") """The hosts the QA serves the viewer and its data from (None: data: and blob: URLs).""" @@ -53,6 +60,7 @@ def main(argv: list[str] | None = None) -> int: help="Serve the folder from another origin than the viewer, with CORS headers as huggingface.co " "sends them, as the Space reads the dataset: an image loaded without crossOrigin then " "taints its WebGL texture and fails the check.") + p.add_argument("--live", action="store_true", help="Also check the viewer's live mode on a fake live server.") args = p.parse_args(argv) width, height = (int(v) for v in args.size.lower().split("x")) args.out.mkdir(parents=True, exist_ok=True) @@ -67,6 +75,10 @@ def main(argv: list[str] | None = None) -> int: minutes = [float(m) for m in args.minutes.split(",")] if args.minutes else default_minutes(ep) report["episodes"].append(episode(base, data, chrome, width, height, entry, ep, minutes, not args.no_room, ignore, args.out, stills)) + if args.live: + days = [e["url"] for e in index["episodes"] if e.get("passengers")][:2] + report["episodes"].append(live(args.web, args.folder, days, chrome, ignore, args.out, stills, args.cross_origin) + if days else {"name": "live", "failures": ["no passenger day to play live"]}) contact(stills, args.out / "contact.jpg") (args.out / "qa.json").write_text(json.dumps(report, indent=1)) failed = [f"{r['name']}: {e}" for r in [report["index"], *report["episodes"]] for e in r["failures"]] @@ -168,6 +180,92 @@ def episode(base: str, data: str, chrome: str, width: int, height: int, entry: d return result +def live(web: Path, folder: Path, days: list[str], chrome: str, ignore, out: Path, stills, cross: bool) -> dict: + """The live mode on a fake live server playing ``days`` (a watch every 8 s): mid-day the clock stays within what is + decided while more is and the page grows, the director cuts and the live panel shows; then from the first day's + end its score card, highlights and next-storm card, and the page onto the second day.""" + result = {"name": "live", "failures": [], "notes": [], "checks": []} + ignore = [*ignore, CHROME] + fake = live_fake.Fake(folder.resolve(), days, 8, 4, 40, 30, 0) + servers = [ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(live_fake.Handler, fake=fake, web=web.resolve())) + for _ in range(2 if cross else 1)] + for server in servers: + threading.Thread(target=server.serve_forever, daemon=True).start() + feed, site = (f"http://127.0.0.1:{s.server_port}" for s in (servers[0], servers[-1])) + first = fake.length(0) + try: + fake.control({"seek": [str(4 + 8 * 7)]}) + with session(f"{site}/web/index.html?live={quote(feed)}&stream&sound=0", chrome, 1920, 1080, "auto", + enable=("Runtime", "Log", "Network")) as (page, ready, gl): + result["gl"] = gl + state = lambda: page.js("airops.live?.state() ?? null") # noqa: E731 + samples, f0 = [], None + for _ in range(36): + st = state() + if st: + samples.append(st) + f0 = f0 if f0 is not None else st["frontier"] + time.sleep(1) + if not samples: + result["failures"].append("no live page") + return result + over = [s for s in samples if s["t"] > s["frontier"] + 1e-6] + back = [b for a, b in zip(samples, samples[1:], strict=False) if b["t"] < a["t"] - 1e-6] + cuts = samples[-1]["director"]["shots"] + panel = page.js("[document.querySelector('.live-panel .lp-live')?.textContent, " + "document.querySelectorAll('.live-panel .lp-desks li').length]") + result["checks"].append({"phase": "day", "t": [samples[0]["t"], samples[-1]["t"]], + "frontier": [f0, samples[-1]["frontier"]], "cuts": len(cuts), + "cameras": sorted({c["camera"] for c in cuts}), "panel": panel, + "errors": samples[-1]["errors"]}) + path = out / "live-day.jpg" + capture(page, path) + stills.append(("live · mid-day", path)) + if over: + result["failures"].append(f"the clock past what is decided: {over[0]['t']:.1f} > {over[0]['frontier']}") + if back: + result["failures"].append("the clock went back mid-day") + if samples[-1]["frontier"] <= f0: + result["failures"].append(f"the day did not grow: decided through {f0} all along") + if len(cuts) < 3: + result["failures"].append(f"the director cut {len(cuts)} times in 36 s") + if panel[0] != "Live" or panel[1] < 3: + result["failures"].append(f"the live panel shows {panel}") + if samples[-1]["errors"]: + result["failures"].append(f"{samples[-1]['errors']} frames failed") + result["failures"] += errors(page, ignore, result["notes"]) + fake.control({"seek": [str(first[0] + 1)]}) + page.call("Page.reload") + seen, run, t0 = [], None, time.monotonic() + while time.monotonic() - t0 < 160: + st = page.js("window.airops?.live?.state() ?? null") + if st: + run = run or st["instance"] + if not seen or seen[-1] != st["phase"]: + seen.append(st["phase"]) + if st["phase"] in ("score", "highlights", "next"): + path = out / f"live-{st['phase']}.jpg" + time.sleep(1.5) + capture(page, path) + stills.append((f"live · {st['phase']}", path)) + if st["instance"] != run: + break + time.sleep(1) + result["checks"].append({"phase": "end", "phases": seen, "next run": st and st["instance"] != run}) + if seen[:3] != ["score", "highlights", "next"]: + result["failures"].append(f"the day's end went {' → '.join(seen)}") + if not st or st["instance"] == run: + result["failures"].append("the page did not move on to the next day") + result["failures"] += errors(page, ignore, result["notes"]) + except SystemExit as e: + result["failures"].append(f"did not load: {e}") + finally: + for server in servers: + server.shutdown() + print(f"live: {len(result['checks'])} checks, {len(result['failures'])} failures", flush=True) + return result + + def errors(page, ignore, notes: list | None = None) -> list[str]: """The page's uncaught exceptions and console and log errors, but those ``ignore`` lets pass, and any script or stylesheet it loaded from another host; another host's resource that failed to load or came without CORS headers diff --git a/src/agentenv_airops/export.py b/src/agentenv_airops/export.py index 04d8b39..7300ff9 100644 --- a/src/agentenv_airops/export.py +++ b/src/agentenv_airops/export.py @@ -183,13 +183,15 @@ def stage(task: Task, cache: Path, stem: str) -> dict | None: return {"hub": task.hub, "url": f"../stage/{task.hub}/stage.json.gz", "traffic": f"{stem}.stage.json.gz"} -def sidecar(ep: Mapping, cache: Path, stages: dict[str, dict] | None = None) -> bytes: - """The episode's traffic sidecar, gzipped: its run laid over the task's baked traffic at the hub's stage.""" +def sidecar(ep: Mapping, cache: Path, stages: dict[str, dict] | None = None, traffic: Mapping | None = None) -> bytes: + """The episode's traffic sidecar, gzipped: its run laid over the task's baked traffic (``traffic``, else read from + the cache) at the hub's stage.""" hub, task_id = ep["task"]["hub"], ep["task"]["task_id"] stages = {} if stages is None else stages if hub not in stages: stages[hub] = json.loads(gzip.decompress((cache / "stage" / hub / "stage.json.gz").read_bytes())) - traffic = json.loads(gzip.decompress((cache / "traffic" / f"{task_id}.json.gz").read_bytes())) + if traffic is None: + traffic = json.loads(gzip.decompress((cache / "traffic" / f"{task_id}.json.gz").read_bytes())) crew = {int(i): rows for i, rows in (ep.get("occ") or {}).get("crew", {}).items()} doc = hub_stage.sidecar(ep["task"], ep["executed"], traffic, stages[hub], crew) if ep.get("occ"): diff --git a/src/agentenv_airops/occ/episode.py b/src/agentenv_airops/occ/episode.py index f1464f9..af4f015 100644 --- a/src/agentenv_airops/occ/episode.py +++ b/src/agentenv_airops/occ/episode.py @@ -61,13 +61,14 @@ def episode(world: World, data: Mapping, *, model: str, run: str, inputs: Path, cache: Path = export.CACHE, hub: str | None = None, record: Mapping | None = None, - projected: Sequence[Sequence[Mapping]] | None = None) -> dict: + projected: Sequence[Sequence[Mapping]] | None = None, news: Sequence[Mapping] | None = None) -> dict: """``data`` is the env's data/get record of a finished day (days/-a.json); ``hub`` the airport shown in 3D, - the task's by default.""" + the task's by default; ``news``, the hub's advisories and METARs through the last watch when already read + (`day_news`), else they are read from ``inputs``.""" task, grade = world.task, data["grade"] hub = hub or task.hub times = programs.watches(task)[: data["watch"] + 1] - news = advisories(task, inputs, times[-1]) + metars(task, hub, inputs, times[-1]) + news = day_news(task, hub, inputs, times[-1]) if news is None else list(news) by_id = {n["id"]: n for n in news} plan = data["plan"] executed = [outcome(task, x, plan.get(str(x["id"]), {})) for x in grade["executed"]] @@ -126,7 +127,12 @@ def passengers(world: World, data: Mapping) -> dict: if json.loads(json.dumps(done.to_dict())) != data["passengers"]: raise ValueError(f"{data['task_id']} {data['mode']}: the plan played again doesn't give the recorded " "passengers") - held = held_minutes(world, data, plan, day, truth) + return ledger_rows(world, done, led, held_minutes(world, data, plan, day, truth)) + + +def ledger_rows(world: World, done: pax.Day, led: Mapping, held: Mapping[int, int]) -> dict: + """`passengers`' shape from the pass's day and ledger (pax.ledger) and the minutes each flight was held.""" + task = world.task booked, cin, cout = Counter(), Counter(), Counter() for g in world.passengers.groups: for k in g.legs: @@ -158,8 +164,9 @@ def passengers(world: World, data: Mapping) -> dict: key=lambda e: (e["t"], e["kind"], e["flight"], e["group"]))} -def held_minutes(world: World, data: Mapping, plan: dict, day, truth) -> dict[int, int]: - """Per flight held for the customer desk (its HOLD accepted), the minutes it left later than without those holds.""" +def held_minutes(world: World, data: Mapping, plan: dict, day, truth, play=None) -> dict[int, int]: + """Per flight held for the customer desk (its HOLD accepted), the minutes it left later than without those holds; + ``play`` runs a plan (the truth by default).""" asked = {r["id"] for r in data["requests"] if r["sender"] == CUSTOMER and r["type"] == "HOLD"} mine = [c for c in data["changes"] if c["action"] == "hold" and c["ref"] in asked] if not mine: @@ -168,7 +175,7 @@ def held_minutes(world: World, data: Mapping, plan: dict, day, truth) -> dict[in for c in mine: e = without[str(c["target"])] e["holds"] = [h for h in e["holds"] if not (h[0] == c["t"] and h[1] == parse_hhmm(c["detail"]))] - other = runs.truth(world, plan_of(without), truth).run.flights + other = (play or (lambda p: runs.truth(world, p, truth)))(plan_of(without)).run.flights out = {} for c in mine: i = c["target"] @@ -389,6 +396,11 @@ def crew_tags(status: Mapping[str, Sequence[Mapping]], executed: Mapping[int, Ma return out +def day_news(task, hub: str, inputs: Path, last: int) -> list[dict]: + """The FAA's advisories and ``hub``'s METARs through minute ``last``, as an episode's news reads them.""" + return advisories(task, inputs, last) + metars(task, hub, inputs, last) + + def advisories(task, inputs: Path, last: int) -> list[dict]: """The FAA's advisories for both hubs signed from LOOKBACK before local midnight to the last watch, as v1's news.""" out = [] diff --git a/src/agentenv_airops/occ/onair.py b/src/agentenv_airops/occ/onair.py new file mode 100644 index 0000000..71ecb0e --- /dev/null +++ b/src/agentenv_airops/occ/onair.py @@ -0,0 +1,403 @@ +"""A day of the ops control center followed as it is played, for a live stream: the env's calls so far (its data/get +record) played again in order through a local day on the same pack, each result checked against the env's digest, +and the viewer's Episode JSON of the day so far. Nothing here reaches the env: it reads data/get and plays its own +copy. + +While the day runs the episode is the recorded episode's shape (episode.episode) on the day so far: the room as it +stands, the closed watches as they were left, the flights, crews and passengers on the projection at the open watch +(the truth before its minute, the day as known after), the projected cost, and no grade. Once the day is done it is +the recorded episode itself, built from the replay and its watches' projections. `live` says where the day stands. +""" + +import copy +import dataclasses +import gzip +import io +import json +import pickle +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import SimpleNamespace + +from .. import export +from .. import stage as hub_stage +from ..core.model import hhmm +from ..core.sim import FLOWN +from ..programs import crewplan, crewsim, pax +from ..programs import occ as programs +from . import episode, run +from .day import OccDay, costs, digest +from .day3 import OccDay3 +from .pack import World +from .replay import ROLE + +FEED = 60 +ATOMS = (str, int, float, bool, type(None), bytes) +LOOKUPS = {"wait_turn": "reads its page", "ops_flights": "looks up flights", "crew_flights": "looks up flights", + "cust_flights": "looks up flights", "cust_flight": "looks up the passengers of", + "ops_program": "reads the FAA programs", "crew_program": "reads the FAA programs", + "ops_aircraft": "looks up aircraft", "crew_crews": "looks up crews", "crew_reserves": "looks up reserves", + "crew_options": "prices crew options for", "ops_check": "tries changes", "crew_check": "tries crew moves"} + + +class Diverged(ValueError): + """The record isn't a continuation of what was played: another day, or one started again.""" + + +def rows(day: OccDay, proj: run.Projection) -> list[dict]: + """A projection's flights as episode.projections draws a watch.""" + return [{"id": i, "status": r.status, "tail": r.tail, "dep": r.dep, "arr": r.arr, + "arr_delay": r.arr - day.task.flight(i).sched_arr if r.status == FLOWN else None, "held": False, + "decided": r.decided} for i, r in sorted(proj.day.run.flights.items())] + + +def jsonable(obj): + return json.loads(json.dumps(obj, default=str)) + + +class Played(OccDay3): + """The day played again, keeping each watch's projection as the watch left it (`left`).""" + + def __init__(self, world: World, mode: str = "joint"): + self.left: list[list[dict]] = [] + super().__init__(world, mode) + + def _close(self) -> None: + super()._close() + self.left.append(rows(self, self.project(self.plan))) + + +class Follower: + """A day played again from the env's record as it grows: `follow` takes each newer data/get and plays the calls + and the triggers' advisories past those already played, in the env's order, checking each result and its place.""" + + def __init__(self, world: World, mode: str = "joint", make: type[OccDay] = Played): + self.world, self.day = world, make(world, mode) + self.seq = self.played = 0 + self.mismatches: list[dict] = [] + self.checked: str | None = None + + def follow(self, data: Mapping) -> list[dict]: + """The record's new calls played (each with its `out`) and advisories delivered, in seq order; the day ended + as end-day ends it once the record is done.""" + day = self.day + if data.get("task_id") != self.world.task.task_id or data.get("mode") != day.mode: + raise Diverged(f"the record is {data.get('task_id')} {data.get('mode')}, not " + f"{self.world.task.task_id} {day.mode}") + theirs = sum(c["seq"] <= self.seq for c in data["calls"]) + if theirs != self.played: + raise Diverged(f"the record has {theirs} calls up to seq {self.seq}, {self.played} were played") + events = sorted([(c["seq"], 0, c) for c in data["calls"] if c["seq"] > self.seq] + + [(n["seq"], 1, n) for n in data["notices"] if n["seq"] > self.seq + and n["via"] in ("trigger", "late")], key=lambda e: e[:2]) + out = [] + for seq, kind, x in events: + if kind: + day.notice(x["news_id"], x["via"]) + out.append({**x, "tool": "notice"}) + else: + result = day.call(ROLE[x["desk"]], x["tool"], x["args"]) + logged = day.calls[-1]["seq"] if day.calls else None + if digest(result) != x["result"] or logged != seq: + self.mismatches.append({"seq": seq, "desk": x["desk"], "tool": x["tool"], "watch": x["watch"], + "logged": logged}) + out.append({**x, "out": result}) + self.played += 1 + self.seq = seq + if data.get("done") and not day.done: + day.end() + if data.get("digest") and day.done and data["digest"] != self.checked: + self.checked = data["digest"] + if day.digest() != data["digest"]: + self.mismatches.append({"digest": day.digest(), "recorded": data["digest"]}) + return out + + +def pickled(obj, world: World) -> bytes: + """A replay's state on ``world`` pickled with the world's objects named by their place in it, not copied (the + world holds closures, and is the pack's): `unpickled` puts it on the same pack's world in another process.""" + out = io.BytesIO() + _Pickler(out, world).dump(obj) + return out.getvalue() + + +def unpickled(body: bytes, world: World): + return _Unpickler(io.BytesIO(body), world).load() + + +class _Pickler(pickle.Pickler): + def __init__(self, file, world: World): + super().__init__(file, pickle.HIGHEST_PROTOCOL) + self.ids = {id(x): i for i, x in enumerate(anchors(world))} + + def persistent_id(self, obj): + return self.ids.get(id(obj)) + + +class _Unpickler(pickle.Unpickler): + def __init__(self, file, world: World): + super().__init__(file) + self.objs = anchors(world) + + def persistent_load(self, pid): + return self.objs[pid] + + +def anchors(world: World) -> list: + """The world's objects in a fixed order, depth first through its dataclasses' fields, tuples, lists and dicts: the + same objects in the same places in any process that loads the same pack.""" + out, seen, stack = [], set(), [world] + while stack: + x = stack.pop() + if isinstance(x, ATOMS) or id(x) in seen: + continue + seen.add(id(x)) + out.append(x) + if dataclasses.is_dataclass(x) and not isinstance(x, type): + stack.extend(getattr(x, f.name) for f in reversed(dataclasses.fields(x))) + elif isinstance(x, list | tuple): + stack.extend(reversed(x)) + elif isinstance(x, dict): + stack.extend(v for kv in reversed(x.items()) for v in reversed(kv)) + return out + + +def traffic_at(doc: Mapping, task, hub: str, news: Sequence[Mapping]) -> dict: + """The task's baked traffic (stage.traffic, on any news) with its METARs and ground stops as ``news`` gives them, + as stage.traffic would bake it on that news.""" + view = episode.view_task(task, hub, news) + return {**doc, "weather": [hub_stage.metar(n.at, n.text) for n in sorted(view.news, key=lambda n: n.at) + if n.kind == "metar"], "stops": hub_stage.stops(view)} + + +class Statics: + """What a day's episodes share that no play changes: the hub's news through its last watch and its baked traffic + (`prebuild`, under ``root``), the hub's stage, the radar and twin (the viewer's ``assets``); each None or empty + where there is none, and then no sidecar.""" + + def __init__(self, world: World, root: Path, assets: Path): + task = world.task + self.world, self.hub, self.assets = world, task.hub, assets + self.news = read(root / "news" / f"{task.task_id}.json") or [] + self.traffic = read(root / "traffic" / f"{task.task_id}.json.gz") + self.stage = read(assets / "stage" / self.hub / "stage.json.gz") + self.radar = export.radar(task, root if (root / "radar" / task.task_id / "index.json").is_file() else assets) + self.twin = export.twin(SimpleNamespace(hub=self.hub), assets) + + def through(self, t: int) -> list[dict]: + return [n for n in self.news if n["at"] <= t] + + def sidecar(self, ep: Mapping, t: int) -> bytes | None: + if self.traffic is None or self.stage is None: + return None + traffic = traffic_at(self.traffic, self.world.task, self.hub, self.through(t)) + return export.sidecar(ep, self.assets, {self.hub: self.stage}, traffic) + + +def read(path: Path): + if not path.is_file(): + return None + body = path.read_bytes() + return json.loads(gzip.decompress(body) if path.suffix == ".gz" else body) + + +def prebuild(world: World, root: Path, inputs: Path) -> dict: + """A day's news through its last watch and its traffic at the hub (stage.traffic on that news) into ``root``.""" + task = world.task + news = sorted(episode.day_news(task, task.hub, inputs, programs.watches(task)[-1]), + key=lambda n: (n["at"], n["id"])) + doc = hub_stage.traffic(episode.view_task(task, task.hub, news), inputs) + for path, body in ((root / "news" / f"{task.task_id}.json", json.dumps(news).encode()), + (root / "traffic" / f"{task.task_id}.json.gz", + gzip.compress(json.dumps(doc, separators=(",", ":")).encode(), mtime=0))): + path.parent.mkdir(parents=True, exist_ok=True) + part = path.with_suffix(".part") + part.write_bytes(body) + part.replace(path) + return {"news": len(news), "aircraft": len(doc["aircraft"])} + + +def stem(run_label: str, model: str, task_id: str) -> str: + return f"{export.slug(run_label)}.{export.slug(model)}.{task_id}" + + +def final(f: Follower, statics: Statics, *, model: str, run_label: str, record: Mapping | None = None) -> dict: + """The done day's episode as occ_viewer.py builds it from the env's record, from the replay.""" + day = f.day + data = jsonable(day.data()) + times = programs.watches(day.task)[: data["watch"] + 1] + ep = episode.episode(f.world, data, model=model, run=run_label, inputs=Path("/nonexistent"), + cache=statics.assets, record=record, projected=day.left, news=statics.through(times[-1])) + ep["radar"], ep["twin"] = statics.radar, statics.twin + ep["stage"] = None if statics.traffic is None or statics.stage is None else { + "hub": statics.hub, "url": f"../stage/{statics.hub}/stage.json.gz", + "traffic": f"{stem(run_label, model, day.task.task_id)}.stage.json.gz"} + return ep + + +def now(f: Follower, statics: Statics, *, model: str, run_label: str, record: Mapping | None = None, + view: dict | None = None) -> dict: + """The day so far as an episode: the room and the closed watches as they stand, the rest on `view` (`projection`, + the open watch's).""" + world, day = f.world, f.day + task = world.task + data = jsonable(day.data()) + view = view or projection(f) + times = programs.watches(task)[: day.k + 1] + news = statics.through(day.t) + by_id = {n["id"]: n for n in news} + estimates = {e["watch"]: e["total"] for e in data["estimates"]} + refs = world.refs + best, b0 = refs["best"], refs["B0"]["notice"] + c = view["costs"] + fake = {**data, "grade": {"best": best, "B0": b0}, "crews": view["units"], "status": view["status"]} + room = episode.room(world, fake, view["executed"]) + room["grade"] = None + ep = { + "version": 1, + "task": {"task_id": task.task_id, "carrier": task.carrier, "hub": task.hub, "date": task.date, "tz": task.tz, + "window": list(task.window), "start": task.start, "turn": dict(task.turn), "dmax": task.dmax, + "airports": {k: a.to_dict() for k, a in sorted(task.airports.items())}, + "tails": [{"tail": t.tail, "family": t.family, "type": t.type, "seats": t.seats, "pos": t.pos, + "ready": t.ready, "end_req": t.end_req, "actual_end": t.actual_end} for t in task.tails]}, + "model": model, "model_label": export.NAMES.get(model, model), "run": run_label, + "reward": None, + "cost": {"total": c["total"], "pax_delay": c["delay"], "cancel": c["cancel"] + c["notice"], **c}, + "reference": {"anchor": best, "best": best, "bound": refs["bound"], "B0": b0, "flat": view["flat"], + "gap_pct": refs["gap_pct"]}, + "schedule": [episode.schedule_row(x) for x in task.flights], + "executed": view["executed"], + "ferries": [], + "frozen": [], + "watches": [{"t": t, "news": [by_id[n["news_id"]] for n in data["notices"] + if n["watch"] == k and n["news_id"] in by_id], + "changes": episode.changes(task, data["changes"], k), + **({"est_cost": estimates[k]} if k in estimates else {}), + "projected": day.left[k] if k < len(day.left) else view["projected"]} + for k, t in enumerate(times)], + "steps": [{"t": x["t"], "tool": x["tool"], "args": x["args"], "desk": x["desk"], "watch": x["watch"], + "turn": x["turn"]} for x in data["calls"]], + "news": sorted(news, key=lambda n: (n["at"], n["id"])), + "caps": {}, "nominal": {}, + "radar": statics.radar, "twin": statics.twin, + "stage": None, + "end_reason": None, "audit": None, "record": dict(record or {}), + "sources": episode.SOURCES, + "occ": room, + } + if world.passengers is not None: + ep["passengers"] = view["passengers"] + return ep + + +def projection(f: Follower) -> dict: + """What the open watch's projection shows: the flights (executed rows), its cost and flat cost, the crews' log and + gate tags, and the passengers on it, a flight's hold against the projection without the customer desk's holds.""" + world, day = f.world, f.day + task, t = world.task, day.t + proj = day.project(day.plan) + plan = jsonable(day.plan) + executed = [episode.outcome(task, x, plan.get(str(x["id"]), {})) for x in jsonable(proj.day.run.rows())] + out = {"watch": day.k, "version": day.version, "t": t, "executed": executed, "costs": costs(proj.day), + "flat": round(crewplan.worth(proj.day, True), 1), "units": jsonable(proj.day.units), + "status": jsonable(crewsim.statuses(proj.day)), "projected": rows(day, proj), "passengers": None} + if world.passengers is not None: + units = {u.id: u for u in world.units} + seated = {i: sum(units[u].size for u in us) for i, us in proj.day.riders.items() if us} + done, led = pax.ledger(task, world.passengers, proj.day.run.flights, seated) + data = {"requests": day.requests, "changes": day.changes} + held = episode.held_minutes(world, jsonable(data), plan, proj.day, None, + lambda p: run.project(world, p, t, day.world, day.known_at(t)).day) + out["passengers"] = episode.ledger_rows(world, done, led, held) + return out + + +def feed(f: Follower, new: Sequence[Mapping], news: Mapping[str, Mapping]) -> list[dict]: + """The room's events among newly played calls, one line each: requests, replies, decisions, notes, advisories, + desks leaving.""" + task, day = f.world.task, f.day + out = [] + for x in new: + tool, r = x["tool"], x.get("out") or {} + row = {"seq": x["seq"], "watch": x["watch"], "t": x["t"], "desk": x.get("desk"), "request": None, + "flight": None} + if tool == "notice": + n = news.get(x["news_id"]) + out.append(row | {"kind": "notice", "desk": None, + "text": (n or {}).get("text", x["news_id"]).splitlines()[0][:200]}) + elif tool.endswith("_request") and "request" in r: + q = next(q for q in day.requests if q["id"] == r["request"]) + i = q["params"].get("flight") + what = " ".join(p for p in (q["type"], task.flight(i).flight if i is not None else None, + hhmm(q["params"]["time"]) if q["params"].get("time") else None) if p) + text = episode.prose(q["text"], task) + out.append(row | {"kind": "request", "request": q["id"], "flight": i, + "text": f"{q['sender']} to {q['receiver']}: {what}" + (f": {text}" if text else "")}) + elif tool.endswith("_reply") and "answer" in r and "error" not in r: + q = next(q for q in day.requests if q["id"] == r["request"]) + i = q["params"].get("flight") + text = r["lapsed"] if r.get("lapsed") else episode.prose(x["args"].get("text"), task) + out.append(row | {"kind": "reply", "request": q["id"], "flight": i, + "text": f"{x['desk']} {'finds lapsed' if r.get('lapsed') else r['answer'] + 's'} " + f"{q['id']} ({q['type']}{' ' + task.flight(i).flight if i is not None else ''})" + + (f": {text}" if text else "")}) + elif tool.endswith("_confirm") and r.get("confirmed"): + delta = (r.get("delta") or {}).get("total") + out.append(row | {"kind": "decision", "text": f"{x['desk']} confirms " + f"{episode.prose(', '.join(r['confirmed']), task)}" + + (f" ({delta:+,.0f})" if delta else "")}) + elif tool == "end_turn" and x["args"].get("note"): + out.append(row | {"kind": "note", "text": f"{x['desk']}: {episode.prose(x['args']['note'], task)}"}) + elif tool == "resign": + out.append(row | {"kind": "off", "text": f"{x['desk']} desk off ({x['args'].get('reason')})"}) + return out + + +def summary(x: Mapping, task) -> str: + """A call in a few words, for a desk's last action.""" + tool, args, r = x["tool"], x.get("args") or {}, x.get("out") or {} + if tool in LOOKUPS: + flight = args.get("flight") or (args.get("ids") or [None])[0] + return f"{LOOKUPS[tool]}{' ' + episode.prose(str(flight), task) if flight else ''}" + if tool.endswith("_confirm"): + return episode.prose(f"confirms {', '.join(r.get('confirmed') or []) or 'nothing'}", task) + if tool.endswith("_request"): + return episode.prose(f"asks {args.get('to') or ('ops' if not tool.startswith('ops') else 'crew')} " + f"{args.get('type')}{' ' + args['flight'] if args.get('flight') else ''}", task) + if tool.endswith("_reply"): + return f"{args.get('answer')}s {args.get('request')}" + if tool == "end_turn": + return "stands down" if args.get("stand_down") else "ends its turn" + if tool == "resign": + return f"leaves ({args.get('reason')})" + return tool + + +def desks(f: Follower, last: Mapping[str, dict]) -> list[dict]: + day = f.day + out = [] + for d in day.desks: + state = "off" if d in day.absent else "turn" if day.turn and day.turn["desk"] == d else "waiting" + out.append({"role": d, "state": state, "off": (day.absent.get(d) or {}).get("reason"), + "last": (last.get(d) or {}).get("last"), "at": (last.get(d) or {}).get("at")}) + return out + + +def decided_until(day: OccDay, executed: Sequence[Mapping]) -> int: + """The minute before which the day is final: the open watch's while it runs; once done, its last movement.""" + if not day.done: + return day.t + return max([day.t] + [x[k] for x in executed for k in ("dep", "arr") if x.get(k) is not None]) + + +def cut(ep: Mapping, n: int = 1) -> dict: + """The episode with each long list cut to its first n rows, for showing its shape.""" + def short(v): + if isinstance(v, list): + return [short(x) for x in v[:n]] + if isinstance(v, dict): + return {k: short(x) for k, x in (list(v.items())[:max(n, 12)] if len(v) > 40 else v.items())} + if isinstance(v, str) and len(v) > 160: + return v[:160] + "…" + return v + return short(copy.deepcopy(dict(ep))) diff --git a/tests/occ/test_occ_tasks.py b/tests/occ/test_occ_tasks.py index 14682d9..34bfbff 100644 --- a/tests/occ/test_occ_tasks.py +++ b/tests/occ/test_occ_tasks.py @@ -1,20 +1,27 @@ """The ops control center as a bundle: each day's steps (the desks' roles and tools, one trigger per watch with advisories on the end_turn that opens it, the agents, the clock, the plays at once, end-day after both, the grade), the -rules each desk reads, the pack the env loads, and the verifier on a day's data/get.""" +rules each desk reads, the pack the env loads, and the verifier on a day's data/get; a pilot's margin, and the live +runner's order of the days, its spend guard and its pacing.""" +import importlib.util import json import types +from dataclasses import replace from importlib.resources import files +from pathlib import Path import pytest from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score from occ_helpers import play, scripted, storm, write_pack -from agentenv_airops.occ.pack import Pack +from agentenv_airops.occ.pack import Pack, Pack3 from agentenv_airops.occ.sweep import Pilot, verdict from agentenv_airops.occ.tasks import RULES, generate, steps pytestmark = pytest.mark.anyio +SPEC = importlib.util.spec_from_file_location("live_runner", Path(__file__).parents[2] / "scripts" / "live_runner.py") +live = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(live) def test_a_joint_day_deploys_two_desks_in_their_roles_and_triggers_each_watch_with_news(): @@ -99,3 +106,160 @@ async def get_data(url): def test_a_pilot_keeps_a_worst_request_free_per_desk_of_a_run(): p = Pilot("p", "m", ["b6-x-joint", "b6-x-one"], {"joint": 3.0, "one": 5.0}, 20.0, margin=1.5) assert (p.cap("b6-x-joint"), p.cap("b6-x-one")) == (9.0, 6.5) + + +def test_the_live_runner_plays_the_stormiest_training_days_first_a_storms_next_day_later_and_the_seasons_mixed( + tmp_path): + task = {"programs": {"JFK": {"segments": [{"since": 0, "until": 120}, {"since": 60, "until": 180}], + "stops": [{"start": 30, "end": 90}]}}, + "flights": [{"actual": {"cancelled": i < 20}} for i in range(30)]} + s = live.storm("b6-x", {"date": "2024-07-01", "split": "train"}, task) + assert (s["hubs"], s["cancelled"], s["season"], s["score"]) == ( + {"JFK": {"gdp_h": 3.0, "gs": 1, "gs_h": 1.0}, "BOS": {"gdp_h": 0, "gs": 0, "gs_h": 0}}, 20, "summer", 7.0) + + def day(when, score, split="train"): + return {"task_id": f"b6-{when}", "date": when, "split": split, "score": score, + "season": "winter" if int(when[5:7]) in live.WINTER else "summer"} + days = [day("2024-07-01", 90), day("2024-07-02", 80), day("2024-08-10", 70), day("2024-06-01", 60), + day("2023-07-04", 50), day("2024-01-10", 20), day("2025-02-01", 10), day("2023-07-09", 99, "eval")] + assert [d["date"] for d in live.order(days)] == ["2024-07-01", "2024-01-10", "2024-08-10", "2024-06-01", + "2023-07-04", "2025-02-01", "2024-07-02", "2023-07-09"] + shipped = live.Settings.read(live.CONFIG) + assert list(shipped.days) == [d["task_id"] for d in live.order(live.storms(Pack3()))] + assert (shipped.model, shipped.policy, shipped.cap_joint_usd, shipped.request_margin_usd, shipped.budget_usd) == ( + "openai/gpt-6-astra", None, 23.5, 4.0, 430.0) + for bad in ('days = ["b6-20240703-0400"]', 'days = ["b6-20240803-0400", "b6-20240803-0400"]', "budget = 1"): + (tmp_path / "bad.toml").write_text(bad + "\n") + with pytest.raises(ValueError): + live.Settings.read(tmp_path / "bad.toml") + + +def test_the_live_runner_starts_a_day_only_while_the_spend_and_a_worst_day_fit_and_paces_the_days_on_slots( + monkeypatch): + logs = {"agent-local-a": "INFO request 7: 2 tool calls, stop tool_calls, $1.2000 spent\n" + "INFO request 8: 0 tool calls, stop stop, $1.5000 spent\n", "agent-local-b": "starting\n"} + monkeypatch.setattr(live.subprocess, "run", lambda args, **_: types.SimpleNamespace(stdout=logs[args[-1]], + stderr="")) + ctx = {"deployed_agents": [{"role": "crew", "sandbox_id": "local-a", "sandbox_type": "local"}, + {"role": "ops", "sandbox_id": "local-b", "sandbox_type": "local"}]} + assert live.desk_spend(ctx) == {"crew": 1.5} + s = live.Settings(days=("b6-a", "b6-b", "b6-c"), budget_usd=100.0, worst_day_usd=40.0, typical_day_usd=25.0, + slot_minutes=75.0, min_intermission_minutes=5.0, window_hours=4.0) + desks = {"crew": {"cost_usd": 10.0}, "customer": {"cost_usd": None}, "ops": {"cost_usd": None}} + assert live.spend({"desks": desks, "live_usd": {"customer": 2.5}}, 4.0) == 16.5 + assert live.affordable(60.0, s) and not live.affordable(60.01, s) + assert live.over(60.0, 40.0, s) and not live.over(60.0, 39.99, s) + t0 = live.stamp("2026-10-23T16:00:00Z") + + def row(run, attempt, outcome, seconds, usd): + return {"run": f"{run}-joint", "attempt": attempt, "outcome": outcome, "started_utc": live.utc(t0), + "ended_utc": live.utc(t0 + seconds), "spend_usd": usd, "cost_usd": usd, "desks": {}, "instance": None, + "reward": 0.7 if outcome == "scored" else None, "reward_L": None, "wall_seconds": seconds, + "date": run} + assert live.next_start(s, None, t0, t0 - 50) == t0 + assert live.next_start(s, row("b6-a", 1, "scored", 3000, 30.0), t0, t0 + 3000) == t0 + 4500 + assert live.next_start(s, row("b6-a", 1, "scored", 4800, 30.0), t0, t0 + 4800) == t0 + 5100 + assert live.next_start(s, row("b6-a", 1, "failed", 120, 0.0), t0, t0 + 120) == t0 + 420 + rows = [row("b6-a", 1, "failed", 120, 1.0), row("b6-a", 2, "failed", 120, 2.0), row("b6-b", 1, "unscored", 60, 3.0)] + assert (live.settled(rows[:1]), live.settled(rows[:2]), live.settled(rows[2:])) == (None, "skipped", "unscored") + assert live.pending(s, rows) == ["b6-c"] and live.spent(rows) == 6.0 + assert [(h["task_id"], h["outcome"], h["cost_usd"], h["attempts"]) for h in live.history(rows)] == [ + ("b6-a", "skipped", 3.0, 2), ("b6-b", "unscored", 3.0, 1)] + plan, why = live.projection(s, [], t0, t0) + assert [(p["task_id"], p["starts"] - t0, p["spent_before"]) for p in plan] == [ + ("b6-a", 0, 0), ("b6-b", 4500, 25.0), ("b6-c", 9000, 50.0)] and why.startswith("the order") + plan, why = live.projection(replace(s, budget_usd=80.0), [], t0, t0) + assert len(plan) == 2 and why.startswith("the budget: $50.00 spent") + plan, why = live.projection(replace(s, window_hours=3.0), [], t0, t0) + assert len(plan) == 2 and why.startswith("the window") + + +FAKE_RUN = """ +import json, os, sys, time, uuid +from pathlib import Path +here = Path(os.environ["FAKE_AGENT_ENV"]) +task = sys.argv[sys.argv.index("--task") + 1] +calls = here / "calls.jsonl" +n = sum(json.loads(line)["task"] == task for line in calls.read_text().splitlines()) if calls.exists() else 0 +with calls.open("a") as f: + f.write(json.dumps({"task": task, "start": time.time()}) + "\\n") +step = json.loads((here / "plan.json").read_text())[task][n] +time.sleep(0.2) +if step.get("touch"): + Path(step["touch"]).touch() +instance = f"@local/fake/{task}-{uuid.uuid4().hex[:8]}" +(here / instance.replace("/", "_")).write_text(json.dumps(step["context"])) +print(f" tasks/{task}.json v1: done, 0.2s, instance {instance}") +""" + + +def _desks(*usd): + return [{"step_id": f"play-{role}-desk", "structured_output": {"cost_usd": c}} + for role, c in zip(("crew", "customer", "ops"), usd, strict=True)] + + +SCORED = {"metadata": {"verifications": {"airops-occ3": {"score": 0.8, "results": [{"episode": { + "absent": {}, "grade": {"reward_L": 0.4}}}]}}}, "prompt_responses": _desks(10.0, 3.0, 4.0)} +FAILED = {"metadata": {"failed_steps": [{"step_id": "play-ops-desk", "error": "provider_error"}]}, + "prompt_responses": _desks(1.0, 1.0, 0.5)} + + +@pytest.fixture +def live_runner(tmp_path, monkeypatch): + """The live runner in tmp_path, each attempt a stand-in `agent-env run` that plays plan.json's next step for its + task and leaves the step's context where the runner reads it.""" + fake = tmp_path / "fake" + fake.mkdir() + (fake / "run.py").write_text(FAKE_RUN) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("FAKE_AGENT_ENV", str(fake)) + monkeypatch.setattr(live.sweep, "AGENT_ENV", [live.sys.executable, str(fake / "run.py")]) + monkeypatch.setattr(live.sweep, "context", lambda i: json.loads((fake / i.replace("/", "_")).read_text())) + monkeypatch.setattr(live, "instances", lambda task: []) + monkeypatch.setattr(live, "POLL_SECONDS", 0.02) + monkeypatch.setattr(live, "IDLE_SECONDS", 0.02) + + def run(days, plan, **settings): + (fake / "plan.json").write_text(json.dumps({f"{d}-joint": steps for d, steps in zip(days, plan, strict=True)})) + body = {"name": "t", "policy": "script", "budget_usd": 70.0, "worst_day_usd": 40.0, "slot_minutes": 0.06, + "min_intermission_minutes": 0.03, "card_seconds": 0.0, "days": days} | settings + (tmp_path / "live.toml").write_text("".join(f"{k} = {json.dumps(v)}\n" for k, v in body.items())) + runner = live.Runner(tmp_path / "live.toml", tmp_path / "live" / "current.json", {}, None) + assert runner.run() == 0 + calls = [json.loads(line) for line in (fake / "calls.jsonl").read_text().splitlines()] + return runner, calls, json.loads((tmp_path / "live" / "current.json").read_text()) + return run + + +def test_the_live_runner_tries_a_failed_day_once_more_then_skips_it_holds_each_slot_and_stops_at_the_budget( + live_runner): + days = ["b6-20240803-0400", "b6-20230627-0400", "b6-20250119-0400", "b6-20240818-0400"] + runner, calls, current = live_runner(days, [[{"context": FAILED}, {"context": FAILED}], [{"context": SCORED}], + [{"context": SCORED}], [{"context": SCORED}]]) + assert [c["task"] for c in calls] == [f"{d}-joint" for d in days[:1] + days[:3]] + rows = live.sweep.results(runner.out) + assert [(r["run"], r["attempt"], r["outcome"], r["spend_usd"]) for r in rows] == [ + (f"{days[0]}-joint", 1, "failed", 2.5), (f"{days[0]}-joint", 2, "failed", 2.5), + (f"{days[1]}-joint", 1, "scored", 17.0), (f"{days[2]}-joint", 1, "scored", 17.0)] + assert calls[1]["start"] - calls[0]["start"] >= 0.8 and calls[3]["start"] - calls[2]["start"] >= 2.6 + assert (current["final"], current["reason"].split(":")[0], current["next"], current["spent_usd"]) == ( + True, "budget", None, 39.0) + assert [(h["date"], h["hub"], h["outcome"], h["attempts"], h["cost_usd"], h["reward"]) + for h in current["history"]] == [ + ("2024-08-03", "JFK", "skipped", 2, 5.0, None), ("2023-06-27", "BOS", "scored", 1, 17.0, 0.8), + ("2025-01-19", "JFK", "scored", 1, 17.0, 0.8)] + assert current["run_id"] == f"{days[2]}-joint-a1" and current["result"] == current["history"][-1] + assert current["hub"] == "JFK" and current["result"]["desks"] == rows[-1]["desks"] + assert {r: d["cost_usd"] for r, d in current["result"]["desks"].items()} == { + "crew": 10.0, "customer": 3.0, "ops": 4.0} + assert current["day_file"].endswith(f"days/{days[2]}-joint-a1.json") and Path(current["day_file"]).is_file() + pilot = json.loads((runner.out / "pilot.json").read_text()) + assert pilot["runs"] == [f"{d}-joint" for d in days[:3]] and pilot["policy"] == "script" + + +def test_the_live_runner_finishes_the_day_in_progress_when_asked_to_stop(live_runner, tmp_path): + days = ["b6-20240803-0400", "b6-20230627-0400"] + flag = tmp_path / "live" / "stop.flag" + runner, calls, current = live_runner(days, [[{"context": SCORED, "touch": str(flag)}], [{"context": SCORED}]]) + assert [c["task"] for c in calls] == [f"{days[0]}-joint"] and not flag.exists() + assert (current["final"], current["reason"], current["history"][0]["outcome"]) == (True, "stopped", "scored") diff --git a/tests/occ/test_onair.py b/tests/occ/test_onair.py new file mode 100644 index 0000000..c08ee54 --- /dev/null +++ b/tests/occ/test_onair.py @@ -0,0 +1,277 @@ +"""A day of airops-occ3 followed live from the env's record as it grows (agentenv_airops.occ.onair): played again to the +env's state at every point, also resumed from a checkpoint at each, the day so far as an episode with the room and the +projection, no grade and nothing of the truth after the open watch, the done day as the recorded episode; and the live +server following a fake runner's days through failing data/get calls and an instance that dies and starts again, and +going on from its checkpoint when it is itself restarted.""" + +import copy +import importlib.util +import json +import socket +import threading +import time +from argparse import Namespace +from pathlib import Path + +import httpx +import pytest +from jsonschema import Draft202012Validator, FormatChecker +from occ_helpers import ROLE, desk_agent, storm3 + +from agentenv_airops import export +from agentenv_airops.occ import episode, onair +from agentenv_airops.occ.day import costs +from agentenv_airops.occ.day3 import OccDay3 + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("live_server", ROOT / "scripts" / "live_server.py") +live_server = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(live_server) +END = {"note": "watch done", "stand_down": False} +MODEL = "openai/gpt-6-astra" + + +@pytest.fixture(scope="module") +def world(): + return storm3() + + +@pytest.fixture(scope="module") +def day(world): + """The scripts' day at three desks, and the env's record (data/get) after each call.""" + d = OccDay3(world, "joint") + scripts = {k: desk_agent.Script(k) for k in d.desks} + records = [] + + def call(desk: str, tool: str, args: dict) -> dict: + out = d.call(ROLE[desk], tool, args) + records.append(onair.jsonable(d.data())) + return out + while not d.done: + desk = d.turn["desk"] + for n in list(d.pending): + d.notice(n, "trigger") + page = call(desk, "wait_turn", {}) + + async def act(name: str, args: dict, desk: str = desk) -> dict: + return json.loads(json.dumps(call(desk, name, args))) + run(scripts[desk].turn(page["page"], act)) + call(desk, "end_turn", END) + for desk in d.desks: + call(desk, "resign", {"reason": "done"}) + return d, records + + +def run(coro) -> None: + try: + while True: + coro.send(None) + except StopIteration: + pass + + +def live_validator() -> Draft202012Validator: + schema = copy.deepcopy(export.SCHEMA) + schema["properties"]["reward"] = {"type": ["number", "null"], "minimum": 0, "maximum": 1} + schema["properties"]["occ"]["properties"]["grade"] = {"type": ["object", "null"]} + return Draft202012Validator(schema, format_checker=FormatChecker()) + + +def test_a_day_followed_as_its_record_grows_is_the_envs_day_at_every_point(world, day): + d, records = day + assert any(r["requests"] for r in records) and any(n["via"] == "trigger" for n in records[-1]["notices"]) + f, g, worlds = onair.Follower(world, "joint"), onair.Follower(world, "joint"), [world, storm3()] + for i in [*range(0, len(records) - 1, 3), len(records) - 1]: + new = f.follow(records[i]) + assert onair.jsonable(f.day.data()) == records[i] + assert all(x["seq"] > 0 for x in new) + g = onair.unpickled(onair.pickled(g, g.world), worlds[i % 2]) + assert g.world is worlds[i % 2] and g.day.task is g.world.task + assert [x["seq"] for x in g.follow(records[i])] == [x["seq"] for x in new] + assert onair.jsonable(g.day.data()) == records[i] + assert f.day.done and not f.mismatches and f.day.digest() == d.digest() + assert g.day.done and not g.mismatches and g.day.digest() == d.digest() and g.day.left == f.day.left + assert f.follow(records[-1]) == [] + assert len(f.day.left) == len(d.estimates) + with pytest.raises(onair.Diverged, match="calls up to seq"): + f.follow(records[5]) + with pytest.raises(onair.Diverged, match="not"): + onair.Follower(world, "one").follow(records[0]) + + +def test_the_day_so_far_is_an_episode_of_the_room_and_the_projection_with_nothing_after_its_watch(world, day, + tmp_path): + d, records = day + final = records[-1] + statics = onair.Statics(world, tmp_path, tmp_path) + f = onair.Follower(world, "joint") + mid = next(r for r in records if r["t"] >= 660 and r["turns"][-1]["desk"] == "ops") + f.follow(mid) + ep = onair.now(f, statics, model=MODEL, run_label="live-joint") + assert not list(live_validator().iter_errors(ep)) + assert (ep["reward"], ep["occ"]["grade"], ep["end_reason"], ep["audit"], ep["stage"]) == (None,) * 5 + assert len(ep["watches"]) == f.day.k + 1 == mid["watch"] + 1 + assert [w["projected"] for w in ep["watches"][:-1]] == episode.projections(world, final)[: f.day.k] + assert "est_cost" not in ep["watches"][-1] and ep["watches"][-1]["projected"] == onair.rows( + f.day, f.day.project(f.day.plan)) + assert [s["tool"] for s in ep["steps"]] == [c["tool"] for c in mid["calls"]] + assert [r["id"] for r in ep["occ"]["requests"]] == [r["id"] for r in mid["requests"]] + assert ep["cost"]["total"] == costs(f.day.project(f.day.plan).day)["total"] + t, truth = f.day.t, {x["id"]: x for x in final["grade"]["executed"]} + gone = [x for x in ep["executed"] if x["dep"] is not None and x["dep"] < t] + assert gone and all((x["dep"], x["tail"], x["status"]) == (truth[x["id"]]["dep"], truth[x["id"]]["tail"], + truth[x["id"]]["status"]) for x in gone) + rows = ep["passengers"]["flights"] + assert all(r["booked"] + r["added"] == r["flown"] + sum(x["pax"] for x in r["rebooked"]) + r["stranded"] + + r["home"] + r["dropped"] for r in rows) + assert onair.decided_until(f.day, ep["executed"]) == t + + f.follow(final) + done = onair.final(f, statics, model=MODEL, run_label="live-joint") + want = episode.episode(world, final, model=MODEL, run="live-joint", inputs=tmp_path, cache=tmp_path, + projected=episode.projections(world, final)) + assert done == want + export.validate(done) + + +def test_the_feed_and_the_room_say_what_each_desk_did(world, day): + d, records = day + f = onair.Follower(world, "joint") + new = f.follow(records[-1]) + rows = onair.feed(f, new, {}) + kinds = {r["kind"] for r in rows} + assert {"request", "reply", "note", "off"} <= kinds + request = next(r for r in rows if r["kind"] == "request") + assert request["request"] == "R1" and request["text"].startswith(f"{d.requests[0]['sender']} to ") + assert all("f" + str(r["flight"]) not in r["text"] for r in rows if r["flight"] is not None) + last = {x["desk"]: {"last": {"tool": x["tool"], "text": onair.summary(x, world.task)}, "at": "now"} + for x in new if x["tool"] != "notice"} + assert [(x["role"], x["state"], x["off"], x["last"]["tool"]) for x in onair.desks(f, last)] == [ + (k, "off", "done", "resign") for k in ("crew", "customer", "ops")] + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def test_the_server_follows_a_runners_days_through_failed_calls_and_a_restart(world, day, tmp_path, monkeypatch): + d, records = day + file = tmp_path / "day.json" + file.write_text(json.dumps(records[-1])) + monkeypatch.setattr(live_server, "Pack3", lambda path=None: Namespace(world=lambda task_id: world)) + current, feed_port, port = tmp_path / "current.json", free_port(), free_port() + feed = live_server.Feed(Namespace(days=[file], current=current, port=feed_port, watch_seconds=0.15, + start_delay=0.3, linger=2.5, gap=3.0, hiccups=0.2, hang=0.2, restart_at=2, + pilot="occ3-live-test", model=MODEL, cost_usd=1.5)) + server = live_server.Server(Namespace(current=current, env=None, task=None, live=tmp_path / "live", + assets=tmp_path, inputs=None, web=ROOT / "web", pack=None, + host="127.0.0.1", port=port, poll=0.2)) + for target in (feed.serve, feed.play, server.bind().serve_forever, server.loop): + threading.Thread(target=target, daemon=True).start() + base, seen, deadline = f"http://127.0.0.1:{port}", [], time.monotonic() + 120 + while time.monotonic() < deadline: + try: + live = httpx.get(f"{base}/live/state.json", timeout=5).json()["live"] + except httpx.HTTPError: + time.sleep(0.1) + continue + if live.get("run_id"): + seen.append((live["run_id"], live["status"], live["watch"], live["decided_until"], live["seq"])) + if live.get("run_id", "").endswith("a2") and live["status"] == "intermission": + break + time.sleep(0.1) + runs = list(dict.fromkeys(r for r, *_ in seen)) + assert [r.rsplit("-", 1)[1] for r in runs] == ["a1", "a2"] + assert {s for r, s, *_ in seen if r == runs[1]} >= {"playing", "done", "intermission"} + assert all(a[4] <= b[4] for a, b in zip(seen, seen[1:], strict=False)) + a2 = [x for x in seen if x[0] == runs[1]] + assert all(a[3] <= b[3] for a, b in zip(a2, a2[1:], strict=False)) + r = httpx.get(f"{base}/live/episode.json", headers={"Accept-Encoding": "gzip"}, timeout=10) + assert r.headers["access-control-allow-origin"] == "*" and r.headers["content-encoding"] == "gzip" + ep = r.json() + live = ep.pop("live") + assert (live["status"], live["exact"], live["mismatches"], live["result"]["reward"]) == ( + "intermission", True, 0, records[-1]["grade"]["reward"]) + want = episode.episode(world, records[-1], model=MODEL, run="occ3-live-test-joint", inputs=tmp_path, + cache=tmp_path, projected=episode.projections(world, records[-1])) + assert {k: v for k, v in ep.items() if k != "record"} == {k: v for k, v in want.items() if k != "record"} + assert ep["record"]["attempt"] == 2 and ep["record"]["cost_usd"] == 1.5 + assert httpx.get(f"{base}/live/episode.json", headers={"If-None-Match": r.headers["etag"]}).status_code in ( + 200, 304) + [entry] = httpx.get(f"{base}/live/index.json").json()["episodes"] + assert entry["run"] == "occ3-live-test-joint" and entry["reward"] == records[-1]["grade"]["reward"] + kept = httpx.get(f"{base}/live/{entry['url']}").json() + assert kept["live"]["status"] == "done" and kept["executed"] == ep["executed"] + assert httpx.get(f"{base}/web/index.html").status_code == 200 + assert httpx.get(f"{base}/stage/../../current.json").status_code == 404 + + +def test_a_server_started_after_the_day_reads_it_from_the_day_file_watch_by_watch(world, day, tmp_path, monkeypatch): + d, records = day + file = tmp_path / "day.json" + file.write_text(json.dumps(records[-1])) + monkeypatch.setattr(live_server, "Pack3", lambda path=None: Namespace(world=lambda task_id: world)) + current = tmp_path / "current.json" + current.write_text(json.dumps({"status": "intermission", "task_id": records[-1]["task_id"], "run_id": "x-joint-a1", + "mode": "joint", "model": MODEL, "pilot": "late", "day_file": str(file), + "instance_id": "i1", "next": {"task_id": "y"}, "result": {"cost_usd": 2.0}})) + server = live_server.Server(Namespace(current=current, env=None, task=None, live=tmp_path / "live", + assets=tmp_path, inputs=None, web=ROOT / "web", pack=None, + host="127.0.0.1", port=0, poll=0.0)) + seen = [] + while not (server.day and server.day.final): + server.step() + seen.append((server.status, server.day.follower.day.k)) + assert len(seen) > 2 and {s for s, _ in seen[:-1]} == {"ending"} and seen[-1][0] == "intermission" + assert [k for _, k in seen] == sorted(k for _, k in seen) + live = json.loads(server.state)["live"] + assert (live["status"], live["exact"], live["next"], live["result"]["reward"]) == ( + "intermission", True, {"task_id": "y"}, records[-1]["grade"]["reward"]) + ep = json.loads(server.episode[0]) + assert [{k: x[k] for k in ("id", "status", "dep", "arr", "tail")} for x in ep["executed"]] == [ + {k: x[k] for k in ("id", "status", "dep", "arr", "tail")} for x in d.grade["executed"]] + assert ep["record"]["cost_usd"] == 2.0 + + +def test_a_restarted_server_goes_on_from_its_checkpoint_to_the_same_episode(world, day, tmp_path, monkeypatch): + d, records = day + file = tmp_path / "day.json" + file.write_text(json.dumps(records[-1])) + monkeypatch.setattr(live_server, "Pack3", lambda path=None: Namespace(world=lambda task_id: storm3())) + current, ckpt = tmp_path / "current.json", tmp_path / "live" / "ckpt" + cur = {"status": "intermission", "task_id": records[-1]["task_id"], "run_id": "x-joint-a1", "attempt": 1, + "mode": "joint", "model": MODEL, "pilot": "late", "day_file": str(file), "instance_id": "i1"} + current.write_text(json.dumps(cur)) + ckpt.mkdir(parents=True) + (ckpt / "x-joint-a0.pkl").write_bytes(b"another day's") + args = Namespace(current=current, env=None, task=None, live=tmp_path / "live", assets=tmp_path, inputs=None, + web=ROOT / "web", pack=None, host="127.0.0.1", port=0, poll=0.0) + + def played(until: int = 99) -> tuple[live_server.Server, dict]: + server = live_server.Server(args) + server.step() + while not server.day.final and server.day.follower.day.k < until: + server.step() + ep = json.loads((tmp_path / "live" / "live" / "x-joint-a1.json").read_text()) if server.day.final else None + return server, ep + + first, _ = played(until=3) + k, n = first.day.follower.day.k, first.day.follower.played + assert [p.name for p in ckpt.iterdir()] == ["x-joint-a1.pkl"] and not first.day.resumed + assert k >= 3 and 0 < n < len(records[-1]["calls"]) + resumed, ep = played() + assert resumed.day.resumed and resumed.day.started == first.day.started + assert {w: at for w, at in resumed.day.opened.items() if w <= k} == first.day.opened + after, again = played(until=0) + assert after.day.resumed and after.day.final + (ckpt / "x-joint-a1.pkl").write_bytes(b"torn") + whole, want = played() + assert not whole.day.resumed and whole.day.follower.played == resumed.day.follower.played > n + assert all((x["live"]["exact"], x["live"]["mismatches"]) == (True, 0) for x in (ep, again, want)) + assert {k: v for k, v in ep.items() if k != "live"} == {k: v for k, v in again.items() if k != "live"} == { + k: v for k, v in want.items() if k != "live"} + assert not live_server.Day(cur | {"attempt": 2}, args.live, tmp_path, live_server.Pack3(), None).resumed + monkeypatch.setattr(live_server, "code", lambda: "another build") + assert not live_server.Day(cur, args.live, tmp_path, live_server.Pack3(), None).resumed diff --git a/tests/viewer/live.test.mjs b/tests/viewer/live.test.mjs new file mode 100644 index 0000000..f180a64 --- /dev/null +++ b/tests/viewer/live.test.mjs @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Pacer } from "../../web/live.js"; + +/** The pacer driven at 60 frames a second for `seconds`, `land(wall)` moving the frontier; every frame's clock. */ +function run(p, seconds, land = () => {}, from = 0) { + const out = []; + for (let k = 0; k < seconds * 60; k++) { + const now = from + k / 60; + land(p, now); + out.push({ now, ...p.tick(now, 1 / 60), F: p.F }); + } + return out; +} + +test("the clock never passes what is decided and eases to a hold there", () => { + const p = new Pacer(240, 300, 0); + const frames = run(p, 400); + assert.ok(frames.every((f) => f.t <= f.F + 1e-9)); + const end = frames.at(-1); + assert.ok(end.t > 299.5 && end.rate < 1, `held at ${end.t} at ${end.rate}`); + const rates = frames.map((f) => f.rate); + assert.ok(rates.slice(1).every((r, i) => Math.abs(r - rates[i]) < 3), "no jump in speed"); +}); + +test("each decided hour plays over about the time a watch takes", () => { + const p = new Pacer(240, 300, 0); + const frames = run(p, 1200, (q, now) => { + const k = Math.floor(now / 100); + if (300 + 60 * k > q.F) q.advance(300 + 60 * k, now); + }); + const late = frames.filter((f) => f.now > 600); + assert.ok(late.every((f) => f.t <= f.F + 1e-9)); + const speed = (late.at(-1).t - late[0].t) / (late.at(-1).now - late[0].now); + assert.ok(speed > 0.5 && speed < 0.7, `plays ${speed} min/s for watches of 100 s`); + assert.ok(late.every((f) => f.F - f.t < 90), "keeps within an hour and a half of what is decided"); + assert.ok(Math.abs(p.watch() - 100) < 5, `times a watch at ${p.watch()} s`); +}); + +test("a shot's slower clock applies at the cut, and the clock jumps on when far behind", () => { + const p = new Pacer(240, 300, 0); + run(p, 30); + p.advance(1500, 30); + p.cut(0.1, false, 30); + assert.ok(p.v <= 0.1); + const t = p.t; + run(p, 60, () => {}, 30); + p.cut(Infinity, true, 90); + assert.ok(p.t > t + 20, `jumped from ${t} to ${p.t}`); + assert.ok(p.t <= p.F); +}); + +test("once every watch is decided the day's end plays out at the end speed", () => { + const p = new Pacer(1500, 1560, 0); + run(p, 100); + p.advance(1965, 100, true); + const frames = run(p, 120, () => {}, 100); + assert.ok(frames.at(-1).t > 1960, `at ${frames.at(-1).t}`); + assert.ok(frames.every((f) => f.rate <= 5 * 60 + 1e-9)); +}); diff --git a/tests/viewer/test_live_js.py b/tests/viewer/test_live_js.py new file mode 100644 index 0000000..ad7b774 --- /dev/null +++ b/tests/viewer/test_live_js.py @@ -0,0 +1,18 @@ +"""The live viewer's clock (live.test.mjs) under Node's test runner: never past what is decided, paced to the +watches.""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +TEST = Path(__file__).resolve().parent / "live.test.mjs" + + +def test_the_live_clock_keeps_to_what_is_decided_and_to_the_desks_pace(): + node = shutil.which("node") + if not node: + pytest.skip("no node on PATH") + run = subprocess.run([node, "--test", str(TEST)], capture_output=True, text=True, timeout=120) + assert run.returncode == 0, run.stdout[-4000:] + run.stderr[-2000:] diff --git a/tests/viewer/test_vendor.py b/tests/viewer/test_vendor.py index 0c6ad43..146e195 100644 --- a/tests/viewer/test_vendor.py +++ b/tests/viewer/test_vendor.py @@ -36,3 +36,15 @@ def test_no_vendored_file_is_left_out_of_the_repository_by_gitignore(): ignored = subprocess.run(["git", "check-ignore", "--no-index", *files], cwd=WEB.parent, capture_output=True, text=True).stdout.split() assert files and not ignored + + +def test_every_sound_file_is_shipped_listed_with_its_cc0_source_and_small(): + files = set(re.findall(r'"([\w-]+\.ogg)"', (WEB / "audio.js").read_text())) + shipped = {p.name for p in (WEB / "audio").glob("*.ogg")} + sources = (WEB / "audio" / "SOURCES.md").read_text() + assert files and files == shipped + for name in shipped: + row = next(r for r in sources.splitlines() if r.startswith(f"| `{name}` |")) + assert re.search(r"\(https://freesound\.org/people/[^/]+/sounds/\d+/\)", row), name + assert "CC0 1.0" in sources and "creativecommons.org/publicdomain/zero/1.0" in sources + assert sum(p.stat().st_size for p in (WEB / "audio").iterdir()) < 15e6 diff --git a/tests/viewer/test_viewer_browser.py b/tests/viewer/test_viewer_browser.py index aee365d..023ef3f 100644 --- a/tests/viewer/test_viewer_browser.py +++ b/tests/viewer/test_viewer_browser.py @@ -97,6 +97,17 @@ def test_an_episode_without_a_stage_plays_on_the_map(page): assert js("document.querySelectorAll('.expert .calls li').length") > 0 +def test_the_sound_is_off_until_asked_for_and_absent_with_sound_0(page): + goto, js = page + goto("episode=/tests/viewer/fixtures/episode_small.json") + js("airops.ready.then(() => new Promise(r => { const f = () => airops.sound ? r() : setTimeout(f, 50); f(); }))") + assert js("({on: airops.sound.on, ctx: airops.sound.ctx})") == {"on": False, "ctx": None} + assert js("document.querySelector('.timeline .sound-toggle').getAttribute('aria-pressed')") == "false" + goto("episode=/tests/viewer/fixtures/episode_small.json&sound=0") + js("airops.ready.then(() => new Promise(r => setTimeout(r, 300)))") + assert js("!airops.sound && !document.querySelector('.sound-toggle')") + + def test_the_viewer_lists_episodes(page): goto, js = page goto("index=/tests/viewer/fixtures/index.json") diff --git a/web/app.js b/web/app.js index a4f0163..3299d9d 100644 --- a/web/app.js +++ b/web/app.js @@ -1,6 +1,8 @@ // AirOpsEnv v1 viewer: ?episode= replays one episode on its clock, the 3D airport filling the screen with the US -// map in a corner, cards over them and a timeline below; ?index= lists episodes. window.airops {ready, time(t), -// view(name), show(step), shot(spec), state()} lets a recorder drive it frame by frame. +// map in a corner, cards over them and a timeline below; ?index= lists episodes; ?live= plays the day a live +// server is serving as it is decided (live.js). window.airops {ready, time(t), view(name), show(step), shot(spec), +// state()} lets a recorder drive it frame by frame. Sound (audio.js) is off until its speaker button is pressed, on with +// &sound=1 or &stream, and absent with ?sound=0. import { HUB_NAMES, cities, esc, flagged, flightsAt, positionsAt, prepare, programBands, watchAt } from "./model.js"; import { Board, Desk, Header, Passengers, Score, Timeline } from "./cards.js"; import { Room } from "./room.js"; @@ -41,7 +43,8 @@ boot(); async function boot() { const app = document.getElementById("app"); try { - if (params.get("episode")) await episodePage(app, new URL(params.get("episode"), location.href)); + if (params.has("live")) await (await load("./live.js")).livePage(app, new URL(params.get("live") || ".", location.href), { episodePage, params, ready: (x) => resolveReady(x) }); + else if (params.get("episode")) await episodePage(app, new URL(params.get("episode"), location.href)); else await indexPage(app, new URL(params.get("index") ?? "../cache/episodes/index.json", location.href)); } catch (e) { console.error(e); @@ -149,8 +152,10 @@ function note(list) { return `

The board: ${esc(model)} on ${days} storm days scores a mean reward of ${esc(parts.join(" and "))}. 1 is the best plan found; the standard crew desk scores about 0.31. The two desks used about twice the compute, so this doesn't show that talking helps.

`; } -async function episodePage(app, url) { - const ep = await getJSON(url); +/** One episode's page; a `live` day's (live.js) takes its episode, clock and cameras from it and returns the page's + * handle, which grows the day in place as more of it is decided (update()). */ +async function episodePage(app, url, live = null) { + const ep = live ? live.ep : await getJSON(url); const P = prepare(ep, url), task = ep.task; let doc = null, traffic = null, stageUrl = null; if (ep.stage) { @@ -158,16 +163,10 @@ async function episodePage(app, url) { [doc, traffic] = await Promise.all([getJSON(stageUrl), getJSON(new URL(ep.stage.traffic, url))]).catch((e) => (console.warn("no 3D airport:", e), [null, null])); } P.lo = traffic?.span?.[0] ?? task.start; - P.hi = traffic?.span?.[1] ?? P.end; - const city = cities(task, traffic), cityOf = (c) => city.get(c) ?? c, model = ep.model_label ?? ep.model, brand = ep.passengers ? "AirOpsEnv v3" : ep.occ ? "AirOpsEnv v2" : "AirOpsEnv v1"; - const stands = new Map(), terminals = new Map(); - for (const ac of traffic?.aircraft ?? []) { - if (!ac.task) continue; - for (const v of ac.visits) for (const [k, l] of [["arr", v.arr], ["dep", v.dep]]) if (l?.tag) stands.set(`${k}|${l.flight}`, v.stand); - } - for (const ref of stands.values()) if (ref.includes("-")) terminals.set(ref.split("-")[0], (terminals.get(ref.split("-")[0]) ?? 0) + 1); - const home = [...terminals].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null; - for (const [k, ref] of stands) stands.set(k, home && ref.startsWith(`${home}-`) ? ref.slice(home.length + 1) : ref.replace("-", " ")); + P.hi = traffic?.span?.[1] ?? (live ? Math.max(P.end, ...ep.schedule.map((s) => s.sched_arr)) + 60 : P.end); + let city = cities(task, traffic); + const cityOf = (c) => city.get(c) ?? c, model = ep.model_label ?? ep.model, brand = ep.passengers ? "AirOpsEnv v3" : ep.occ ? "AirOpsEnv v2" : "AirOpsEnv v1"; + let { stands, home } = gates(traffic); document.title = `${HUB_NAMES[task.hub] ?? task.hub} ${task.date} · ${model} · ${brand}`; app.innerHTML = `
The airport · click to swap
@@ -186,7 +185,7 @@ async function episodePage(app, url) { try { const { AirportStage, tagAt: tag } = await load("./stage/stage.js"); tagAt = tag; - stage = new AirportStage(root.querySelector(".stage-host"), { stage: doc, traffic, base: new URL(".", stageUrl).href, manual: RECORD, controls: !RECORD, avoid: () => covered(root) }); + stage = new AirportStage(root.querySelector(".stage-host"), { stage: doc, traffic, base: new URL(".", stageUrl).href, manual: RECORD, controls: !RECORD && !live, avoid: () => covered(root) }); await stage.ready; } catch (e) { console.error("the 3D airport failed, showing the map:", e); @@ -198,7 +197,7 @@ async function episodePage(app, url) { if (ep.occ) { try { const { ControlCenter } = await load("./center/center.js"); - center = new ControlCenter(root.querySelector(".room-host"), { P, ep, base: url.href, manual: RECORD, controls: !RECORD }); + center = new ControlCenter(root.querySelector(".room-host"), { P, ep, base: url.href, manual: RECORD, controls: !RECORD && !live }); await center.ready; } catch (e) { console.error("the control-center room failed:", e); @@ -206,15 +205,19 @@ async function episodePage(app, url) { center = null; } } - const views = new Views(root, P, { stage, room: center, interactive: !RECORD, + const views = new Views(root, P, { stage, room: center, interactive: !RECORD && !live, onSwap: (main) => setView(main === "map" ? "network" : main === "room" ? roomView : stageView) }); const ctx = { task, earlier: P.news, city: cityOf, hub: task.hub, sched: P.sched, flightOf: (ref) => P.sched.get(Number(String(ref).replace(/^f/, "")))?.flight }; - const header = new Header(root.querySelector(".head"), P, { model: ep.occ ? `${model} · ${ep.occ.label}` : model, weather: stage ? traffic.weather : null, brand }); - const desk = ep.occ ? new Room(root.querySelector(".desk"), P, { model }) : new Desk(root.querySelector(".desk"), P, { model, city: cityOf }); - const score = new Score(root.querySelector(".score"), P, { model }); - const pax = ep.passengers ? new Passengers(root.querySelector(".pax"), P) : null; - const board = new Board(root.querySelector(".board"), P, { city: cityOf, stands, terminal: home?.replace(/^T/, ""), rows: 8, scene: !!stage, - scope: ep.occ ? "all of its flights here" : "the task's flights" }); + let header, desk, score, pax, board; + const cards = () => { + header = new Header(root.querySelector(".card.head"), P, { model: ep.occ ? `${model} · ${ep.occ.label}` : model, weather: stage ? traffic.weather : null, brand }); + desk = ep.occ ? new Room(root.querySelector(".card.desk"), P, { model }) : new Desk(root.querySelector(".card.desk"), P, { model, city: cityOf }); + score = new Score(root.querySelector(".card.score"), P, { model }); + pax = ep.passengers ? new Passengers(root.querySelector(".card.pax"), P) : null; + board = new Board(root.querySelector(".card.board"), P, { city: cityOf, stands, terminal: home?.replace(/^T/, ""), rows: 8, scene: !!stage, + scope: ep.occ ? "all of its flights here" : "the task's flights" }); + }; + cards(); const expertEl = root.querySelector(".expert"); let expert = null; const crews = !!traffic?.crews?.length; @@ -363,15 +366,32 @@ async function episodePage(app, url) { else if (k === "x") setExpert(expertEl.hidden); }); + /** A frame; a live page's carries on past an error in one, since it runs for a day. */ function tick(now) { + if (!live) return frame(now); + requestAnimationFrame(tick); + try { + frame(now, false); + } catch (e) { + live.error(e); + } + } + + function frame(now, next = true) { const dt = Math.min(0.1, (now - last) / 1000); last = now; - boost = playing && quiet(t) ? 6 : 1; - if (playing) { - t = clamp(t + dt * speed * boost); - if (t >= P.hi) play(false); + if (live) { + const c = live.clock(now, dt); + t = clamp(c.t); + rate = c.rate; + } else { + boost = playing && quiet(t) ? 6 : 1; + if (playing) { + t = clamp(t + dt * speed * boost); + if (t >= P.hi) play(false); + } + rate = playing ? speed * 60 * boost : null; } - rate = playing ? speed * 60 * boost : null; draw(t); if (stage && !busy && views.visible("stage")) { busy = true; @@ -381,24 +401,26 @@ async function episodePage(app, url) { busyRoom = true; Promise.resolve(center.time(t, { dt, speed: rate ?? 0 })).finally(() => (busyRoom = false)); } - requestAnimationFrame(tick); + live?.frame(t, now, dt); + if (next) requestAnimationFrame(tick); } await views.map.loaded; + if (live) t = clamp(live.clock(performance.now(), 0).t); let open = moment ? stage.moment(moment) : null; if (open) { draw(clamp(open.t), { force: true }); open = stage.moment(moment); t = clamp(open.t); } - await setView(asked ?? viewName, open ? { lounge: open.lounge, focus: open.focus, auto: true } : {}); + await setView(live ? live.view ?? viewName : asked ?? viewName, open ? { lounge: open.lounge, focus: open.focus, auto: true } : {}); if (params.get("expert") === "1") setExpert(true); draw(t, { force: true }); if (stage) await stage.time(t, { dt: 0, speed: 0 }); if (center) await center.time(t, { dt: 0, speed: 0 }); await settle(); if (!RECORD) { - if (params.get("play") === "1") play(true); + if (params.get("play") === "1" && !live) play(true); requestAnimationFrame(tick); } @@ -440,7 +462,44 @@ async function episodePage(app, url) { room: center, map: views.map, }; + if (!RECORD && params.get("sound") !== "0") { + load("./audio.js").then(({ mountSound }) => mountSound(window.airops, root.querySelector(".timeline"), { on: params.has("stream") || params.get("sound") === "1", stream: params.has("stream") })) + .catch((e) => console.warn("no sound:", e)); + } resolveReady({ steps: P.steps.length, watches: P.watches.map((w) => w.t), start: task.start, t0: P.lo, end: P.hi, horizon: P.hi, last_watch: P.lastWatch, views: [...(stage ? stageViews : []), ...(center ? ROOM_VIEWS : []), "network"], radar: views.radar?.count ?? 0, twin: !!ep.twin, stage: !!stage, room: !!center, span: [P.lo, P.hi], featured: traffic?.featured ?? [], hub: task.hub }); + if (!live) return null; + + /** The live day grown: the episode and its prepared index in place, the cards and the room built again over them, a + * new traffic sidecar into the 3D airport. */ + async function update(next, nextTraffic = null) { + for (const k of Object.keys(ep)) if (!(k in next)) delete ep[k]; + Object.assign(ep, next); + Object.assign(P, prepare(ep, url), { lo: P.lo, hi: P.hi }); + if (nextTraffic) { + traffic = nextTraffic; + city = cities(task, traffic); + ({ stands, home } = gates(traffic)); + } + cards(); + await Promise.all([nextTraffic && stage ? stage.retraffic(nextTraffic) : null, center?.reload(P, ep)]); + draw(t, { force: true }); + } + + return { P, ep, root, stage, room: center, views, setView, update, draw, traffic: () => traffic, + view: () => viewName, time: () => t, rate: () => rate, stageViews, terminal, crews }; +} + +/** Where each of the day's flights parks in 3D (`dep|B6 123` to a gate's name) and the terminal most of them use. */ +function gates(traffic) { + const stands = new Map(), terminals = new Map(); + for (const ac of traffic?.aircraft ?? []) { + if (!ac.task) continue; + for (const v of ac.visits) for (const [k, l] of [["arr", v.arr], ["dep", v.dep]]) if (l?.tag) stands.set(`${k}|${l.flight}`, v.stand); + } + for (const ref of stands.values()) if (ref.includes("-")) terminals.set(ref.split("-")[0], (terminals.get(ref.split("-")[0]) ?? 0) + 1); + const home = [...terminals].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null; + for (const [k, ref] of stands) stands.set(k, home && ref.startsWith(`${home}-`) ? ref.slice(home.length + 1) : ref.replace("-", " ")); + return { stands, home }; } diff --git a/web/audio.js b/web/audio.js new file mode 100644 index 0000000..3f896c4 --- /dev/null +++ b/web/audio.js @@ -0,0 +1,878 @@ +// The viewer's sound, made each frame from what is on screen. Outdoors: the airport's bed and the aircraft nearest the +// camera by phase (idle and taxi, pushback with its tug, APU and engine start, the takeoff roar with Doppler, approach, +// reverse thrust and spool-down), each with gain, air absorption and pan by its distance and bearing. In the terminal: +// its hall and crowd by the gate area's people, a soft chime now and then and the jets muffled by the glass. In the +// control-center room: its tone, the keys of the desk whose turn it is and clicks as requests fly. On the map: the +// airport faint and muffled. Rain, wind and thunder follow the stage's weather. A leveller and a sample-peak limiter on +// the audio thread hold it near -20 LUFS for a stream. Recordings are CC0 (audio/SOURCES.md); what has none is synthesized. +// const sound = new AirportSound().attach(window.airops); await sound.enable(); then sound.update(state, camera) per frame. + +const FILES = { + apron: "apron.ogg", idle: "idle.ogg", roar: "roar.ogg", far: "roar-far.ogg", approach: "approach.ogg", reverse: "reverse.ogg", + tug: "tug.ogg", terminal: "terminal.ogg", crowd: "crowd.ogg", keys: "keys.ogg", click: "click.ogg", rain: "rain.ogg", + wind: "wind.ogg", thunder: "thunder.ogg", rumble: "rumble.ogg", +}; +const VOICES = 8; +const R0 = 60; +const HEARD = 0.004; +/** Each scene's bus gains, the low-pass on the outdoors heard from it, and whether the leveller holds its gain there (the map's bed is quiet on purpose). */ +const MIX = { + outdoor: { air: 1, hall: 0, room: 0, lp: 20000, hold: 0 }, + terminal: { air: 0.42, hall: 1, room: 0, lp: 650, hold: 0 }, + room: { air: 0, hall: 0, room: 1, lp: 400, hold: 0 }, + map: { air: 0.55, hall: 0, room: 0, lp: 420, hold: 1 }, +}; +/** Per aircraft class: pitch and loudness. */ +const CLASS = { widebody: [0.86, 1.3], a321: [0.97, 1.08], narrowbody: [1, 1], a320: [1, 1], a220: [1.06, 0.92], e190: [1.08, 0.88], + regional: [1.12, 0.8], regional_t: [1.3, 0.62] }; +const LEVEL = { jets: 2.2, apron: 0.8, wind: 0.75, rain: 0.8, terminal: 1.1, crowd: 0.85, keys: 0.85, chime: 0.22, tone: 3.2, hvac: 3 }; + +const clamp = (x, a, b) => Math.max(a, Math.min(b, x)); +const smooth = (a, b, x) => { + const u = clamp((x - a) / (b - a), 0, 1); + return u * u * (3 - 2 * u); +}; +/** setTargetAtTime unless the value is already headed there, so a param's timeline stays short over a day of frames. */ +const ramp = (p, v, now, tau) => { + if (p._v !== undefined && Math.abs(p._v - v) <= 0.0015 + 0.01 * Math.abs(v)) return; + p._v = v; + p.setTargetAtTime(v, now, tau); +}; + +/** A looped buffer whose source runs only while it is heard: set(level, rate) each frame. */ +class Loop { + constructor(ctx, out, buf) { + this.ctx = ctx; + this.buf = buf; + this.g = ctx.createGain(); + this.g.gain.value = 0; + this.g.connect(out); + this.src = null; + this.quiet = 0; + } + + set(level, rate = null, tau = 0.08) { + const now = this.ctx.currentTime; + if (level > 1e-4 && !this.src && this.buf) { + const s = this.ctx.createBufferSource(); + s.buffer = this.buf; + s.loop = true; + s.playbackRate.value = rate ?? 1; + s.connect(this.g); + s.start(now, Math.random() * this.buf.duration); + this.src = s; + } + if (!this.src) return; + ramp(this.g.gain, level, now, tau); + if (rate != null) ramp(this.src.playbackRate, rate, now, 0.12); + if (level > 1e-4) this.quiet = 0; + else if (!this.quiet) this.quiet = now; + else if (now - this.quiet > 2.5) this.stop(); + } + + stop() { + if (!this.src) return; + this.src.stop(); + this.src.disconnect(); + this.src = null; + this.quiet = 0; + } +} + +/** One aircraft's sound: its layers through a low-pass for air absorption, a gain for distance and a pan for bearing. */ +class Voice { + constructor(ctx, out, B) { + this.ctx = ctx; + this.lp = ctx.createBiquadFilter(); + this.lp.type = "lowpass"; + this.lp.Q.value = 0.55; + this.g = ctx.createGain(); + this.g.gain.value = 0; + this.pan = ctx.createStereoPanner(); + this.lp.connect(this.g).connect(this.pan).connect(out); + this.L = Object.fromEntries(["idle", "roar", "far", "approach", "reverse", "tug", "apu"].map((k) => [k, new Loop(ctx, this.lp, B[k])])); + this.key = null; + this.free = 0; + this.d = null; + this.vr = 0; + } + + play(c, now, tau, cut, dts) { + if (this.key !== c.key || cut) { + this.d = null; + this.vr = 0; + } + this.key = c.key; + if (this.d != null && dts > 0) this.vr += (clamp((this.d - c.d) / dts, -110, 110) - this.vr) * 0.35; + else if (dts <= 0) this.vr *= 0.8; + this.d = c.d; + const dop = clamp(343 / (343 - this.vr), 0.72, 1.4); + const s = c.spec, r = c.rate * dop; + const far = smooth(260, 1600, c.d); + ramp(this.g.gain, c.g, now, tau); + ramp(this.lp.frequency, clamp(19000 * (150 / Math.max(150, c.de)) ** 0.65, 400, 19000), now, tau); + ramp(this.pan.pan, c.pan, now, tau); + this.L.idle.set(s.idle || 0, r * (s.idleRate || 1), tau); + this.L.roar.set((s.roar || 0) * (1 - far), r * (s.roarRate || 1), tau); + this.L.far.set((s.roar || 0) * far * 1.25 + (s.reverse || 0) * far * 0.6, r * (s.roarRate || 1), tau); + this.L.approach.set(s.approach || 0, r, tau); + this.L.reverse.set((s.reverse || 0) * (1 - 0.5 * far), r, tau); + this.L.tug.set(s.tug || 0, 1, tau); + this.L.apu.set(s.apu || 0, c.rate, tau); + } + + release(now) { + if (this.key == null) return; + this.key = null; + this.free = now + 0.8; + ramp(this.g.gain, 0, now, 0.18); + for (const l of Object.values(this.L)) l.set(0, null, 0.18); + } + + tick() { + if (this.key != null) return; + for (const l of Object.values(this.L)) if (l.src) l.set(0, null, 0.18); + } +} + +/** The master: a slow leveller that holds the loudness of the last quarter-minute or so near `target` (and its gain + * while `hold`), a compressor and a lookahead sample-peak limiter at `ceiling`, with BS.1770 meters (M, S, I) posted back + * four times a second. */ +const MASTER = ` +const kw = (fs) => { + let K = Math.tan(Math.PI * 1681.974450955533 / fs), Q = 0.7071752369554196, Vh = Math.pow(10, 3.999843853973347 / 20), Vb = Math.pow(Vh, 0.4996667741545416); + let a0 = 1 + K / Q + K * K; + const pre = [(Vh + Vb * K / Q + K * K) / a0, 2 * (K * K - Vh) / a0, (Vh - Vb * K / Q + K * K) / a0, 2 * (K * K - 1) / a0, (1 - K / Q + K * K) / a0]; + K = Math.tan(Math.PI * 38.13547087602444 / fs); Q = 0.5003270373238773; a0 = 1 + K / Q + K * K; + return [pre, [1, -2, 1, 2 * (K * K - 1) / a0, (1 - K / Q + K * K) / a0]]; +}; +const db = (x) => (x > 1e-12 ? 10 * Math.log10(x) : -120); +const lufs = (e) => -0.691 + db(e); +class Master extends AudioWorkletProcessor { + static get parameterDescriptors() { + return [{ name: "target", defaultValue: -20, automationRate: "k-rate" }, { name: "ceiling", defaultValue: -4.5, automationRate: "k-rate" }, + { name: "hold", defaultValue: 0, automationRate: "k-rate" }]; + } + constructor() { + super(); + this.k = kw(sampleRate); + this.z = [new Float64Array(4), new Float64Array(4), new Float64Array(4), new Float64Array(4)]; + this.blk = Math.round(sampleRate / 10); + this.e = 0; this.n = 0; + this.ring = new Float64Array(30); this.ri = 0; this.filled = 0; + this.hist = new Float64Array(760); this.histN = new Uint32Array(760); + this.agc = 0; this.env = 1e-6; this.cg = 1; this.lg = 1; + this.d = [new Float32Array(128), new Float32Array(128)]; this.w = [new Float32Array(128), new Float32Array(128)]; this.dPeak = 0; + this.peak = 0; this.minLg = 1; this.post = 0; + this.att = Math.exp(-1 / (0.012 * sampleRate)); this.rel = Math.exp(-1 / (0.35 * sampleRate)); + this.gatt = Math.exp(-1 / (0.004 * sampleRate)); this.grel = Math.exp(-1 / (0.25 * sampleRate)); + this.port.onmessage = (m) => { if (m.data === "reset") { this.hist.fill(0); this.histN.fill(0); } }; + } + biquad(c, z, x) { + const y = c[0] * x + z[0] + 1e-20; + z[0] = c[1] * x - c[3] * y + z[1]; + z[1] = c[2] * x - c[4] * y; + return y; + } + block(target, hold) { + this.ring[this.ri] = this.e / this.n; this.ri = (this.ri + 1) % 30; this.filled = Math.min(30, this.filled + 1); + this.e = 0; this.n = 0; + let m = 0, s = 0; + for (let i = 0; i < this.filled; i++) { const v = this.ring[(this.ri - 1 - i + 30) % 30]; if (i < 4) m += v; s += v; } + const M = lufs(m / Math.min(4, this.filled)), S = lufs(s / this.filled); + if (M > -70) { const b = Math.min(759, Math.max(0, Math.round((M + 70) * 10))); this.hist[b] += Math.pow(10, (M + 0.691) / 10); this.histN[b]++; } + this.M = M; this.S = S; + this.lt = Math.max(1e-12, this.lt == null ? s / this.filled : this.lt + (s / this.filled - this.lt) * 0.007); + const LT = lufs(this.lt); + if (!hold && LT > target - 16 && this.filled >= 10) { + const err = target - LT; + this.agc += Math.max(-0.03, Math.min(0.015, err * 0.01)); + this.agc = Math.max(-9, Math.min(6, this.agc)); + } + } + integrated() { + let e = 0, n = 0; + for (let i = 0; i < 760; i++) { e += this.hist[i]; n += this.histN[i]; } + if (!n) return -70; + const gate = lufs(e / n) - 10; + e = 0; n = 0; + for (let i = Math.max(0, Math.ceil((gate + 70) * 10)); i < 760; i++) { e += this.hist[i]; n += this.histN[i]; } + return n ? lufs(e / n) : -70; + } + process(inputs, outputs, P) { + const out = outputs[0], inp = inputs[0], n = out[0].length; + const L = inp && inp[0], R = inp && (inp[1] || inp[0]); + const target = P.target[0], hold = P.hold[0] >= 0.5, ceil = Math.pow(10, P.ceiling[0] / 20); + const g0 = Math.pow(10, this.agc / 20), thr = Math.pow(10, (target + 9) / 10); + if (this.w[0].length !== n) this.w = [new Float32Array(n), new Float32Array(n)]; + const yl = this.w[0], yr = this.w[1]; + let pk = 0; + for (let i = 0; i < n; i++) { + const l = L ? L[i] * g0 : 0, r = R ? R[i] * g0 : 0; + const ms = (l * l + r * r) / 2; + this.env = Math.max(1e-12, ms > this.env ? this.att * this.env + (1 - this.att) * ms : this.rel * this.env + (1 - this.rel) * ms); + const want = this.env > thr ? Math.pow(thr / this.env, 0.5 * (1 - 1 / 3)) : 1; + this.cg = want < this.cg ? this.gatt * this.cg + (1 - this.gatt) * want : this.grel * this.cg + (1 - this.grel) * want; + const a = l * this.cg, b = r * this.cg; + yl[i] = a; yr[i] = b; + pk = Math.max(pk, Math.abs(a), Math.abs(b)); + const kl = this.biquad(this.k[1], this.z[1], this.biquad(this.k[0], this.z[0], a)); + const kr = this.biquad(this.k[1], this.z[3], this.biquad(this.k[0], this.z[2], b)); + this.e += kl * kl + kr * kr; this.n++; + if (this.n >= this.blk) this.block(target, hold); + } + const T = Math.min(1, ceil / Math.max(pk, this.dPeak, 1e-9)); + const end = T < this.lg ? T : this.lg + (T - this.lg) * (1 - Math.exp(-n / (0.15 * sampleRate))); + const o0 = out[0], o1 = out[1] || out[0], d0 = this.d[0], d1 = this.d[1]; + for (let i = 0; i < n; i++) { + const g = this.lg + (end - this.lg) * (i + 1) / n; + o0[i] = Math.max(-ceil, Math.min(ceil, d0[i] * g)); + if (o1 !== o0) o1[i] = Math.max(-ceil, Math.min(ceil, d1[i] * g)); + this.peak = Math.max(this.peak, Math.abs(o0[i]), Math.abs(o1[i])); + } + this.minLg = Math.min(this.minLg, end); + this.lg = end; + this.w = this.d; this.d = [yl, yr]; this.dPeak = pk; + if ((this.post += n) >= sampleRate / 4) { + this.post = 0; + this.port.postMessage({ M: this.M ?? -70, S: this.S ?? -70, I: this.integrated(), peak: 20 * Math.log10(Math.max(this.peak, 1e-6)), + limit: 20 * Math.log10(this.minLg), agc: this.agc, comp: 20 * Math.log10(this.cg) }); + this.peak = 0; this.minLg = 1; + } + return true; + } +} +registerProcessor("airops-master", Master); +`; + +/** A seeded noise source for the synthesized buffers. */ +function rng(seed) { + let s = seed >>> 0 || 1; + return () => ((s = (Math.imul(s, 1664525) + 1013904223) >>> 0) / 4294967296) * 2 - 1; +} + +/** A stereo buffer of `sec` seconds filled by f(i, t, ch) with its last `fade` seconds crossfaded into its start, so it loops. */ +function synth(ctx, sec, f, fade = 0.5) { + const sr = ctx.sampleRate, n = Math.round(sec * sr), x = Math.round(fade * sr); + const buf = ctx.createBuffer(2, n, sr); + for (let ch = 0; ch < 2; ch++) { + const tmp = new Float32Array(n + x); + for (let i = 0; i < n + x; i++) tmp[i] = f(i, i / sr, ch); + const d = buf.getChannelData(ch); + for (let i = 0; i < n; i++) d[i] = tmp[i]; + for (let i = 0; i < x; i++) { + const u = i / x; + d[i] = tmp[i] * Math.sin((u * Math.PI) / 2) + tmp[n + i] * Math.cos((u * Math.PI) / 2); + } + } + return buf; +} + +/** Coloured noise: white through two one-pole low-passes at `lo` Hz and a one-pole high-pass at `hi` Hz, near `level` RMS. */ +function noise(sr, seed, lo, hi, level) { + const r = rng(seed), a = Math.exp((-2 * Math.PI * lo) / sr), b = Math.exp((-2 * Math.PI * hi) / sr); + let y = 0, y2 = 0, h = 0, px = 0; + const k = level * Math.sqrt((1 + a) / (1 - a)) * 2.7; + return () => { + y = a * y + (1 - a) * r(); + y2 = a * y2 + (1 - a) * y; + h = b * (h + y2 - px); + px = y2; + return h * k; + }; +} + +/** The sounds no recording covers, and a stand-in for any recording that fails to load. */ +function synthesized(ctx) { + const sr = ctx.sampleRate; + const S = {}; + const apuN = [noise(sr, 7, 4500, 900, 0.025), noise(sr, 8, 4500, 900, 0.025)]; + const wob = (t, f) => 2 * Math.PI * f * t + 0.3 * Math.sin(2 * Math.PI * 0.5 * t); + S.apu = synth(ctx, 6, (i, t, ch) => apuN[ch]() + 0.018 * Math.sin(wob(t, 1166)) + 0.007 * Math.sin(wob(t, 2332) + ch) + 0.004 * Math.sin(wob(t, 4310) + 2 * ch) + + 0.012 * Math.sin(2 * Math.PI * 400 * t), 1); + const toneN = [noise(sr, 11, 140, 25, 0.03), noise(sr, 12, 140, 25, 0.03)], hvacN = [noise(sr, 13, 2200, 260, 0.006), noise(sr, 14, 2200, 260, 0.006)]; + S.tone = synth(ctx, 9, (i, t, ch) => toneN[ch]() + 0.0025 * Math.sin(2 * Math.PI * 120 * t) + 0.0012 * Math.sin(2 * Math.PI * 180 * t), 1.5); + S.hvac = synth(ctx, 11, (i, t, ch) => hvacN[ch]() * (1 + 0.15 * Math.sin(2 * Math.PI * 0.09 * t + ch)), 1.5); + const hallN = [noise(sr, 15, 700, 60, 0.012), noise(sr, 16, 700, 60, 0.012)]; + S.hall = synth(ctx, 10, (i, t, ch) => hallN[ch](), 1.5); + const jet = [noise(sr, 21, 1800, 40, 0.25), noise(sr, 22, 1800, 40, 0.25)]; + S.jet = synth(ctx, 6, (i, t, ch) => jet[ch]() * (1 + 0.2 * Math.sin(2 * Math.PI * 3.1 * t + ch)) + 0.02 * Math.sin(2 * Math.PI * 2650 * t), 1); + const whine = [noise(sr, 23, 5000, 300, 0.06), noise(sr, 24, 5000, 300, 0.06)]; + S.whine = synth(ctx, 6, (i, t, ch) => whine[ch]() + 0.025 * Math.sin(2 * Math.PI * 3300 * t) + 0.015 * Math.sin(2 * Math.PI * 1650 * t + ch), 1); + const air = [noise(sr, 25, 500, 30, 0.08), noise(sr, 26, 500, 30, 0.08)]; + S.air = synth(ctx, 8, (i, t, ch) => air[ch]() * (0.8 + 0.25 * Math.sin(2 * Math.PI * 0.13 * t + 2 * ch)), 1.5); + const hiss = [noise(sr, 27, 12000, 900, 0.07), noise(sr, 28, 12000, 900, 0.07)]; + S.hiss = synth(ctx, 7, (i, t, ch) => hiss[ch](), 1); + return S; +} + +/** The terminal's chime: two soft bell tones, rendered once through a hall's reverb. */ +async function chime(ctx) { + const sr = ctx.sampleRate, sec = 5; + const off = new OfflineAudioContext(2, sr * sec, sr); + const ir = off.createBuffer(2, Math.round(sr * 2.6), sr); + for (let ch = 0; ch < 2; ch++) { + const r = rng(31 + ch), d = ir.getChannelData(ch); + for (let i = 0; i < d.length; i++) d[i] = r() * Math.exp((-i / sr) * 2.6) * 0.5; + } + const verb = off.createConvolver(); + verb.buffer = ir; + const wet = off.createGain(); + wet.gain.value = 0.35; + verb.connect(wet).connect(off.destination); + for (const [at, f] of [[0.1, 784], [0.75, 622.3]]) { + for (const [h, a] of [[1, 0.5], [2.76, 0.12], [5.4, 0.04]]) { + const o = off.createOscillator(), g = off.createGain(); + o.frequency.value = f * h; + g.gain.setValueAtTime(0, at); + g.gain.linearRampToValueAtTime(a, at + 0.012); + g.gain.setTargetAtTime(0, at + 0.012, 0.55 / h); + o.connect(g); + g.connect(off.destination); + g.connect(verb); + o.start(at); + o.stop(at + 3.5); + } + } + return off.startRendering(); +} + +export class AirportSound { + /** base: where the recordings are; volume: the output's level (0..1). */ + constructor({ base = new URL("./audio/", import.meta.url).href, volume = 1 } = {}) { + this.base = base; + this.volume = volume; + this.viewer = null; + this.ctx = null; + this.on = false; + this.ready = null; + this.meter = null; + this.scene = null; + this.voices = []; + this.shots = 0; + this.last = 0; + this.external = 0; + this.lite = null; + this.prev = { T: null, rig: null, view: null }; + this.weather = { flash: 0, far: 0, thunderAt: 0, gust: 0 }; + this.terminal = { chimeAt: 0 }; + this.room = { cards: null, turn: null, clickAt: 0, keyAt: 0 }; + this.listeners = new Set(); + } + + /** The viewer to listen to: window.airops (its stage, room and state()). */ + attach(viewer) { + this.viewer = viewer; + return this; + } + + /** Start the sound (from a click where the browser needs one); resolves to whether it plays. */ + async enable() { + this.on = true; + if (!this.ctx) { + const AC = globalThis.AudioContext || globalThis.webkitAudioContext; + this.ctx = new AC({ latencyHint: "playback", sampleRate: 48000 }); + this.ready = this.#build().catch((e) => { + console.warn("sound: the audio graph failed", e); + throw e; + }); + this.#notify(); + } + await this.ctx.resume().catch(() => {}); + await this.ready; + const now = this.ctx.currentTime; + this.out.gain.cancelScheduledValues(now); + this.out.gain.setTargetAtTime(this.volume, now, 0.5); + this.out.gain._v = this.volume; + this.#notify(); + return this.ctx.state === "running"; + } + + /** Fade out and let the context sleep. */ + async disable() { + this.on = false; + this.#notify(); + if (!this.ctx) return; + await this.ready; + const now = this.ctx.currentTime; + this.out.gain.cancelScheduledValues(now); + this.out.gain.setTargetAtTime(0, now, 0.12); + this.out.gain._v = 0; + clearTimeout(this.sleep); + this.sleep = setTimeout(() => !this.on && this.ctx.suspend(), 900); + } + + toggle() { + return this.on ? this.disable() : this.enable(); + } + + /** The output's level (0..1), eased. */ + setVolume(v) { + this.volume = clamp(v, 0, 1); + if (this.out && this.on) ramp(this.out.gain, this.volume, this.ctx.currentTime, 0.4); + } + + /** Called by the viewer each frame: state as window.airops.state() gives it (main, view; derived when absent) and the + * camera on screen (derived when absent). */ + update(state = null, camera = null) { + this.external = performance.now(); + this.#update(state, camera); + } + + /** Drive update() from its own animation frames whenever the viewer has not called it lately. */ + auto() { + if (this.raf) return this; + const loop = () => { + this.raf = requestAnimationFrame(loop); + if (performance.now() - this.external > 300) this.#update(null, null); + }; + this.raf = requestAnimationFrame(loop); + return this; + } + + /** What plays: the scene, the voices and their aircraft, the meters (LUFS, dBFS) and the live one-shots. */ + state() { + return { + on: this.on, + context: this.ctx?.state ?? null, + worklet: !!this.master?.port, + scene: this.scene, + voices: this.voices.filter((v) => v.key != null).map((v) => ({ key: v.key, gain: +v.g.gain.value.toFixed(4), phase: v.phase })), + sources: this.voices.reduce((n, v) => n + Object.values(v.L).filter((l) => l.src).length, 0) + Object.values(this.beds ?? {}).filter((l) => l.src).length, + shots: this.shots, + meter: this.meter, + time: this.ctx?.currentTime ?? 0, + }; + } + + /** A speaker button for the timeline that turns the sound on and off. */ + button() { + const b = document.createElement("button"); + b.type = "button"; + b.className = "sound-toggle"; + const paint = () => { + b.classList.toggle("on", this.on); + b.setAttribute("aria-pressed", String(this.on)); + b.title = this.on ? "Sound on: the airport, the terminal and the room (click to mute)" : "Sound off (click for the airport's sound)"; + b.innerHTML = ``; + b.setAttribute("aria-label", this.on ? "Mute the sound" : "Turn the sound on"); + }; + b.addEventListener("click", () => this.toggle()); + this.listeners.add(paint); + paint(); + return b; + } + + dispose() { + cancelAnimationFrame(this.raf); + this.raf = null; + this.ctx?.close(); + this.ctx = null; + this.on = false; + } + + #notify() { + for (const f of this.listeners) f(); + } + + async #build() { + const ctx = this.ctx; + const S = synthesized(ctx); + const loaded = await Promise.all(Object.entries(FILES).map(async ([k, f]) => { + try { + const r = await fetch(new URL(f, this.base)); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return [k, await ctx.decodeAudioData(await r.arrayBuffer())]; + } catch (e) { + console.warn(`sound: ${f} did not load, synthesizing it`, e); + return [k, null]; + } + })); + const B = Object.fromEntries(loaded); + const stand = { apron: S.air, idle: S.whine, roar: S.jet, far: S.air, approach: S.whine, reverse: S.jet, tug: S.air, terminal: S.hall, crowd: S.hall, keys: null, + click: null, rain: S.hiss, wind: S.air, thunder: null, rumble: null }; + for (const k of Object.keys(FILES)) B[k] ??= stand[k]; + B.apu = S.apu; + B.chime = await chime(ctx); + + this.out = ctx.createGain(); + this.out.gain.value = 0; + this.out.connect(ctx.destination); + this.mix = ctx.createGain(); + const hp = ctx.createBiquadFilter(), shelf = ctx.createBiquadFilter(), air = ctx.createBiquadFilter(); + hp.type = "highpass"; + hp.frequency.value = 55; + hp.Q.value = 0.7; + shelf.type = "lowshelf"; + shelf.frequency.value = 200; + shelf.gain.value = -8; + air.type = "highshelf"; + air.frequency.value = 3500; + air.gain.value = 3; + const eq = this.mix.connect(hp).connect(shelf).connect(air); + try { + const url = URL.createObjectURL(new Blob([MASTER], { type: "application/javascript" })); + await ctx.audioWorklet.addModule(url); + URL.revokeObjectURL(url); + this.master = new AudioWorkletNode(ctx, "airops-master", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [2], channelCount: 2, channelCountMode: "explicit" }); + this.master.port.onmessage = (e) => (this.meter = e.data); + eq.connect(this.master).connect(this.out); + } catch (e) { + console.warn("sound: no AudioWorklet (needs https, localhost or a secure-origin flag), using the built-in compressor and a clipper", e); + this.master = null; + const comp = ctx.createDynamicsCompressor(), lim = ctx.createDynamicsCompressor(), trim = ctx.createGain(), clip = ctx.createWaveShaper(); + const c = 10 ** (-2 / 20), knee = 0.6 * c; + clip.curve = Float32Array.from({ length: 4097 }, (_, i) => { + const x = (i / 2048) - 1, a = Math.abs(x); + return Math.sign(x) * (a <= knee ? a : knee + (c - knee) * Math.tanh((a - knee) / (c - knee))); + }); + comp.threshold.value = -20; + comp.knee.value = 8; + comp.ratio.value = 3; + comp.attack.value = 0.01; + comp.release.value = 0.3; + lim.threshold.value = -3; + lim.knee.value = 0; + lim.ratio.value = 20; + lim.attack.value = 0.002; + lim.release.value = 0.12; + trim.gain.value = 0.6; + eq.connect(comp).connect(lim).connect(trim).connect(clip).connect(this.out); + } + const bus = (to = this.mix) => { + const g = ctx.createGain(); + g.gain.value = 0; + g.connect(to); + return g; + }; + this.airLp = ctx.createBiquadFilter(); + this.airLp.type = "lowpass"; + this.airLp.frequency.value = 20000; + this.airLp.Q.value = 0.5; + this.bus = { air: bus(), hall: bus(), room: bus() }; + this.airIn = ctx.createGain(); + this.airIn.connect(this.airLp).connect(this.bus.air); + this.jets = ctx.createGain(); + this.jets.gain.value = LEVEL.jets; + this.jets.connect(this.airIn); + this.voices = Array.from({ length: VOICES + 2 }, () => new Voice(ctx, this.jets, B)); + const thunderBus = ctx.createGain(); + thunderBus.connect(this.airIn); + this.thunderBus = thunderBus; + this.hallBus = ctx.createGain(); + this.hallBus.connect(this.bus.hall); + this.roomBus = ctx.createGain(); + this.roomBus.connect(this.bus.room); + this.panes = {}; + for (const [k, p] of [["customer", -0.35], ["ops", 0], ["crew", 0.35], ["one", 0]]) { + const s = ctx.createStereoPanner(); + s.pan.value = p; + s.connect(this.roomBus); + this.panes[k] = s; + } + this.beds = { + apron: new Loop(ctx, this.airIn, B.apron), + apron2: new Loop(ctx, this.airIn, B.idle), + wind: new Loop(ctx, this.airIn, B.wind), + rain: new Loop(ctx, this.airIn, B.rain), + terminal: new Loop(ctx, this.hallBus, B.terminal), + crowd: new Loop(ctx, this.hallBus, B.crowd), + hall: new Loop(ctx, this.hallBus, S.hall), + tone: new Loop(ctx, this.roomBus, S.tone), + hvac: new Loop(ctx, this.roomBus, S.hvac), + }; + this.keys = Object.fromEntries(["customer", "ops", "crew"].map((k) => [k, new Loop(ctx, this.panes[k], B.keys)])); + this.B = B; + } + + #update(state, camera) { + if (!this.on || !this.ctx || !this.B || !this.viewer) return; + const wall = performance.now(); + if (wall - this.last < 30) return; + const dtw = Math.min(0.5, (wall - this.last) / 1000); + this.last = wall; + if (this.ctx.state !== "running") { + if (wall - (this.resumed ?? 0) > 2000) { + this.resumed = wall; + this.ctx.resume().catch(() => {}); + } + return; + } + const v = this.viewer; + if (!state?.main) { + if (!this.lite || wall - this.lite.at > 250) this.lite = { at: wall, s: v.state?.() ?? {} }; + state = { ...this.lite.s, ...(state ?? {}) }; + } + const scene = state.main === "room" ? "room" : state.main === "map" || state.view === "network" || (!v.stage && state.main !== "room") ? "map" + : state.view === "terminal" ? "terminal" : "outdoor"; + const stage = v.stage ?? null; + camera ??= scene === "room" ? v.room?.camera : stage?.camera; + const now = this.ctx.currentTime; + const cutScene = scene !== this.scene; + if (cutScene) this.#scene(scene, now); + const rig = stage?.rig ?? null; + const T = stage?.T ?? state.t ?? 0; + const dts = this.prev.T == null ? 0 : (T - this.prev.T) * 60; + const rate = stage?.rate ?? 0; + const cut = cutScene || rig !== this.prev.rig || state.view !== this.prev.view || dts < 0 || dts > Math.max(8, Math.abs(rate) * dtw * 3 + 2); + this.prev = { T, rig, view: state.view }; + const fast = smooth(20, 300, Math.abs(rate)); + const tau = cut ? 0.3 : 0.07 + 0.35 * fast; + const outside = scene === "outdoor" || scene === "terminal"; + if (stage && camera && outside) this.#fleet(stage, camera, now, tau, cut, dts, scene, 1 - 0.4 * fast); + else for (const vc of this.voices) vc.release(now); + for (const vc of this.voices) vc.tick(); + if (stage && camera && scene !== "room") this.#air(stage, camera, now, scene, dtw); + else for (const k of ["apron", "apron2", "wind", "rain"]) this.beds[k].src && this.beds[k].set(0, null, 0.4); + if (stage && scene === "terminal") this.#terminal(stage, now, wall); + else for (const k of ["terminal", "crowd", "hall"]) this.beds[k].src && this.beds[k].set(0, null, 0.4); + if (scene === "room" && v.room) this.#room(v.room, now, wall); + else { + for (const k of ["tone", "hvac"]) this.beds[k].src && this.beds[k].set(0, null, 0.4); + for (const l of Object.values(this.keys)) l.src && l.set(0, null, 0.4); + } + } + + #scene(scene, now) { + const m = MIX[scene]; + const tau = this.scene ? 0.3 : 0.05; + ramp(this.bus.air.gain, m.air, now, tau); + ramp(this.bus.hall.gain, m.hall, now, tau); + ramp(this.bus.room.gain, m.room, now, tau); + ramp(this.airLp.frequency, m.lp, now, tau); + if (this.master) this.master.parameters.get("hold").setValueAtTime(m.hold, now); + this.scene = scene; + } + + /** The aircraft heard from the camera: the loudest VOICES of them by phase and distance, each kept on its voice while + * it stays near the top; softer and smoother when the clock runs many times real time. */ + #fleet(stage, camera, now, tau, cut, dts, scene, trim) { + const live = stage.fleet?.live ?? []; + const ts = stage.T * 60; + const cp = camera.position, e = camera.matrixWorldInverse.elements; + const zoom = clamp(((camera.fov || 50) / 50) ** 0.7, 0.35, 1.2); + const wet = (scene === "terminal" ? 0.32 : 1) * trim; + const cands = []; + for (const f of live) { + const spec = phaseOf(f, ts); + if (!spec) continue; + const m = f.matrix.elements, x = m[12], y = m[13], z = m[14]; + const d = Math.hypot(x - cp.x, y - cp.y, z - cp.z); + const de = Math.max(R0, d * zoom); + const g = spec.loud * (R0 / de) ** 0.78; + if (g < HEARD) continue; + const xc = e[0] * x + e[4] * y + e[8] * z + e[12], zc = e[2] * x + e[6] * y + e[10] * z + e[14]; + const k = CLASS[f.c?.track?.spec?.class] ?? [1, 1]; + cands.push({ key: f.key, spec, d, de, g: g * k[1] * wet, rate: k[0], pan: Math.sin(Math.atan2(xc, -zc)) * (0.85 - 0.35 * smooth(800, 4000, d)), phase: f.pose.phase }); + } + cands.sort((a, b) => b.g - a.g); + const near = new Map(cands.slice(0, VOICES + 2).map((c) => [c.key, c])); + const placed = new Set(); + for (const vc of this.voices) { + if (vc.key == null) continue; + const c = near.get(vc.key); + if (!c) vc.release(now); + else { + placed.add(c.key); + vc.phase = c.phase; + vc.play(c, now, tau, cut, dts); + } + } + let busy = this.voices.filter((vc) => vc.key != null).length; + for (const c of cands.slice(0, VOICES)) { + if (placed.has(c.key) || busy >= VOICES) continue; + const vc = this.voices.find((x) => x.key == null && x.free <= now) ?? this.voices.find((x) => x.key == null); + if (!vc) break; + vc.phase = c.phase; + vc.play(c, now, cut ? tau : 0.15, true, dts); + busy++; + } + } + + /** The bed, wind, rain and thunder outdoors, heard through the scene's filter indoors. */ + #air(stage, camera, now, scene, dtw) { + const w = stage.env?.weatherAt?.(stage.T) ?? {}; + const cp = camera.position; + const away = smooth(1500, 4500, Math.hypot(cp.x, cp.z)); + const B = this.beds; + const tau = 0.6; + B.apron.set(LEVEL.apron * (1 - 0.15 * away), 1, tau); + B.apron2.set(LEVEL.apron * 0.18 * (1 - away), 0.97, tau); + if (scene === "outdoor") ramp(this.airLp.frequency, 20000 - 16500 * away, now, 0.3); + const kt = w.wind_kt ?? 0; + this.weather.gust += (Math.random() - 0.5) * dtw * 0.8 - this.weather.gust * dtw * 0.25; + const gust = clamp(1 + this.weather.gust, 0.6, 1.4); + const wind = Math.max(smooth(6, 34, kt), 0.35 * away) * gust; + B.wind.set(LEVEL.wind * wind * (scene === "outdoor" ? 1 : 0.6), 0.92 + 0.12 * wind, 0.5); + const rain = clamp((w.rain ?? 0) * 0.75 + (w.drizzle ?? 0) * 0.3 + (w.ts ?? 0) * 0.25 + (w.snow ?? 0) * 0.08, 0, 1.2); + B.rain.set(LEVEL.rain * rain, 1, 0.8); + const lt = scene === "map" ? { local: 0, far: 0 } : stage.lightning ?? { local: 0, far: 0 }; + const wall = performance.now(); + const odds = Math.random() < 0.5; + if (lt.local > 0.45 && this.weather.flash <= 0.45 && wall > this.weather.thunderAt && odds) this.#thunder(true, now, wall); + else if (lt.far > 0.4 && this.weather.far <= 0.4 && wall > this.weather.thunderAt && odds) this.#thunder(false, now, wall); + this.weather.flash = lt.local; + this.weather.far = lt.far; + } + + #thunder(near, now, wall) { + const buf = near ? this.B.thunder : this.B.rumble; + if (!buf || this.shots > 10) return; + this.weather.thunderAt = wall + (near ? 14000 : 18000); + const delay = near ? 0.25 + Math.random() * 1.2 : 2 + Math.random() * 5; + const offs = near ? [0] : [0, 9.5, 19, 28.5]; + const off = offs[Math.floor(Math.random() * offs.length)]; + this.#shot(buf, this.thunderBus, { at: now + delay, offset: off, dur: near ? undefined : 9.5, gain: near ? 0.6 : 0.5 + 0.3 * Math.random(), + rate: 0.9 + Math.random() * 0.15, lp: near ? 9000 : 1600, pan: (Math.random() - 0.5) * 0.8 }); + } + + /** The terminal: its hall's air and murmur, a crowd by the gate area's people and the line, and a chime now and then. */ + #terminal(stage, now, wall) { + const g = stage.gate?.() ?? null; + const people = (g?.people ?? 0) + (g?.line ?? 0); + const busy = smooth(5, 140, people); + const B = this.beds; + B.hall.set(LEVEL.hvac * 0.8, 1, 0.5); + B.terminal.set(LEVEL.terminal * (0.55 + 0.45 * busy), 1, 1); + B.crowd.set(LEVEL.crowd * busy, 1, 1.2); + if (!this.terminal.chimeAt) this.terminal.chimeAt = wall + 15000 + Math.random() * 30000; + if (wall > this.terminal.chimeAt) { + this.terminal.chimeAt = wall + 55000 + Math.random() * 80000; + this.#shot(this.B.chime, this.hallBus, { at: now + 0.05, gain: LEVEL.chime, pan: (Math.random() - 0.5) * 0.3 }); + } + } + + /** The control-center room: its tone and air, the keys of the desk whose turn it is (a little of the room's other desks always), and clicks as cards fly. */ + #room(room, now, wall) { + const s = room.state?.() ?? {}; + const B = this.beds; + B.tone.set(LEVEL.tone, 1, 0.5); + B.hvac.set(LEVEL.hvac, 1, 0.5); + const turn = s.turn ?? null, desk = turn === "one" ? "ops" : turn; + const pane = this.panes[desk] ?? this.panes.ops; + if (turn && turn !== this.room.turn) this.room.keyAt = wall; + const typing = 0.4 + 0.6 * (0.5 + 0.5 * Math.sin((wall - this.room.keyAt) / 1900)); + for (const [k, l] of Object.entries(this.keys)) l.set(LEVEL.keys * (k === desk ? typing : 0.07), k === "crew" ? 1.04 : k === "customer" ? 0.96 : 1, 0.25); + if (s.cards != null && this.room.cards != null && s.cards > this.room.cards) this.#clicks(1 + Math.floor(Math.random() * 3), pane, now); + else if (turn && wall > this.room.clickAt) { + this.room.clickAt = wall + 2500 + Math.random() * 5000; + this.#clicks(1 + (Math.random() < 0.3), pane, now); + } + this.room.cards = s.cards ?? null; + this.room.turn = turn; + } + + #clicks(n, pane, now) { + if (!this.B.click) return; + let at = now + 0.03; + for (let i = 0; i < n; i++) { + this.#shot(this.B.click, pane, { at, gain: 0.25 + Math.random() * 0.2, rate: 0.9 + Math.random() * 0.25 }); + at += 0.09 + Math.random() * 0.25; + } + } + + /** A one-shot that disconnects itself when done; at most a dozen at once. */ + #shot(buf, out, { at, offset = 0, dur, gain = 1, rate = 1, lp = null, pan = 0 }) { + if (!buf || this.shots >= 12) return; + const ctx = this.ctx; + const s = ctx.createBufferSource(), g = ctx.createGain(), p = ctx.createStereoPanner(); + s.buffer = buf; + s.playbackRate.value = rate; + g.gain.value = gain; + p.pan.value = pan; + let head = s; + let f = null; + if (lp) { + f = ctx.createBiquadFilter(); + f.type = "lowpass"; + f.frequency.value = lp; + s.connect(f); + head = f; + } + head.connect(g).connect(p).connect(out); + this.shots++; + s.onended = () => { + this.shots--; + s.disconnect(); + f?.disconnect(); + g.disconnect(); + p.disconnect(); + }; + if (dur) { + g.gain.setValueAtTime(gain, at + dur - 1.5); + g.gain.linearRampToValueAtTime(0, at + dur); + s.start(at, offset, dur); + } else s.start(at, offset); + } +} + +/** The track segment in force at ts (seconds), as motion.js's poseAt finds it. */ +function segAt(track, ts) { + const segs = track.segs; + let lo = 0, hi = segs.length - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (ts >= segs[mid].b) lo = mid + 1; + else hi = mid; + } + return segs[lo]; +} + +/** What an aircraft sounds like at ts: its layers' levels (0..1), their pitches and its loudness at the source. */ +function phaseOf(f, ts) { + const p = f.pose, ph = p.phase, track = f.c?.track; + const seg = track ? segAt(track, ts) : null; + const tau = seg ? ts - seg.a : 0; + const v = p.speed || 0; + switch (ph) { + case "parked": + return { apu: 0.3, idle: 0.3, idleRate: 1.08, loud: 0.12 }; + case "pushback": { + const u = seg && seg.b > seg.a ? clamp((ts - seg.a) / (seg.b - seg.a), 0, 1) : 0.5; + const start = smooth(0.3, 1, u); + return { apu: 0.45, tug: 0.75, idle: 0.85 * start, idleRate: 0.55 + 0.45 * start, loud: 0.5 }; + } + case "taxi": + case "taxi_in": + case "lineup": + case "hold": + return { idle: 1, idleRate: 0.97 + 0.06 * Math.min(1, v / 12), loud: 0.5 + 0.12 * Math.min(1, v / 12) }; + case "roll": + case "climb": { + const k = smooth(0, 7, tau); + return { idle: 1 - k, roar: 0.35 + 0.65 * k, roarRate: 0.84 + 0.16 * k, idleRate: 1 + 0.2 * k, loud: 3 * (ph === "climb" ? 1 - 0.25 * smooth(60, 120, tau) : 1) }; + } + case "approach": + case "flare": + return { approach: 1, loud: 1.5 }; + case "rollout": { + const rev = smooth(0.8, 2.5, tau) * (1 - smooth(11, 19, tau)); + return { reverse: rev, approach: 1 - smooth(0, 3, tau), idle: smooth(9, 17, tau), idleRate: 1.06 - 0.1 * smooth(14, 24, tau), loud: 0.7 + 1.5 * rev }; + } + default: + return null; + } +} + +/** The viewer's sound under its timeline: a speaker button (none on a stream), on from the start with `on`, driven by + * its own frames until the viewer calls update(). Returns the AirportSound. */ +export function mountSound(viewer, bar, { on = false, stream = false } = {}) { + const sound = new AirportSound().attach(viewer).auto(); + viewer.sound = sound; + if (bar && !stream) bar.append(sound.button()); + if (on) { + sound.enable().catch(() => false).then((playing) => { + if (playing) return; + const go = () => { + removeEventListener("pointerdown", go, true); + removeEventListener("keydown", go, true); + sound.enable(); + }; + addEventListener("pointerdown", go, true); + addEventListener("keydown", go, true); + }); + } + return sound; +} diff --git a/web/audio/SOURCES.md b/web/audio/SOURCES.md new file mode 100644 index 0000000..aae46a2 --- /dev/null +++ b/web/audio/SOURCES.md @@ -0,0 +1,30 @@ +# The viewer's sound: sources + +Every recording here is from [Freesound](https://freesound.org) under **CC0 1.0** (public domain dedication, +), each licence checked on its page. No ATC, LiveATC or +announcement audio is used. The files are cut from Freesound's high-quality previews, because downloading an original +needs an account; Pixabay refuses automated access (HTTP 403), so nothing here comes from it. + +Each loop is cut, filtered (a high-pass, and for some a dynamic normaliser or soft limiter), made to loop by crossfading +its tail into its head, levelled to -23 LUFS (one-shots to a peak) and encoded as Opus in Ogg, 48 kHz. + +| File | Plays | Source | Author | Cut (s) | +|---|---|---|---|---| +| `apron.ogg` | the airport's bed outdoors | [Runway Ambience](https://freesound.org/people/clif_creates/sounds/251977/) (LAX, jets on the runway) | clif_creates | 0.3–26.4, mono | +| `idle.ogg` | engines at idle: taxi, hold, parked, engine start | [Runway of airport with airplane](https://freesound.org/people/BonnyOrbit/sounds/465440/) | BonnyOrbit | 0.5–35.6 | +| `roar.ogg` | the takeoff roar, near | [AEROJet_Plane Takeoff_OLS_SFX_Gravelly Point](https://freesound.org/people/orange.line.sound/sounds/872922/) (DCA) | orange.line.sound | 2.6–12.4 | +| `roar-far.ogg` | the takeoff roar, far | [Takeoff 5 (Distant)](https://freesound.org/people/clif_creates/sounds/251976/) (LAX) | clif_creates | 1.6–21.6, mono | +| `approach.ogg` | an airliner on approach | [Boeing777-236ER](https://freesound.org/people/freezound5/sounds/788872/) | freezound5 | 9.0–31.0, normalised | +| `reverse.ogg` | reverse thrust on the rollout | [Boeing777-236ER](https://freesound.org/people/freezound5/sounds/788872/) (after touchdown) | freezound5 | 51.0–69.0, normalised | +| `tug.ogg` | the pushback tug | [Diesel idle 1](https://freesound.org/people/C-V/sounds/565598/) | C-V | 1.0–41.0, mono | +| `terminal.ogg` | the terminal's hall | [SeaTac Airport Terminal Ambience](https://freesound.org/people/loganzsound/sounds/868379/) | loganzsound | 20.0–110.0 | +| `crowd.ogg` | a gate area's crowd, by its people | [Crowd walla](https://freesound.org/people/knufds/sounds/504906/) | knufds | 10.0–70.0 | +| `keys.ogg` | a desk's keyboard | [Typing on a keyboard](https://freesound.org/people/SamsterBirdies/sounds/489424/) | SamsterBirdies | 2.0–42.0, mono | +| `click.ogg` | a mouse click | [Mouse Click Sound](https://freesound.org/people/Pixeliota/sounds/678248/) | Pixeliota | 0–0.5, mono | +| `rain.ogg` | rain | [AMB_M_City_Rain_Heavy](https://freesound.org/people/conleec/sounds/171981/) | conleec | 5.0–50.0 | +| `wind.ogg` | wind | [Strong Wind Recorded with Soundfield Microphone Manchester 3](https://freesound.org/people/keirofinch/sounds/376533/) | keirofinch | 0.5–58.5 | +| `thunder.ogg` | thunder near the field | [Thunder Clap OWB KY](https://freesound.org/people/Dave%20Welsh/sounds/194364/) | Dave Welsh | 11.5–41.5 | +| `rumble.ogg` | distant thunder, four rumbles of 9.5 s | [Thunder DRY rolling distant low rumbles](https://freesound.org/people/TRP/sounds/717845/) | TRP | 1.5–11, 46–55.5, 76.5–86, 147–156.5 | + +Synthesized in `audio.js` (no recording): the APU's whine, the control-center room's tone and air conditioning, the +terminal's air, the chime (two bell tones through a synthetic hall), and a stand-in for any file that fails to load. diff --git a/web/audio/approach.ogg b/web/audio/approach.ogg new file mode 100644 index 0000000..3e65997 Binary files /dev/null and b/web/audio/approach.ogg differ diff --git a/web/audio/apron.ogg b/web/audio/apron.ogg new file mode 100644 index 0000000..736f4ce Binary files /dev/null and b/web/audio/apron.ogg differ diff --git a/web/audio/click.ogg b/web/audio/click.ogg new file mode 100644 index 0000000..2dd95b1 Binary files /dev/null and b/web/audio/click.ogg differ diff --git a/web/audio/crowd.ogg b/web/audio/crowd.ogg new file mode 100644 index 0000000..fd9f2aa Binary files /dev/null and b/web/audio/crowd.ogg differ diff --git a/web/audio/idle.ogg b/web/audio/idle.ogg new file mode 100644 index 0000000..3f589b9 Binary files /dev/null and b/web/audio/idle.ogg differ diff --git a/web/audio/keys.ogg b/web/audio/keys.ogg new file mode 100644 index 0000000..7453b34 Binary files /dev/null and b/web/audio/keys.ogg differ diff --git a/web/audio/rain.ogg b/web/audio/rain.ogg new file mode 100644 index 0000000..dcc8dcd Binary files /dev/null and b/web/audio/rain.ogg differ diff --git a/web/audio/reverse.ogg b/web/audio/reverse.ogg new file mode 100644 index 0000000..b418b7a Binary files /dev/null and b/web/audio/reverse.ogg differ diff --git a/web/audio/roar-far.ogg b/web/audio/roar-far.ogg new file mode 100644 index 0000000..e99d3ab Binary files /dev/null and b/web/audio/roar-far.ogg differ diff --git a/web/audio/roar.ogg b/web/audio/roar.ogg new file mode 100644 index 0000000..c79b348 Binary files /dev/null and b/web/audio/roar.ogg differ diff --git a/web/audio/rumble.ogg b/web/audio/rumble.ogg new file mode 100644 index 0000000..6127ddc Binary files /dev/null and b/web/audio/rumble.ogg differ diff --git a/web/audio/terminal.ogg b/web/audio/terminal.ogg new file mode 100644 index 0000000..ee2d49c Binary files /dev/null and b/web/audio/terminal.ogg differ diff --git a/web/audio/thunder.ogg b/web/audio/thunder.ogg new file mode 100644 index 0000000..2ff484c Binary files /dev/null and b/web/audio/thunder.ogg differ diff --git a/web/audio/tug.ogg b/web/audio/tug.ogg new file mode 100644 index 0000000..029febf Binary files /dev/null and b/web/audio/tug.ogg differ diff --git a/web/audio/wind.ogg b/web/audio/wind.ogg new file mode 100644 index 0000000..d07ed09 Binary files /dev/null and b/web/audio/wind.ogg differ diff --git a/web/center/center.js b/web/center/center.js index 28ead47..d7e04e1 100644 --- a/web/center/center.js +++ b/web/center/center.js @@ -11,6 +11,7 @@ import { ROOM, buildHall } from "./hall.js"; import { buildPods } from "./pods.js"; import { THREE_DESKS, BASE as VIEWS, compose, framed } from "./shots.js"; import { WALL, buildWall } from "./wall.js"; +import { freed } from "../gc.js"; const CSS = `.cc-caption { position: absolute; z-index: 2; top: 12px; left: 50%; transform: translateX(-50%); padding: 3px 10px; border-radius: 4px; font: 500 11.5px/1.4 Inter, "Helvetica Neue", Arial, sans-serif; color: rgba(225,233,240,0.82); background: rgba(10,14,20,0.55); pointer-events: none; white-space: nowrap; } @@ -30,6 +31,7 @@ export class ControlCenter { this.cards = 0; this.seen = 0; this.frameMs = []; + this.stagger = false; this.rig = { kind: "preset", name: "room", set: false }; this.renderer = new THREE.WebGLRenderer({ antialias: true, preserveDrawingBuffer: manual, powerPreference: "high-performance" }); this.renderer.setPixelRatio(pixelRatio); @@ -149,6 +151,24 @@ export class ControlCenter { return this.rig.name; } + /** A live day's episode grown since the room was built: the wall, the desks' pods and the cards between them built + * again over the same hall, the camera kept. */ + async reload(P, ep) { + await this.ready; + this.P = P; + this.ep = ep; + const old = [this.screen.group, this.pods.group, this.flying.group]; + this.screen = buildWall(P, ep); + this.pods = buildPods(P, ep); + this.flying = buildCards(P, ep, this.pods.pods); + this.scene.remove(...old); + this.scene.add(this.screen.group, this.pods.group, this.flying.group); + freed(old, [this.scene]); + this.flying.mark(this.rig.kind === "shot" ? this.rig.ref : null); + await Promise.race([this.screen.ready, new Promise((r) => setTimeout(r, 3000))]); + this.seen = performance.now(); + } + state() { const ms = this.frameMs.slice().sort((a, b) => a - b); const shot = this.rig.kind === "shot" ? { from: this.rig.from, to: this.rig.to, at: this.rig.at, fov: this.rig.fov, desk: this.rig.desk, ref: this.rig.ref } : null; @@ -196,7 +216,7 @@ export class ControlCenter { #frame(dt, force = false) { const t0 = performance.now(), T = this.T; - this.screen.update(T, { force }); + this.screen.update(T, { force, one: this.stagger && !force }); this.pods.update(T, this.wall); this.#aim(); this.cards = this.flying.update(T, this.camera); diff --git a/web/center/wall.js b/web/center/wall.js index 02ce7ca..2c14b04 100644 --- a/web/center/wall.js +++ b/web/center/wall.js @@ -78,7 +78,10 @@ function trace(g, proj, lines, close) { } } -async function geography() { +let GEOGRAPHY = null; +const geography = () => (GEOGRAPHY ??= coastlines()); + +async function coastlines() { const [topo, states, land] = await Promise.all([import(TOPO), fetch(STATES).then((r) => r.json()), fetch(LAND).then((r) => r.json())]); const near = (ring) => ring.every(([lon, lat]) => lon > -175 && lon < -30 && lat > 0 && lat < 80); return { @@ -610,24 +613,24 @@ export function buildWall(P, ep) { group, ready, span: [-WALL.w / 2, -WALL.w / 2 + w], - /** Redraw what changed at minute t; `now` (wall ms) throttles the map when the clock runs fast, `force` skips that. */ - update(t, { now = performance.now(), force = false } = {}) { + /** Redraw what changed at minute t; `now` (wall ms) throttles the map when the clock runs fast, `force` skips that; + * `one` redraws one window a frame at most, the rest on the frames after, so no frame uploads them all. */ + update(t, { now = performance.now(), force = false, one = false } = {}) { const flights = flightsAt(P, t), air = positionsAt(P, flights, t), m = Math.floor(t); + let budget = one ? 1 : Infinity; + const redraw = (win, key, ...args) => win && win.key !== key && budget-- > 0 && win.update(key, ...args); if (force || now - lastMap > 110 || windows.map.key == null) { const k = `${Math.floor(t * 3)}|${!!geo.data}`; - if (windows.map.key !== k) { - windows.map.update(k, t, air, rings(P, t)); - lastMap = now; - } + if (redraw(windows.map, k, t, air, rings(P, t))) lastMap = now; } const fr = radar?.at(t); const okA = fr && image(fr.a.url).ok, okB = fr && image(fr.b.url).ok; - windows.sky.update(radar ? `${fr?.a.url}|${fr?.b.url}|${Math.round((fr?.f ?? 0) * 8)}|${okA}|${okB}|${!!geo.data}` : metarAt(t)?.id ?? "none", t); - windows.board.update(`${m}`, t, flights); - windows.faa.update(`${m}`, t); - windows.ops.update(`${m}|${air.length}`, t, air.length); - windows.pax?.update(`${m}`, t); - drawLed(t); + redraw(windows.sky, radar ? `${fr?.a.url}|${fr?.b.url}|${Math.round((fr?.f ?? 0) * 8)}|${okA}|${okB}|${!!geo.data}` : metarAt(t)?.id ?? "none", t); + redraw(windows.board, `${m}`, t, flights); + redraw(windows.faa, `${m}`, t); + redraw(windows.ops, `${m}|${air.length}`, t, air.length); + redraw(windows.pax, `${m}`, t); + if (budget > 0) drawLed(t); }, /** Resolves once the radar frames shown at t are loaded, so a recorded frame has them. */ loaded(t) { diff --git a/web/director.js b/web/director.js new file mode 100644 index 0000000..98d9fb3 --- /dev/null +++ b/web/director.js @@ -0,0 +1,379 @@ +// The live director: cuts between the control-center room, the airport and the map on the day's events as the clock +// reaches them, as record.py's Center reads a day: the room when a desk asks, answers or decides, the gate of a flight it +// cancels, the terminal at a gate it holds, the tower as an FAA program starts; between them the airport's own +// movements (a landing, a takeoff, an aircraft followed, the terminal's busiest gate, a crew), the room while a desk +// works and now and then the map, each shot held between its least and most wall seconds. Airport shots slow the clock +// (record.py's FAST) and the room and the map let it catch up. +import { HUB_NAMES, cities, hhmm, programsAt } from "./model.js"; +import { unitName } from "./room.js"; +import { homeOf } from "./stage/cameras.js"; + +const DESK = { ops: "the ops controller", crew: "the crew scheduler", customer: "the customer desk", one: "the desk" }; +const ASKS = { CANCEL: "to cancel", CANCEL_PAIR: "to cancel the round trip of", HOLD: "to hold", COVER: "to find a crew for", + PROTECT: "not to cancel", SUGGEST_CANCEL: "to cancel early" }; +const ASK_PRIO = { CANCEL: 82, CANCEL_PAIR: 82, HOLD: 80, SUGGEST_CANCEL: 74, PROTECT: 72, COVER: 66, TEXT: 34 }; +const DECIDE_PRIO = { cancel: 78, cancel_pair: 78, hold: 70, rebook: 52, hotel: 52, assign: 50, call_reserve: 50, deadhead: 46, swap: 46, + claim: 42, swap_slots: 40, release: 36, rest: 34 }; +/** The clock's fastest rate for each camera, in minutes of the day a second (record.py FAST); the room and the map none. */ +const SLOW = { landing: 0.1, takeoff: 0.1, gate: 0.12, terminal: 0.12, crew: 0.12, follow: 0.3, tower: 0.25, harbour: 0.3, threshold: 0.2 }; +const LEN = { ask: [5, 12], answer: [4, 8], decide: [4, 8], cancelled: [4, 7], held: [5, 9], faa: [6, 10], landing: [5, 8], + takeoff: [5, 8], follow: [6, 14], terminal: [6, 13], crew: [6, 11], tower: [5, 10], harbour: [5, 9], gate: [5, 9], map: [5, 8], + work: [6, 12], room: [6, 12] }; +const REPLY = 5.5; +const MAP_EVERY = 150; +const DAWN = -2; +/** Wall seconds behind where the clock should be past which the director keeps to the room and the map, where it may + * catch up. */ +const LAG = 45; + +const upper = (s) => (s ? s[0].toUpperCase() + s.slice(1) : s); +const people = (n, what = "passengers") => `${n} ${n === 1 ? what.replace(/s$/, "") : what}`; + +export class Director { + /** `page` is app.js episodePage's live handle; `pacer` takes the slowest rate a shot wants (cap). */ + constructor(page, { pacer = null, caption = () => {}, log = false } = {}) { + this.page = page; + this.pacer = pacer; + this.caption = caption; + this.log = log; + this.used = new Set(); + this.cur = null; + this.lastMap = -Infinity; + this.recent = []; + this.shots = []; + this.index(); + } + + /** The day's beats from what is decided so far, again after each update; those already cut stay used. */ + index() { + const { P, ep } = this.page, occ = ep.occ ?? {}, hub = P.task.hub; + const traffic = this.page.traffic(); + this.city = cities(P.task, traffic); + this.where = HUB_NAMES[hub] ?? hub; + this.legs = new Map(); + for (const a of traffic?.aircraft ?? []) { + if (!a.task) continue; + for (const v of a.visits) for (const g of [v.arr, v.dep]) if (g) this.legs.set(`${g.flight}|${g.kind}`, { a, v, g }); + } + const beats = []; + const one = occ.mode === "one"; + const desk = (d) => (one ? "the desk" : DESK[d] ?? d); + for (const r of occ.requests ?? []) { + if (r.shown == null) continue; + const typed = r.type !== "TEXT", said = this.quote(r.reply_text); + const ask = r.type === "TEXT" ? `${upper(desk(r.sender))} messages ${desk(r.receiver)}${this.quote(r.text) ? `: “${this.quote(r.text)}”` : ""}` + : `${upper(desk(r.sender))} asks ${desk(r.receiver)} ${ASKS[r.type] ?? r.type.toLowerCase()} ${this.trip(r.flight)}${r.pax && r.type === "HOLD" ? ` for ${people(r.pax, "connecting passengers")}` : r.pax ? `, ${people(r.pax)} booked` : ""}`; + const answer = r.answered == null ? "" : `${upper(desk(r.receiver))} ${r.reply === "accept" ? "accepts" : "declines"}${said ? `: “${said}”` : ""}`; + const end = r.answered != null && r.answered - r.shown < 9 ? r.answered + REPLY : r.shown + 7; + beats.push({ id: `ask:${r.id}`, kind: "ask", at: r.shown - 0.4, end, prio: ASK_PRIO[r.type] ?? 50, camera: "room-desks", + spec: { desk: r.sender, ref: r.id, t0: r.shown - 0.5, t1: end }, kicker: `${hhmm(r.shown)} · The control center`, caption: ask, + subline: (t) => (r.answered != null && t >= r.answered ? answer : ""), flight: r.flight }); + if (typed && r.answered != null && r.answered - r.shown >= 9) { + beats.push({ id: `answer:${r.id}`, kind: "answer", at: r.answered - 0.4, end: r.answered + REPLY, prio: (ASK_PRIO[r.type] ?? 50) - 12, camera: "room-desks", + spec: { desk: r.receiver, ref: r.id, t0: r.answered - 0.5, t1: r.answered + REPLY }, kicker: `${hhmm(r.answered)} · The control center`, + caption: `${upper(desk(r.receiver))} ${r.reply === "accept" ? "accepts" : "declines"} ${r.id}: ${ASKS[r.type] ?? ""} ${this.trip(r.flight)}`.replace(/ {2}/g, " "), + subline: () => (said ? `“${said}”` : "") }); + } + } + const groups = new Map(); + for (const d of occ.decisions ?? []) { + if (d.shown == null || DECIDE_PRIO[d.action] == null) continue; + const k = `${d.desk}|${Math.floor(d.shown / 1.5)}|${d.action}`; + groups.set(k, [...(groups.get(k) ?? []), d]); + } + for (const ds of groups.values()) { + const d = ds[0], at = d.shown - 0.3, asked = d.ref != null; + const bulk = ds.length >= 3 && ["cancel", "cancel_pair"].includes(d.action); + const beat = { id: `decide:${d.desk}:${d.shown}:${d.action}`, kind: "decide", at, end: at + 4, prio: DECIDE_PRIO[d.action] - (asked ? 40 : 0), + camera: bulk ? "room-wall" : "room", spec: { desk: d.desk, t0: at, t1: at + 4 }, kicker: `${hhmm(d.shown)} · The control center`, + caption: `${upper(desk(d.desk))} ${this.decision(ds)}`, subline: () => (asked ? `As ${d.ref} asked` : "") }; + beats.push(beat); + const after = this.after(d, asked ? [beat.id, `ask:${d.ref}`, `answer:${d.ref}`] : [beat.id]); + if (after) beats.push(after); + } + for (const n of P.news) { + const data = n.data ?? {}; + if (n.kind !== "advisory" || data.element !== hub || !["gs", "gdp"].includes(data.kind)) continue; + const period = data[data.kind === "gs" ? "gs_period" : "program_period"]; + if (!period) continue; + const start = Math.max(n.at, period[0]), what = this.announce(data); + beats.push({ id: `faa:${n.id}`, kind: "faa", at: start - 0.5, end: start + 3, prio: 90, camera: "tower", spec: {}, + kicker: `${hhmm(start)} · ${what[0]}`, caption: what[1], subline: () => (period[0] > n.at + 2 ? `Issued at ${hhmm(n.at)}` : "") }); + if (period[0] > n.at + 20) { + beats.push({ id: `faa-news:${n.id}`, kind: "map", at: n.at, end: n.at + 6, prio: 64, camera: "network", spec: {}, + kicker: `${hhmm(n.at)} · The FAA`, caption: `From ${hhmm(period[0])}, ${what[1][0].toLowerCase()}${what[1].slice(1)}`, subline: () => "" }); + } + } + this.beats = beats.sort((a, b) => a.at - b.at); + this.events = this.page.stage?.events?.() ?? []; + } + + /** What a decision plays out as at the airport: the cancelled flight at its gate, the held one in the terminal. */ + after(d, parents) { + const s = this.page.P.sched.get(d.flight); + if (!s) return null; + const hub = this.page.P.task.hub, kind = s.orig === hub ? "dep" : s.dest === hub ? "arr" : null; + const leg = kind && this.legs.get(`${s.flight}|${kind}`); + if (!leg) return null; + const at = d.shown + 0.6; + if (d.action === "cancel" || d.action === "cancel_pair") { + return { id: `cancelled:${d.flight}:${d.shown}`, kind: "cancelled", after: parents, at, end: at + 2, prio: 74, camera: "gate", outdoor: true, + spec: { key: leg.a.key }, key: leg.a.key, kicker: hhmm(at), caption: `${s.flight} ${kind === "dep" ? "to" : "from"} ${this.cityOf(kind === "dep" ? s.dest : s.orig)} stays at ${this.gate(leg.v.stand)}`, + subline: () => `Cancelled by ${DESK[d.desk] ?? "the desk"}` }; + } + if (d.action === "hold" && kind === "dep") { + const lounge = this.page.stage?.gates?.()?.find((g) => g.stand === leg.v.stand); + const p = this.page.traffic()?.passengers?.tags?.[s.flight]; + const sub = () => [d.detail ? `Held until ${d.detail}` : "Held", p?.conn ? `for ${people(p.conn, "connecting passengers")}` : ""].filter(Boolean).join(" "); + return { id: `held:${d.flight}:${d.shown}`, kind: "held", after: parents, at, end: at + 3, prio: 72, camera: lounge ? "terminal" : "gate", + outdoor: !lounge, spec: lounge ? { stand: leg.v.stand, flight: s.flight } : { key: leg.a.key }, kicker: hhmm(at), + caption: `${s.flight} to ${this.cityOf(s.dest)} waits at ${this.gate(leg.v.stand)}`, subline: sub }; + } + return null; + } + + /** Called each frame with the day's clock: cuts when a beat comes due past the shot's least length, or the shot runs out. */ + tick(t, wall, speed) { + if (this.busy) return; + const cur = this.cur, held = cur ? wall - cur.start : Infinity; + const lead = Math.max(0.25, speed * 1.2); + const mine = (b) => !!cur && !!b.after?.includes(cur.beat.id); + for (const b of this.beats) if (!this.used.has(b.id) && b.at < t - 3 && !mine(b)) this.used.add(b.id); + const due = this.beats.filter((b) => !this.used.has(b.id) && (mine(b) || (b.at <= t + lead && b.at >= t - 3)) && this.ready(b, t, wall)); + const best = due.sort((a, b) => b.prio - a.prio || a.at - b.at)[0]; + if (cur) this.caption(cur.beat, t); + const [lo, hi] = cur ? LEN[cur.beat.kind] ?? [5, 10] : [0, 0]; + const over = cur && (held >= hi || (cur.beat.end != null && t > cur.beat.end + 0.5 && held >= lo) || (cur.check && !cur.check(t))); + if (best && (!cur || (held >= lo && (best.prio > cur.beat.prio || mine(best) || over)))) return this.cut(best, t, wall); + if (!cur || over) return this.cut(this.filler(t, wall, speed), t, wall); + } + + /** A follow-up comes once its decision is made, after the shot of it (or of the request it answers) has run its + * least length; an outdoor one only in light. */ + ready(b, t, wall) { + if (b.outdoor && !this.light(t)) return false; + if (!b.after) return true; + if (b.at > t + 0.2) return false; + return !b.after.includes(this.cur?.beat.id) || wall - this.cur.start >= (LEN[this.cur.beat.kind]?.[0] ?? 4); + } + + light(t) { + const env = this.page.stage?.env; + return !env || env.sunAt(t).el >= DAWN; + } + + /** Between beats: the airport's movements in light, the room while a desk works, the terminal, the map now and then. */ + filler(t, wall, speed) { + const { stage, room } = this.page, light = this.light(t), late = this.pacer ? this.pacer.behind() > LAG : false, end = !!this.pacer?.ending; + const options = []; + const add = (beat, w) => w > 0 && options.push([beat, w * (beat.camera === this.recent[0] ? 0.02 : this.recent.includes(beat.camera) ? 0.3 : 1)]); + const up = this.airborne(t); + if (stage && (end || wall - this.lastMap > MAP_EVERY) && up) add({ kind: "map", camera: "network", spec: {}, prio: 15, kicker: `${hhmm(t)} · The network`, caption: up }, end ? 4 : 6); + const active = room?.pods?.active(t), turn = active && (this.page.ep.occ?.turns ?? []).find((u) => u.from <= t && t < u.to); + if (room && active) add({ kind: "work", camera: "room-desks", spec: { desk: active, t0: t, t1: t + 6 }, prio: 30, kicker: `${hhmm(t)} · The control center`, + caption: `${upper(this.page.ep.occ?.mode === "one" ? "the desk" : DESK[active])} works through the ${hhmm(turn?.t ?? t)} watch`, subline: turn?.calls ? `${turn.calls} calls to its tools this turn` : "" }, late ? 8 : 4); + if (room && !active) add({ kind: "room", camera: "room", spec: { t0: t, t1: t + 6 }, prio: 12, kicker: `${hhmm(t)} · The control center`, caption: "" }, end ? 0.5 : late ? 5 : 1); + if (stage && !late) { + const soon = this.events.filter((e) => e.t > t + Math.max(0.15, speed * 1.5) && e.t < t + Math.max(0.7, speed * 3) && this.page.stage.plan.tracks.get(e.key)?.task); + const land = soon.find((e) => e.kind === "landing"), off = soon.find((e) => e.kind === "takeoff"); + if (light && land) add(this.movement(land, "landing"), 7); + if (light && off) add(this.movement(off, "takeoff"), 6); + const moving = this.moving(t); + if (light && moving) add({ kind: "follow", camera: "follow", spec: { key: moving }, prio: 20, kicker: hhmm(t), caption: this.tagOf(moving, t) ?? "", check: (u) => this.phase(moving, u) !== "parked" && this.phase(moving, u) !== "absent" }, 4); + if (this.page.terminal && this.crowd(t) >= 12) add({ kind: "terminal", camera: "terminal", spec: { auto: true }, prio: 18, kicker: `${hhmm(t)} · ${this.terminalName()}`, caption: "" }, light ? 3 : 6); + if (light && this.page.crews) add({ kind: "crew", camera: "crew", spec: {}, prio: 16, kicker: hhmm(t), caption: "" }, 2); + if (light) add({ kind: "harbour", camera: "harbour", spec: {}, prio: 10, kicker: `${this.where} · ${hhmm(t)}`, caption: "" }, 1.5); + add({ kind: "tower", camera: "tower", spec: {}, prio: 10, kicker: `${this.where} · ${hhmm(t)}`, caption: "" }, light ? 2 : 3); + } + if (!options.length) add({ kind: room ? "room" : "map", camera: room ? "room" : "network", spec: { t0: t, t1: t + 6 }, prio: 10, kicker: hhmm(t), caption: "" }, 1); + const total = options.reduce((s, [, w]) => s + w, 0); + let r = this.random() * total; + for (const [beat, w] of options) if ((r -= w) <= 0) return { id: `fill:${beat.kind}:${wall.toFixed(1)}`, ...beat }; + return { id: `fill:${wall}`, ...options.at(-1)[0] }; + } + + /** People in the terminal's busiest gate area at t. */ + crowd(t) { + const S = this.page.stage, k = S.terminal?.busiest(t); + return k == null || !S.pax ? 0 : S.pax.at(k, t); + } + + random() { + this.seed = ((this.seed ?? 0x2f6b) * 1103515245 + 12345) & 0x7fffffff; + return this.seed / 0x7fffffff; + } + + movement(e, kind) { + const tag = this.tagOf(e.key, e.t); + return { kind, camera: kind, key: e.key, at: e.t, spec: { key: e.key, at: e.t }, prio: 30, end: e.t + (kind === "landing" ? 0.4 : 0.3), kicker: hhmm(e.t), + caption: tag ? `${tag.split(" · ")[0]} ${kind === "landing" ? "lands" : "takes off"}` : `${e.flight} ${kind === "landing" ? "lands" : "takes off"}` }; + } + + /** A task aircraft moving on the ground now or about to, the soonest first. */ + moving(t) { + const stage = this.page.stage; + for (const e of this.events) { + if (e.t < t - 4 || e.t > t + 6 || !stage.plan.tracks.get(e.key)?.task) continue; + const p = this.phase(e.key, t); + if (["pushback", "taxi", "taxi_in", "hold", "rollout"].includes(p)) return e.key; + } + return null; + } + + phase(key, t) { + return this.page.stage?.pose(t, key)?.phase ?? "absent"; + } + + tagOf(key, t) { + const tr = this.page.stage?.plan.tracks.get(key); + const legs = tr?.visits.flatMap((v) => [v.arr, v.dep]).filter((l) => l?.tag) ?? []; + const near = legs.sort((a, b) => Math.abs((a.kind === "dep" ? a.out ?? a.sched : a.on ?? a.sched) - t) - Math.abs((b.kind === "dep" ? b.out ?? b.sched : b.on ?? b.sched) - t))[0]; + return near?.tag ?? null; + } + + async cut(beat, t, wall) { + this.used.add(beat.id); + const prev = this.cur; + this.cur = { beat, start: wall, at: t, check: beat.check ?? null }; + this.recent = [beat.camera, ...this.recent].slice(0, 3); + if (beat.camera === "network") this.lastMap = wall; + if (this.pacer) this.pacer.cut(SLOW[beat.camera] ?? Infinity, !SLOW[beat.camera]); + this.shots.push({ id: beat.id, kind: beat.kind, camera: beat.camera, prio: beat.prio, at: t, wall, caption: beat.caption }); + if (this.shots.length > 400) this.shots.splice(0, 100); + if (this.log) console.info(`cut ${hhmm(t)} ${beat.camera} ${beat.kind} ${beat.caption ?? ""}`); + this.busy = true; + try { + await this.apply(beat, t); + } catch (e) { + console.warn("the director's shot failed:", beat.id, e); + if (prev?.beat.camera !== "tower") await this.page.setView(this.page.stage ? "tower" : "network").catch(() => {}); + } finally { + this.busy = false; + } + this.caption(beat, t); + } + + async apply(beat, t) { + const { setView, stage, room } = this.page; + if (beat.camera.startsWith("room")) { + if (!room) return setView("network"); + await setView(beat.camera); + return room.shot({ name: beat.camera, ...beat.spec }); + } + if (beat.camera === "network") return setView("network"); + if (!stage) return setView("network"); + if (beat.camera === "landing" || beat.camera === "takeoff") { + const spec = this.scripted(beat.camera, beat.spec.key, beat.spec.at, t); + if (!spec) return setView("tower"); + await setView("tower"); + return stage.shot(spec); + } + if (beat.camera === "gate" && beat.spec.key && !["parked", "pushback"].includes(this.phase(beat.spec.key, t + 1))) return setView("tower"); + return setView(beat.camera, beat.spec); + } + + /** A landing or takeoff from beside the runway ahead of the aircraft, as record.py films one (FILM_JS camera()). */ + scripted(kind, key, at, t) { + const S = this.page.stage, p = S.pose(at, key); + if (!p || p.phase === "absent") return null; + const D = Math.PI / 180, dir = (h) => [Math.sin(h * D), Math.cos(h * D)], add = (a, b, k = 1) => [a[0] + b[0] * k, a[1] + b[1] * k]; + const home = homeOf(S.data).centre, l = dir(p.heading - 90); + const sd = (p.x - home[0]) * l[0] + (p.y - home[1]) * l[1] >= 0 ? l : [-l[0], -l[1]]; + const from = add(add([p.x, p.y], dir(p.heading), kind === "takeoff" ? 260 : 380), sd, 165); + return { name: kind, from: [from[0], from[1], 5], at: key, fov: 40, frame: 65, t0: t, t1: at + 1 }; + } + + /** The shots of the day's best moments to replay: its biggest beats, those that play out at the airport first and + * no more than three in the room alone, apart in time, each with its airport shot, in time order. */ + highlights(n = 6) { + const after = (b) => this.beats.find((x) => x.after?.includes(b.id) && (!x.outdoor || this.light(x.at))); + const score = (b) => b.prio + (after(b) ? 15 : 0) + (b.kind === "faa" ? 10 : 0); + const picked = []; + let room = 0; + for (const b of this.beats.filter((b) => !b.after && b.prio >= 60 && b.at > this.page.P.lo).sort((a, b) => score(b) - score(a) || a.at - b.at)) { + const f = after(b); + if (picked.some((p) => Math.abs(p.at - b.at) < 45) || (!f && b.camera.startsWith("room") && room >= 3)) continue; + room += !f && b.camera.startsWith("room") ? 1 : 0; + picked.push(b); + if (f) picked.push({ ...f, follows: true }); + if (picked.filter((p) => !p.follows).length >= n) break; + } + return picked.sort((a, b) => a.at - b.at || (a.follows ? 1 : -1)); + } + + decision(ds) { + const d = ds[0], f = (x) => this.trip(x.flight), unit = unitName(d.detail); + const list = ds.length > 1 ? `${ds.slice(0, 2).map((x) => this.flightNo(x.flight)).join(" and ")}${ds.length > 2 ? ` and ${ds.length - 2} more` : ""}` : f(d); + switch (d.action) { + case "cancel": return `cancels ${list}`; + case "cancel_pair": return `cancels ${list} and ${ds.length > 1 ? "their returns" : "its return"}`; + case "hold": return `holds ${list}${ds.length === 1 && d.detail ? ` until ${d.detail}` : ""}`; + case "claim": return `takes a slot for ${list}`; + case "swap_slots": return `swaps slots on ${list}`; + case "release": return `lets ${list} go when ${ds.length > 1 ? "they can" : "it can"}`; + case "assign": return `puts ${unit} on ${f(d)}`; + case "call_reserve": return `calls ${unit} for ${f(d)}`; + case "deadhead": return `flies ${unit} as passengers on ${f(d)}`; + case "swap": return `swaps ${unit} onto ${f(d)}`; + case "rest": return `sends ${unit} to rest`; + case "rebook": return `rebooks ${ds.reduce((s, x) => s + (x.pax ?? 0), 0) || "passengers"} passengers off ${list}`; + case "hotel": return `moves ${ds.reduce((s, x) => s + (x.pax ?? 0), 0) || "the"} passengers off ${list} to a later day`; + default: return `${d.action} ${list}`; + } + } + + announce(d) { + if (d.kind === "gs") { + const who = /JBU/.test(d.flt_incl ?? "") ? "JetBlue flights" : "flights"; + return ["Ground stop", `The FAA stops ${who} to ${this.where} at their departure airports until ${hhmm(d.gs_period[1])}`]; + } + const rate = d.program_rate ?? [], cap = rate.length ? (Math.min(...rate) === Math.max(...rate) ? rate[0] : `${Math.min(...rate)}–${Math.max(...rate)}`) : "fewer"; + return ["Ground delay program", `The FAA holds arrivals at ${this.where} to ${cap} an hour until ${hhmm(d.program_period[1])}`]; + } + + /** The JetBlue flights in the air at t in words, or "" with none. */ + airborne(t) { + const up = this.page.P.ep.executed.filter((e) => e.status === "flown" && e.dep != null && e.dep <= t && t < e.arr).map((e) => this.page.P.sched.get(e.id)).filter(Boolean); + if (!up.length) return ""; + const bound = up.filter((s) => s.dest === this.page.P.task.hub).length, { gs, gdp } = programsAt(this.page.P, t); + return `${up.length} JetBlue flight${up.length === 1 ? "" : "s"} in the air${bound ? `, ${bound} of them bound for ${this.where}` : ""}${gs ? " · ground stop" : gdp ? " · delay program" : ""}`; + } + + trip(fid) { + const s = this.page.P.sched.get(fid); + if (!s) return fid == null ? "" : `f${fid}`; + const hub = this.page.P.task.hub; + if (s.orig === hub) return `${s.flight} to ${this.cityOf(s.dest)}`; + if (s.dest === hub) return `${s.flight} from ${this.cityOf(s.orig)}`; + return `${s.flight} ${this.cityOf(s.orig)}–${this.cityOf(s.dest)}`; + } + + flightNo(fid) { + return this.page.P.sched.get(fid)?.flight ?? `f${fid}`; + } + + cityOf(code) { + return this.city.get(code) ?? code; + } + + gate(stand) { + return stand ? `gate ${String(stand).split("-").at(-1)}` : "its gate"; + } + + terminalName() { + return { JFK: "JetBlue's Terminal 5", BOS: "JetBlue's Terminal C" }[this.page.P.task.hub] ?? "The terminal"; + } + + /** A desk's words as a short quote: their first clause with flights by number, or "" when it doesn't read as words. */ + quote(text) { + const words = String(text ?? "").replace(/\bf(\d{1,4})\b/g, (m, id) => this.page.P.sched.get(Number(id))?.flight ?? m).trim(); + let first = words.split(/(?<=[.;])\s|,\s(?=and\b)/)[0].replace(/[.;]$/, ""); + if (first.length > 84) first = first.lastIndexOf(", ", 84) > 0 ? first.slice(0, first.lastIndexOf(", ", 84)) : ""; + return !first || /[a-z]\d|\d[a-z]{2}|\d{2}:\d{2}\+1|\b[A-Z]\d{3,}/.test(first) ? "" : first; + } + + state() { + return { shot: this.cur ? { id: this.cur.beat.id, kind: this.cur.beat.kind, camera: this.cur.beat.camera, held: this.cur.start } : null, + beats: this.beats.length, used: this.used.size, shots: this.shots.slice(-30) }; + } +} diff --git a/web/gc.js b/web/gc.js new file mode 100644 index 0000000..ac134ae --- /dev/null +++ b/web/gc.js @@ -0,0 +1,24 @@ +// Freeing a 3D view's parts as a live day replaces them, keeping what the parts that stay share with them. + +/** The geometries, materials and textures under `objs`, and the instanced meshes, whose buffers are their own. */ +function owned(objs) { + const out = new Set(); + for (const root of objs) { + root.traverse((o) => { + if (o.isInstancedMesh) out.add(o); + if (o.geometry) out.add(o.geometry); + for (const m of [].concat(o.material || [])) { + out.add(m); + for (const v of Object.values(m)) if (v?.isTexture) out.add(v); + for (const u of Object.values(m.uniforms || {})) if (u?.value?.isTexture) out.add(u.value); + } + }); + } + return out; +} + +/** Free what `old` holds that nothing under `keep` uses (module-level kits and materials are shared across builds). */ +export function freed(old, keep) { + const kept = owned(keep); + for (const r of owned(old)) if (!kept.has(r)) r.dispose?.(); +} diff --git a/web/live.css b/web/live.css new file mode 100644 index 0000000..dcffe12 --- /dev/null +++ b/web/live.css @@ -0,0 +1,106 @@ +/* The viewer live (live.js): the live panel, the director's captions, the day's score and next-storm cards, and + html.stream's 1920x1080 broadcast layout with nothing to click. */ +html.live { --tl: 0px; } +html.live .timeline, html.live .expert, html.live .follow-chip button, html.live .gate-chip button { display: none !important; } +html.live .view.corner { cursor: default; } +html.live .view.corner .inset-caption { pointer-events: none; } + +.live-panel { flex: none; display: flex; flex-direction: column; gap: 7px; border-color: rgba(255, 106, 92, 0.35); } +.lp-top { display: flex; align-items: center; gap: 9px; font-size: 12px; color: var(--muted); } +.lp-live { display: inline-flex; align-items: center; gap: 6px; padding: 2px 9px 2px 7px; border-radius: 4px; background: #d8312a; color: #fff; + font-size: 11px; font-weight: 800; letter-spacing: 0.1em; text-transform: uppercase; } +.lp-live::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: #fff; animation: lp-pulse 1.6s ease-in-out infinite; } +.lp-live.off { background: #6b7780; } +.lp-live.off::before { animation: none; opacity: 0.6; } +@keyframes lp-pulse { 50% { opacity: 0.25; } } +.lp-now { font-variant-numeric: tabular-nums; font-weight: 600; color: #dce7ec; } +.lp-badge { margin-left: auto; padding: 1px 8px; border-radius: 4px; background: rgba(90, 168, 255, 0.2); border: 1px solid rgba(90, 168, 255, 0.45); + color: #cfe3ff; font-size: 11px; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; } +.live-panel h2 { margin: 0; font-size: 17px; font-weight: 700; } +.live-panel h2 small { margin-left: 8px; font-size: 12.5px; font-weight: 500; color: var(--muted); } +.lp-storm { margin: -4px 0 0; font-size: 12.5px; color: #c3d3db; } +.lp-watch b { display: block; font-size: 14px; font-weight: 650; } +.lp-watch span { display: block; font-size: 11.5px; color: var(--muted); min-height: 1.3em; } +.lp-desks { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 4px; } +.lp-desks li { display: grid; grid-template-columns: 70px 1fr auto; gap: 8px; align-items: baseline; padding: 4px 8px; border-radius: 5px; + background: rgba(255, 255, 255, 0.06); font-size: 12.5px; } +.lp-desks li b { font-size: 11px; font-weight: 750; letter-spacing: 0.05em; text-transform: uppercase; color: #9fc2d2; } +.lp-desks li span { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lp-desks li time { font-size: 11px; color: var(--faint); white-space: nowrap; } +.lp-kpis { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; } +.lp-kpis b { display: block; font-size: 20px; font-weight: 700; font-variant-numeric: tabular-nums; } +.lp-kpis span { display: block; font-size: 11px; color: var(--muted); } +.lp-grade { display: flex; align-items: baseline; gap: 9px; padding: 6px 9px; border-radius: 6px; background: rgba(90, 208, 143, 0.14); + border: 1px solid rgba(90, 208, 143, 0.4); } +.lp-grade b { font-size: 22px; } +.lp-grade em { font-style: normal; font-size: 11px; color: var(--muted); } + +.live-cap { position: absolute; z-index: 9; left: 50%; bottom: 40px; transform: translateX(-50%); max-width: min(1000px, calc(100% - 980px)); + min-width: 360px; width: max-content; padding: 10px 24px 12px; border-radius: 12px; text-align: center; background: rgba(10, 22, 32, 0.82); + border: 1px solid rgba(190, 214, 224, 0.18); box-shadow: 0 8px 30px rgba(4, 16, 26, 0.4); backdrop-filter: blur(8px); pointer-events: none; } +.live-cap[hidden] { display: none; } +.live-cap small { display: block; margin-bottom: 3px; font-size: 12px; font-weight: 650; letter-spacing: 0.08em; text-transform: uppercase; color: #9fc2d2; } +.live-cap b { display: block; font-size: 22px; font-weight: 600; line-height: 1.25; color: #fff; } +.live-cap span { display: block; margin-top: 4px; font-size: 15.5px; line-height: 1.3; color: #c9dbe3; } +.live-cap span:empty { display: none; } + +.live-over { position: absolute; inset: 0; z-index: 20; display: flex; align-items: center; justify-content: center; + background: radial-gradient(ellipse at center, rgba(8, 18, 28, 0.5), rgba(8, 18, 28, 0.86)); } +.ep.over .view.main canvas { filter: blur(3px) brightness(0.8); } +.ep.over .cards, .ep.over .view.corner, .ep.over .live-cap { visibility: hidden; } +.live-wait { position: fixed; inset: 0; display: flex; align-items: center; justify-content: center; + background: radial-gradient(ellipse at 50% 40%, #1d2c3a, #0b1118 75%); } +.live-wait.cover { z-index: 1000; } +.live-card { width: min(980px, calc(100vw - 80px)); padding: 30px 40px 32px; border-radius: 16px; background: rgba(14, 26, 37, 0.92); + border: 1px solid rgba(190, 214, 224, 0.2); box-shadow: 0 30px 90px rgba(0, 0, 0, 0.5); } +.live-card small { display: block; font-size: 13px; font-weight: 700; letter-spacing: 0.1em; text-transform: uppercase; color: #ff8a7e; } +.live-card h1 { margin: 6px 0 4px; font-size: 42px; font-weight: 750; letter-spacing: -0.01em; } +.live-card > p { margin: 0 0 20px; font-size: 18px; color: #c9dbe3; } +.live-card .refs { margin: 0 0 18px; } +.live-card .refs-h { font-size: 13px; } +.live-card .ref { grid-template-columns: 260px 1fr 64px; font-size: 17px; margin: 7px 0; } +.live-card .ref i { height: 12px; border-radius: 5px; } +.live-card dl { display: grid; grid-template-columns: 1fr 1fr; gap: 12px 28px; margin: 0; } +.live-card dt { font-size: 12.5px; color: var(--muted); } +.live-card dd { margin: 2px 0 0; font-size: 18px; font-weight: 600; } +.live-card table { width: 100%; border-collapse: collapse; font-size: 16px; font-variant-numeric: tabular-nums; } +.live-card th { text-align: left; font-size: 12px; font-weight: 650; letter-spacing: 0.06em; text-transform: uppercase; color: var(--muted); + padding: 0 10px 6px 0; border-bottom: 1px solid var(--line); } +.live-card td { padding: 7px 10px 7px 0; border-bottom: 1px solid rgba(190, 214, 224, 0.08); } + +.live-credit { position: absolute; z-index: 6; left: 0; right: 0; bottom: 10px; text-align: center; font-size: 12px; color: rgba(225, 235, 240, 0.78); + text-shadow: 0 1px 3px rgba(0, 0, 0, 0.8); pointer-events: none; } +.live-credit b { color: #fff; font-weight: 650; } + +html.stream, html.stream * { cursor: none !important; } +html.stream { --gap: 18px; --left: 500px; --right: 450px; --inset-w: 420px; } +html.stream .credit, html.stream .follow-chip, html.stream .gate-chip, html.stream .inset-caption, html.stream .cc-caption, html.stream .radar-key, + html.stream .card.desk { display: none !important; } +html.stream .cards.left, html.stream .cards.right { bottom: calc(var(--gap) + 26px); } +html.stream .cards.right { bottom: calc(var(--inset-h) + var(--gap) * 2 + 26px); } +html.stream .view.corner { bottom: calc(var(--gap) + 26px); border-width: 2px; } +html.stream .live-cap { bottom: 50px; } +html.stream .live-panel h2 { font-size: 19px; } +html.stream .lp-desks li { font-size: 13.5px; } +html.stream .cm-key { top: 84px !important; bottom: auto !important; left: 50% !important; right: auto !important; transform: translateX(-50%); } +html.stream .live-credit { bottom: 0; padding: 6px 0 8px; background: linear-gradient(rgba(6, 12, 18, 0), rgba(6, 12, 18, 0.55)); } +.lp-desks li.on { background: rgba(90, 168, 255, 0.16); box-shadow: inset 3px 0 0 #5aa8ff; } +.live-card p.total { margin: 16px 0 0; font-size: 15px; color: var(--muted); } +.live-title { position: absolute; z-index: 6; top: var(--gap); left: 50%; transform: translateX(-50%); max-width: calc(100% - var(--left) - var(--right) - 80px); + padding: 8px 20px 9px; border-radius: 10px; text-align: center; background: rgba(10, 22, 32, 0.72); border: 1px solid rgba(190, 214, 224, 0.16); + backdrop-filter: blur(8px); pointer-events: none; white-space: nowrap; } +.live-title b { display: block; font-size: 21px; font-weight: 750; letter-spacing: 0.01em; } +.live-title b i { font-style: normal; color: #ff7a6e; text-transform: uppercase; font-size: 15px; letter-spacing: 0.12em; margin-left: 4px; vertical-align: 2px; } +.live-title span { display: block; font-size: 13px; color: #c3d3db; overflow: hidden; text-overflow: ellipsis; } +.ep.over .live-title { visibility: hidden; } +.live-brief { position: absolute; z-index: 8; left: 50%; bottom: 46px; transform: translateX(-50%); width: min(900px, calc(100% - var(--left) - var(--right) - 60px)); + padding: 18px 26px 20px; border-radius: 14px; background: rgba(10, 22, 32, 0.86); border: 1px solid rgba(190, 214, 224, 0.18); + box-shadow: 0 16px 50px rgba(0, 0, 0, 0.45); backdrop-filter: blur(8px); pointer-events: none; } +.live-brief[hidden] { display: none; } +.live-brief small { display: block; font-size: 12px; font-weight: 700; letter-spacing: 0.1em; text-transform: uppercase; color: #ff8a7e; } +.live-brief h1 { margin: 4px 0 6px; font-size: 30px; font-weight: 750; } +.live-brief p { margin: 0 0 12px; font-size: 15px; line-height: 1.4; color: #c9dbe3; } +.live-brief dl { display: grid; grid-template-columns: 1fr 1fr; gap: 8px 22px; margin: 0; } +.live-brief dt { font-size: 11.5px; color: var(--muted); } +.live-brief dd { margin: 1px 0 0; font-size: 14px; line-height: 1.35; } +.ep.briefing .live-cap { visibility: hidden; } diff --git a/web/live.js b/web/live.js new file mode 100644 index 0000000..071843a --- /dev/null +++ b/web/live.js @@ -0,0 +1,584 @@ +// The viewer live (?live=): the day a live server (scripts/live_server.py; scripts/live_fake.py for tests) serves, +// played as it is decided (CONTRACT.md). It polls /live/state.json and fetches /live/episode.json when more +// of the day is decided, grows the page in place (app.js episodePage's update()), plays each newly decided hour over about the time +// the desks take to decide the next and eases to a hold where decisions end, never past them; the director +// (director.js) cuts between the room and the airport on the day's events; a live panel shows the run, each desk's +// latest move and the projected cost; at the day's end its score card, its best moments replayed and the next storm +// until the next day appears, which reloads the page. &stream lays it out for a 1920x1080 broadcast with nothing to +// click; the sound (web/audio.js, where the page mounts it as window.airops.sound) is updated from its frames. +import { HUB_NAMES, esc, hhmm, short } from "./model.js"; +import { Director } from "./director.js"; + +const POLL = 3000; +/** Wall seconds the desks take over a watch until the day's own watches time them. */ +const WATCH = 150; +/** The fastest the clock runs to catch up, in minutes of the day a second; once every watch is decided, through the + * rest of the day's flying. */ +const SPEED_MAX = 2.5; +const SPEED_END = 5; +const TAU = 4; +/** Each decided hour is paced to take this share more than a watch is expected to, so the clock rarely stops. */ +const STRETCH = 1.2; +/** Minutes behind where it should be past which the clock jumps on at a cut to the room or the map. */ +const SKIP = 60; +const SCORE_SECONDS = 22; +const HIGHLIGHT_SECONDS = 7; +/** The least the next-storm card shows, when the next day is already on. */ +const NEXT_SECONDS = 10; +const CARD_KEY = "airops-live-card"; +const FETCH_MS = 30000; +/** Bytes of JavaScript heap past which the page reloads onto the day, a safety net over a day of updates. */ +const HEAP_MAX = 3e9; +/** Milliseconds after more of the day is decided that the episode is fetched once more, for the traffic sidecar the + * server rebuilds after the watch's last decision. */ +const SETTLE = 20000; +const DESKS = { ops: "Ops", crew: "Crew", customer: "Customer", one: "Desk" }; +const BRAND = "AirOpsEnv"; + +const longDate = (iso) => new Date(`${iso}T12:00:00Z`).toLocaleDateString("en-GB", { weekday: "short", day: "numeric", month: "short", year: "numeric", timeZone: "UTC" }).replace(",", ""); +const utc = (d) => `${String(d.getUTCHours()).padStart(2, "0")}:${String(d.getUTCMinutes()).padStart(2, "0")}:${String(d.getUTCSeconds()).padStart(2, "0")}`; +const runOf = (l) => (l ? l.run_id ?? l.instance_id ?? `${l.task_id ?? l.date}|${l.started_utc ?? ""}` : null); +const finished = (l) => ["done", "intermission"].includes(l?.status); +const keyOf = (l) => [runOf(l), l?.status, l?.decided_until].join("|"); + +export async function livePage(app, base, { episodePage, params, ready }) { + const stream = params.has("stream"); + document.documentElement.classList.add("live", ...(stream ? ["stream"] : [])); + const css = document.head.appendChild(Object.assign(document.createElement("link"), { rel: "stylesheet", href: new URL("./live.css", import.meta.url).href })); + await Promise.race([new Promise((r) => css.addEventListener("load", r, { once: true })), sleep(3000)]); + const feed = new Feed(base); + const saved = sessionStorage.getItem(CARD_KEY); + sessionStorage.removeItem(CARD_KEY); + if (saved) app.innerHTML = saved; + else app.innerHTML = `
`; + for (let n = 0; ; n++) { + try { + await feed.poll(); + if (feed.ep?.task && runOf(feed.ep.live) === runOf(feed.state) && !["intermission", "stale"].includes(feed.state.status)) break; + if (feed.state?.status === "intermission" || (feed.state && feed.ep && runOf(feed.ep.live) !== runOf(feed.state))) { + app.innerHTML = `
${nextCard(feed.state, feed.ep)}
`; + } + } catch (e) { + if (n % 10 === 0) console.warn("the live feed is not answering yet:", e.message); + } + if (n === 0) ready({ live: true, waiting: true }); + await sleep(POLL); + } + const cover = app.querySelector(".live-wait"); + if (cover) document.body.appendChild(cover).classList.add("cover"); + const live = new Live(feed, { stream, params }); + const page = await episodePage(app, new URL("live/episode.json", feed.root), live); + live.attach(page, app); + cover?.remove(); + ready?.({ live: true }); + return live; +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** A live episode as the recorded viewer reads one: what the day has yet to decide empty, and its cost and grade, which + * only a finished day has, null before. */ +function normal(ep) { + if (!ep?.task) return ep; + const done = finished(ep.live); + for (const k of ["schedule", "executed", "watches", "steps", "news", "frozen", "ferries"]) ep[k] ??= []; + if (ep.occ) for (const k of ["turns", "requests", "decisions", "notes", "pages"]) ep.occ[k] ??= []; + ep.cost = { ...(ep.cost ?? {}), total: done ? ep.cost?.total ?? null : null }; + if (!done) ep.reward = ep.reward_L = null; + return ep; +} + +async function fetchJSON(url) { + const r = await fetch(url, { cache: "no-store", signal: AbortSignal.timeout(FETCH_MS) }); + if (!r.ok) throw Object.assign(new Error(`${url}: HTTP ${r.status}`), { status: r.status }); + const buf = new Uint8Array(await r.arrayBuffer()); + if (buf[0] !== 0x1f || buf[1] !== 0x8b) return JSON.parse(new TextDecoder().decode(buf)); + return JSON.parse(await new Response(new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"))).text()); +} + +/** The live server's state, polled, and its episode, fetched again only when more of the day is decided. */ +class Feed { + constructor(base) { + this.root = new URL(base.href.replace(/\/?$/, "/")); + this.state = null; + this.ep = null; + this.key = null; + this.light = true; + this.at = 0; + } + + /** The state; the episode too when more of the day is decided (and once more SETTLE after), or without a state. */ + async poll() { + let st = null; + if (this.light) { + try { + const s = await fetchJSON(new URL("live/state.json", this.root)); + st = s.live ?? s; + } catch (e) { + if (e.status !== 404) throw e; + this.light = false; + } + } + const now = performance.now(); + if (!st || !this.ep || keyOf(st) !== this.key || (this.settle && now >= this.settle)) { + try { + const ep = normal(await fetchJSON(new URL("live/episode.json", this.root))); + const key = keyOf(ep.live); + this.settle = key !== this.key ? now + SETTLE : null; + this.key = key; + this.ep = ep; + st ??= ep.live; + } catch (e) { + if (e.status !== 404 || !st) throw e; + } + } + this.state = st ?? this.state; + this.at = performance.now(); + } +} + +/** The clock: each decided hour played over about the wall time a watch takes, eased to a hold where decisions end. */ +export class Pacer { + constructor(t, frontier, now) { + this.t = t; + this.F = frontier; + this.v = 0; + this.cap = Infinity; + this.max = SPEED_MAX; + this.marks = []; + this.seg = { s0: t, s1: frontier, w0: now, D: Math.max(1, ((frontier - t) / 60) * WATCH * STRETCH) }; + } + + /** Decisions final through minute F from wall second `now`; `end` once the day is. */ + advance(F, now, end = false) { + this.max = end ? SPEED_END : SPEED_MAX; + if (F === this.F) return; + if (F > this.F) this.marks = [...this.marks, [now, F, F - this.F]].slice(-6); + const v = Math.min(this.virtual(now), F); + this.F = F; + this.seg = { s0: v, s1: F, w0: now, D: Math.max(end ? 0 : this.watch() * STRETCH, (F - v) / this.max, 1) }; + } + + /** Wall seconds a watch (an hour) takes: the median of the latest ones seen decided one at a time, else the server's + * mean over the day's watches, else a guess. */ + watch() { + const xs = []; + for (let i = 1; i < this.marks.length; i++) { + const [w, , dF] = this.marks[i]; + if (dF > 0 && dF <= 90) xs.push(((w - this.marks[i - 1][0]) / dF) * 60); + } + xs.sort((a, b) => a - b); + const W = xs.length >= 3 ? xs[xs.length >> 1] : this.wall ?? WATCH; + return Math.max(4, Math.min(900, W)); + } + + virtual(now) { + const { s0, s1, w0, D } = this.seg; + return s0 + (s1 - s0) * Math.max(0, Math.min(1, (now - w0) / D)); + } + + lag(now = performance.now() / 1000) { + return this.virtual(now) - this.t; + } + + /** Wall seconds the clock is behind where it should be, at the pace it should go. */ + behind(now = performance.now() / 1000) { + const { s0, s1, D } = this.seg; + return this.lag(now) / Math.max(0.2, (s1 - s0) / D); + } + + get ending() { + return this.max > SPEED_MAX; + } + + tick(now, dt) { + const { s0, s1, w0, D } = this.seg, V = this.virtual(now), slope = now - w0 < D ? (s1 - s0) / D : 0; + const want = Math.max(0, Math.min(this.cap, this.max, slope + (V - this.t) / TAU)); + this.v += (want - this.v) * (1 - Math.exp(-dt / 1.2)); + this.t = Math.min(this.F, this.t + this.v * dt); + return { t: this.t, rate: this.v * 60 }; + } + + /** A cut: the clock at once no faster than the new shot wants, and far behind where it should be, on to near it. */ + cut(cap, jump, now = performance.now() / 1000) { + this.cap = cap; + this.v = Math.min(this.v, cap); + if (jump && this.lag(now) > SKIP) this.t = Math.min(this.F, this.virtual(now) - SKIP / 3); + } + + holding() { + return this.F - this.t < 0.5 && this.v < 0.03; + } +} + +class Live { + constructor(feed, { stream, params }) { + this.feed = feed; + this.stream = stream; + this.params = params; + this.ep = feed.ep; + this.instance = runOf(feed.ep.live); + this.applied = { key: keyOf(feed.ep.live), traffic: feed.ep.stage?.traffic ?? null }; + this.phase = "day"; + const l = feed.ep.live ?? {}, ws = feed.ep.watches ?? []; + const lo = feed.ep.task.start, F = this.frontier(l); + const t0 = finished(l) ? F : Math.max(lo, Math.min(F, ws.findLast((w) => w.t < F)?.t ?? lo)); + this.pacer = new Pacer(t0, F, performance.now() / 1000); + this.view = "room"; + this.override = null; + } + + /** Where the day may be played to: what is decided (the end of the day's flying once it is all decided), within + * the page's span once it has one. */ + frontier(l) { + const hi = this.page?.P.hi ?? Infinity; + return Math.min(hi, l.decided_until ?? (finished(l) ? hi : this.ep.watches?.at(-1)?.t ?? this.ep.task.start)); + } + + /** The server's state while it is about this day, else the day's last (once the next day is on). */ + mine() { + const st = this.feed.state; + return (runOf(st) === this.instance && !this.behind(st) ? st : this.ep.live) ?? {}; + } + + /** A state of this day behind where the page has played to: a live server restarted mid-day replays it from its start. */ + behind(st) { + return !finished(st) && this.frontier(st) < this.pacer.F; + } + + /** app.js asks for the clock each frame. */ + clock(now, dt) { + if (this.override) return this.override(now, dt); + if (!this.page) return { t: this.pacer.t, rate: 0 }; + return this.pacer.tick(now / 1000, dt); + } + + attach(page, app) { + this.page = page; + this.app = app; + const lo = page.P.lo, hi = page.P.hi, done = finished(this.ep.live), F = this.frontier(this.ep.live ?? {}); + this.pacer.F = F; + this.pacer.t = done ? F : Math.max(lo, Math.min(F, this.pacer.t)); + this.pacer.seg = { ...this.pacer.seg, s0: Math.min(this.pacer.seg.s0, this.pacer.F), s1: this.pacer.F }; + this.root = page.root; + if (page.room) page.room.stagger = true; + this.panel = new Panel(this.root, { stream: this.stream }); + this.cap = this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-cap", hidden: true })); + this.cap.innerHTML = ""; + if (this.stream) { + const model = this.ep.model_label ?? this.ep.model; + this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-title", + innerHTML: `${BRAND} live${esc(model)} runs JetBlue's control center through real storm days, one after another` })); + this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-credit", innerHTML: credit() })); + } + this.director = new Director(page, { pacer: this.pacer, caption: (beat, t) => this.caption(beat, t), log: this.params.get("log") === "1" }); + this.panel.render(this.mine(), this); + this.timer = setInterval(() => this.poll(), POLL); + window.airops.live = this; + for (const el of this.root.querySelectorAll("canvas")) el.addEventListener("webglcontextlost", () => this.restart("the GPU dropped a 3D view")); + if (done) this.score(); + } + + async poll() { + if (this.polling) return; + this.polling = true; + try { + await this.feed.poll(); + const st = this.feed.state, ep = this.feed.ep; + if (runOf(st) !== this.instance) return this.nextDay(st); + if (this.behind(st)) return; + if (st.watch_wall_s) this.pacer.wall = st.watch_wall_s; + if (runOf(ep.live) === this.instance && (keyOf(ep.live) !== this.applied.key || (ep.stage?.traffic ?? null) !== this.applied.traffic)) await this.grow(ep); + this.panel.render(this.mine(), this); + } catch (e) { + this.panel.offline(e); + } finally { + this.polling = false; + } + } + + /** More of the day decided: the page grown, its traffic sidecar fetched if it changed, the clock let further. */ + async grow(ep) { + this.ep = ep; + let traffic = null; + if (ep.stage?.traffic && ep.stage.traffic !== this.applied.traffic && this.page.stage) { + traffic = await fetchJSON(new URL(ep.stage.traffic, new URL("live/episode.json", this.feed.root))).catch((e) => (console.warn("the traffic sidecar did not load:", e), null)); + } + this.applied = { key: keyOf(ep.live), traffic: ep.stage?.traffic ?? null }; + await this.page.update(ep, traffic); + this.director.index(); + this.pacer.advance(this.frontier(ep.live ?? {}), performance.now() / 1000, finished(ep.live)); + } + + /** Each frame: the director, the transitions at the day's end, the sound. */ + frame(t, now, dt) { + const st = this.mine(); + if (this.page.stage) this.page.stage.hidden = !this.page.views.visible("stage"); + if (this.phase === "day") { + this.brief(t <= this.page.P.lo + 0.5 && this.pacer.holding()); + this.director.tick(t, now / 1000, this.pacer.v); + if (finished(this.ep.live) && t >= this.pacer.F - 0.5) this.score(); + } else if (this.phase === "highlights") this.highlight(t, now / 1000); + if (now - (this.drawn ?? 0) > 500) { + this.drawn = now; + this.panel.tick(st, this); + if ((performance.memory?.usedJSHeapSize ?? 0) > HEAP_MAX && this.phase === "day") this.restart("the heap passed its bound"); + } + const sound = window.airops?.sound; + if (sound && !this.mute) { + const main = this.page.views.main; + try { + sound.update({ t, rate: this.page.rate() ?? 0, view: this.page.view(), main, phase: this.phase, holding: this.pacer.holding() }, + main === "room" ? this.page.room?.camera : main === "stage" ? this.page.stage?.camera : null); + } catch (e) { + console.warn("the live sound failed, off:", e); + this.mute = true; + } + } + } + + /** An error in a frame, logged once a minute at most so a day's stream goes on. */ + error(e) { + const now = performance.now(); + if (now - (this.errAt ?? -1e9) > 60000) console.error("live frame:", e); + this.errAt = now; + this.errors = (this.errors ?? 0) + 1; + } + + /** The day's briefing while it holds at its start for the desks' first decisions: the hub and date, the network, the + * weather and the FAA as the header has them, and what the run is scored against. */ + brief(on) { + this.root.classList.toggle("briefing", on); + if (!on) { + if (this.briefing) this.briefing.hidden = true; + return; + } + if (!this.briefing) this.briefing = this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-brief", hidden: true })); + const ep = this.ep, task = ep.task, pax = ep.schedule.reduce((a, s) => a + (s.pax ?? 0), 0), here = ep.schedule.filter((s) => s.orig === task.hub || s.dest === task.hub).length; + const std = (ep.occ?.refs ?? []).find((r) => /standard/i.test(r.label)), day = (this.mine().history ?? []).filter((h) => h.outcome !== "skipped").length + 1; + const text = (sel) => { + const el = this.root.querySelector(sel), chip = el?.querySelector(".chip")?.textContent ?? ""; + const rest = (el?.textContent ?? "").replace(chip, "").trim(); + return chip ? `${chip}: ${rest}` : rest; + }; + const html = `${BRAND} · live · storm day ${day}

${esc(HUB_NAMES[task.hub] ?? task.hub)} · ${esc(longDate(task.date))}

+

${esc(ep.model_label ?? ep.model)} takes JetBlue's control center from ${hhmm(task.start)} ${esc(ep.occ?.label ? `as ${ep.occ.label}` : "")}: an ops controller for the flights, a crew scheduler for the crews${ep.passengers ? " and a customer desk for the passengers" : ""}. The airport plays each hour once they have decided it.

+
JetBlue's network today
${ep.schedule.length.toLocaleString("en-US")} flights, ${here} of them at ${esc(task.hub)} · ${pax.toLocaleString("en-US")} passengers booked
+
The weather
${esc(text(".card.head .wx"))}
The FAA
${esc(text(".card.head .faa"))}
+
Scored against
the best plan found in hindsight (1.0)${std ? ` and the standard crew desk (${std.reward.toFixed(2)})` : ""}
`; + if (html !== this.briefing._html) this.briefing.innerHTML = this.briefing._html = html; + this.briefing.hidden = false; + } + + caption(beat, t) { + const text = [beat?.kicker ?? "", beat?.caption ?? "", typeof beat?.subline === "function" ? beat.subline(t) : beat?.subline ?? ""]; + const key = text.join("|"); + if (this.capKey === key) return; + this.capKey = key; + const [k, c, s] = this.cap.children; + k.textContent = text[0]; + c.textContent = text[1]; + s.textContent = text[2]; + this.cap.hidden = !text[1]; + } + + /** The day's score card, then its highlights. */ + score() { + if (this.phase !== "day") return; + this.phase = "score"; + this.caption(null); + this.card = this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-over", innerHTML: scoreCard(this.ep, this.mine()) })); + this.root.classList.add("over"); + setTimeout(() => this.highlights(), SCORE_SECONDS * 1000); + } + + async highlights() { + this.card?.remove(); + this.root.classList.remove("over"); + this.reel = this.director.highlights(6); + this.phase = "highlights"; + this.panel.badge("Highlights"); + if (!this.reel.length) return this.next(); + this.shot = -1; + await this.cue(0); + } + + async cue(k) { + if (k >= this.reel.length) return this.next(); + const b = this.reel[k], speed = b.camera.startsWith("room") || b.camera === "network" ? 0.8 : 0.12; + const t0 = Math.max(this.page.P.lo, b.at - speed * 1.5), start = performance.now() / 1000; + this.shot = k; + this.override = (now) => ({ t: Math.min(this.page.P.hi, t0 + (now / 1000 - start) * speed), rate: speed * 60 }); + this.until = start + HIGHLIGHT_SECONDS; + this.director.busy = true; + try { + await this.director.apply(b, t0); + } catch (e) { + console.warn("a highlight failed:", e); + } + this.director.busy = false; + this.caption({ ...b, kicker: `Highlights · ${b.kicker}` }, b.at + 9); + } + + highlight(t, now) { + if (this.shot >= 0 && now >= this.until && !this.cueing) { + this.cueing = true; + this.cue(this.shot + 1).finally(() => (this.cueing = false)); + } + } + + /** The next storm's card over the finished day, until the next day appears. */ + next() { + this.phase = "next"; + this.caption(null); + this.override = () => ({ t: this.pacer.F, rate: 0 }); + this.page.setView(this.page.stage ? "harbour" : "network").catch(() => {}); + this.panel.badge("Next storm"); + this.card?.remove(); + this.card = this.root.appendChild(Object.assign(document.createElement("div"), { className: "live-over", innerHTML: nextCard(this.feed.state, this.ep) })); + this.root.classList.add("over"); + this.nextTimer = setInterval(() => (this.card.innerHTML = nextCard(this.feed.state, this.ep)), 5000); + this.nextAt = performance.now(); + if (this.waiting) setTimeout(() => this.reload(), NEXT_SECONDS * 1000); + } + + /** Another day on the server: once this one's card, highlights and next card have shown, the page reloads onto it. */ + nextDay(st) { + if (!st?.status || finished(st)) return; + this.waiting = true; + if (this.phase === "day" && !finished(this.ep.live)) return this.reload(); + if (this.phase === "next" && performance.now() - this.nextAt >= NEXT_SECONDS * 1000) this.reload(); + } + + /** The page again onto the day where it is decided so far, after the GPU drops a view or the heap grows past bounds. */ + restart(why) { + console.warn(`live: reloading, ${why}`); + this.reload(false); + } + + reload(next = true) { + if (this.reloading) return; + this.reloading = true; + clearInterval(this.timer); + clearInterval(this.nextTimer); + try { + sessionStorage.setItem(CARD_KEY, `
${next ? nextCard(this.feed.state, this.ep, true) + : ``}
`); + } catch {} + location.reload(); + } + + state() { + const st = this.feed.state ?? {}; + return { phase: this.phase, t: this.page?.time() ?? this.pacer.t, errors: this.errors ?? 0, frontier: this.pacer.F, speed: this.pacer.v, cap: this.pacer.cap, lag: this.pacer.lag(), watch: this.pacer.watch(), + holding: this.pacer.holding(), status: st.status, seq: st.seq, instance: this.instance, view: this.page?.view(), main: this.page?.views.main, + director: this.director?.state(), reel: this.reel?.map((b) => b.id) ?? null, sound: !!window.airops?.sound && !this.mute }; + } +} + +/** The live panel: LIVE and the time, the model and the storm, the watch, each desk's latest move, the cost so far. */ +class Panel { + constructor(root, { stream }) { + this.el = document.createElement("section"); + this.el.className = "card live-panel"; + const left = root.querySelector(".cards.left"); + left.insertBefore(this.el, left.querySelector(".desk")); + this.stream = stream; + this.memo = ""; + this.label = ""; + } + + render(st, live) { + this.st = st ?? {}; + this.live = live; + this.tick(this.st, live); + } + + badge(label) { + this.label = label; + this.tick(this.st, this.live); + } + + offline(e) { + this.down = e.message; + this.tick(this.st, this.live); + } + + tick(st, live) { + if (!live?.page) return; + if (live.feed.at && performance.now() - live.feed.at < POLL * 3) this.down = null; + const ep = live.ep, P = live.page.P, task = P.task, now = new Date(), done = finished(st) || live.phase !== "day"; + const model = st.model_label ?? ep.model_label ?? ep.model, mode = ep.occ?.label ?? ""; + const off = this.down || st.status === "stale"; + const head = `
${off ? "Reconnecting" : "Live"}${utc(now)} UTC${this.label ? `${esc(this.label)}` : ""}
`; + const title = `

${esc(model)}${esc(mode)}

${esc(HUB_NAMES[task.hub] ?? task.hub)} · ${esc(longDate(task.date))} · a real storm day, played live

`; + const holding = !done && live.pacer.holding(), next = st.next_watch_at ? Math.round((new Date(st.next_watch_at) - now) / 1000) : null; + const watch = st.status === "starting" ? `The desks open at ${hhmm(task.start)}` + : done ? "The day is done: every watch decided" + : st.status === "ending" ? "The desks are done; the day plays out" + : `Deciding ${st.t != null ? hhmm(st.t) : "the next hour"}${st.watches ? ` · watch ${Math.min((st.watch ?? 0) + 1, st.watches)} of ${st.watches}` : ""}`; + const sub = done ? "" : holding ? "The airport waits for the desks' decisions" : `The airport plays ${hhmm(live.pacer.t)} of what's decided${next != null && next > 0 && st.status === "playing" ? ` · the next watch in about ${next < 90 ? `${next} s` : `${Math.round(next / 60)} min`}` : ""}`; + const words = (text) => String(text ?? "").replace(/\bf(\d{1,4})\b/g, (m, id) => P.sched.get(Number(id))?.flight ?? m); + const desks = (ep.occ?.mode === "one" ? ["one"] : ep.occ?.desks?.includes("customer") ? ["ops", "crew", "customer"] : ["ops", "crew"]).map((d) => { + const x = (st.desks ?? []).find((y) => y.role === d || d === "one"), ev = (st.feed ?? []).findLast((f) => (f.desk === d || d === "one") && ["request", "reply", "decision"].includes(f.kind)); + const what = ev ? words(ev.text).replace(/^(ops|crew|customer|one)\b:?\s*/i, "") : x?.state === "off" || done ? "Stood down" : x?.state === "turn" ? "At work" : "Waiting for its turn"; + return `
  • ${DESKS[d]}${esc(what)}
    • `; + }).join(""); + const cost = done && ep.cost?.total != null ? ep.cost.total : st.projected?.total ?? st.projected_cost ?? ep.watches?.findLast((w) => w.est_cost != null)?.est_cost ?? null; + const day = st.result?.cost_usd ?? st.spent_usd ?? null; + const total = st.total_usd ?? (st.history ?? []).filter((h) => h.run_id !== st.run_id).reduce((a, h) => a + (h.cost_usd ?? 0), day ?? 0); + const reward = st.result?.reward ?? ep.reward; + const grade = done && reward != null ? `
    Final reward${reward.toFixed(3)}1 = the best plan found
    ` : ""; + const html = `${head}${title}
    ${esc(watch)}${esc(sub)}
      ${desks}
    +
    ${cost != null ? short(cost) : "–"}${done ? "the day's cost" : "projected cost of the day"}, passenger-minutes
    +
    ${day != null ? `$${day.toFixed(2)}` : "–"}model spend this day${total > (day ?? 0) + 0.005 ? ` · $${total.toFixed(2)} on the stream` : ""}
    ${grade}`; + if (html !== this.memo) this.el.innerHTML = this.memo = html; + } +} + +function ago(at, now) { + if (at == null) return ""; + if (typeof at === "number") return hhmm(at); + const s = Math.round((now - new Date(at)) / 1000); + return !Number.isFinite(s) ? "" : s < 60 ? "just now" : s < 3600 ? `${Math.floor(s / 60)} min ago` : `${Math.floor(s / 3600)} h ago`; +} + +function scoreCard(ep, st) { + const P = ep, ref = P.reference ?? {}, std = (P.occ?.refs ?? []).find((r) => /standard/i.test(r.label)); + const h = (st?.history ?? []).find((x) => x.run_id === (st?.run_id ?? ep.live?.run_id)); + const model = ep.model_label ?? ep.model, task = ep.task, reward = st?.result?.reward ?? ep.reward ?? h?.reward; + const rows = [[model, reward, true], ["Best plan found", 1, false], ...(std ? [["Standard crew desk (B0)", std.reward, false]] : [])].filter((r) => r[1] != null); + const bar = ([label, r, me]) => `
    ${esc(label)}${r.toFixed(3)}
    `; + const over = ref.anchor && ep.cost?.total != null ? (100 * (ep.cost.total - ref.anchor)) / ref.anchor : null; + const wall = st?.result?.wall_seconds ?? ep.record?.wall_seconds ?? h?.wall_seconds ?? st?.wall_seconds; + const spend = st?.result?.cost_usd ?? ep.record?.cost_usd ?? h?.cost_usd ?? st?.spent_usd; + const dl = [["The day's cost", ep.cost?.total != null ? `${short(ep.cost.total)} passenger-minutes${over != null ? `, ${over >= 0 ? "+" : ""}${over.toFixed(1)}% vs the best plan` : ""}` : "–"], + ["Doing what the standard desks do (B0)", ref.B0 != null ? `${short(ref.B0)} passenger-minutes` : "–"], + ["Wall time", wall != null ? `${Math.floor(wall / 3600) ? `${Math.floor(wall / 3600)} h ` : ""}${Math.round((wall % 3600) / 60)} min` : "–"], + ["Model spend", spend != null ? `$${spend.toFixed(2)}` : "–"]]; + return `
    ${BRAND} · storm day done

    ${esc(HUB_NAMES[task.hub] ?? task.hub)} · ${esc(longDate(task.date))}

    +

    ${esc(model)} ran JetBlue's control center${esc(ep.occ?.label ? ` at ${ep.occ.label}` : "")} through the storm

    +
    Reward (1 = the best plan found in hindsight)
    ${rows.map(bar).join("")}
    +
    ${dl.map(([k, v]) => `
    ${esc(k)}
    ${esc(v)}
    `).join("")}
    `; +} + +function nextCard(st, ep, loading = false) { + const on = !!st?.date && runOf(st) !== runOf(ep?.live) && !finished(st); + const n = on ? { date: st.date, hub: st.hub } : st?.next ?? null, hist = (st?.history ?? []).filter((h) => h.outcome !== "skipped"); + const when = n?.starts_utc ? new Date(n.starts_utc) : null, mins = when ? Math.round((when - Date.now()) / 60000) : null; + const last = !on && !n && !loading && !!st?.final; + const what = n ? `${n.hub ? `${HUB_NAMES[n.hub] ?? n.hub} · ` : ""}${longDate(n.date)}` : loading ? "The next storm day" : last ? "The last storm day is done" : "Between storm days"; + const rows = hist.slice(-6).map((h) => `${esc(longDate(h.date))}${h.reward != null ? h.reward.toFixed(3) : esc(h.outcome ?? "–")}${h.cost_usd != null ? `$${h.cost_usd.toFixed(2)}` : "–"}${h.wall_seconds != null ? `${Math.round(h.wall_seconds / 60)} min` : "–"}`).join(""); + const scored = hist.filter((h) => h.reward != null), spend = hist.reduce((a, h) => a + (h.cost_usd ?? 0), 0); + const total = scored.length ? `${scored.length} storm day${scored.length > 1 ? "s" : ""}${last ? "" : " so far"} · mean reward ${(scored.reduce((a, h) => a + h.reward, 0) / scored.length).toFixed(3)} · $${spend.toFixed(2)} of model spend` : ""; + const model = st?.model_label ?? ep?.model_label ?? "The model"; + const lead = last ? `${model} ran JetBlue's control center live through ${scored.length} storm day${scored.length === 1 ? "" : "s"}. Thanks for watching.` + : `${model} takes JetBlue's control center ${hist.length ? "again " : ""}from 04:00 local${on ? ", starting now" : when ? `, ${mins > 1 ? `in about ${mins} min` : "any moment now"} (${utc(when).slice(0, 5)} UTC)` : ""}.`; + return ``; +} + +function credit() { + return `AirOpsEnv · built on AgentEnv · data and viewer on Hugging Face · flights BTS, advisories FAA ATCSCC, weather NWS and IEM, airport © OpenStreetMap contributors · the room, gates and passengers are drawn for display`; +} diff --git a/web/stage/stage.js b/web/stage/stage.js index 8529ff6..d156440 100644 --- a/web/stage/stage.js +++ b/web/stage/stage.js @@ -15,6 +15,7 @@ import { buildPassengers, cutouts } from "./passengers.js"; import { buildSpray } from "./spray.js"; import { aim, buildTerminal } from "./terminal.js"; import { buildWeather } from "./weather.js"; +import { freed } from "../gc.js"; const DEG = Math.PI / 180; const VIEWS = ["tower", "gate", "threshold", "harbour", "follow", "crew"]; @@ -58,6 +59,7 @@ export class AirportStage { this.rig = { kind: "preset", name: "tower" }; this.featured = null; this.frameMs = []; + this.hidden = false; this.renderer = new THREE.WebGLRenderer({ antialias: true, logarithmicDepthBuffer: true, preserveDrawingBuffer: manual, powerPreference: "high-performance" }); this.renderer.setPixelRatio(pixelRatio); this.renderer.shadowMap.enabled = true; @@ -117,7 +119,7 @@ export class AirportStage { renderer.setAnimationLoop(() => { const dt = Math.min(clock.getDelta(), 0.1); this.wall += dt; - this.#frame(dt); + if (!this.hidden) this.#frame(dt); }); } return { hub: stage.hub, start: traffic.span[0], end: traffic.span[1], views: VIEWS, far, aircraft: this.plan.tracks.size, movements: this.plan.report.movements }; @@ -286,6 +288,33 @@ export class AirportStage { this.camera.updateProjectionMatrix(); } + /** Another traffic sidecar of the same day, as a live day's decisions land: its aircraft, crews and passengers built + * again in place, the airport, the buildings, the weather and the camera kept, and what only the old ones used freed. */ + async retraffic(traffic) { + await this.ready; + const { data: stage, scene } = this; + const old = [this.fleet, this.bridges, this.crews, this.pax, this.terminal].filter(Boolean).map((x) => x.group).concat(this.spray.mesh); + this.traffic = traffic; + this.plan = buildTracks(stage, traffic); + const pax = traffic.passengers?.lounges?.length ? traffic.passengers : null; + if (pax) annotate(traffic, pax.tags || {}); + const people = pax ? buildPassengers(pax, stage) : null; + this.bridges = buildBridges(stage, this.plan, people ? { fit: (pts, tip) => people.fit(pts, tip, ROTUNDA) } : undefined); + this.fleet = buildFleet(this.plan); + this.crews = buildCrews(traffic.crews || []); + this.pax = people; + this.terminal = people ? buildTerminal(people, stage, traffic, { home: homeOf(stage).ref, bridges: this.bridges, page: { camera: this.camera, band: () => this.#band(true) } }) : null; + this.spray = buildSpray(this.plan); + const fresh = [this.bridges.group, this.fleet.group, this.crews.group, this.pax?.group, this.terminal?.group, this.spray.mesh].filter(Boolean); + scene.remove(...old); + scene.add(...fresh); + freed(old, [scene]); + if (this.rig.kind === "terminal" && !this.terminal) this.#view("tower"); + else if (this.rig.kind === "terminal") this.#terminalAt(); + if (this.rig.kind === "crew" && !this.crews.focus(this.rig.key, this.T)) this.#view("crew", {}); + return { aircraft: this.plan.tracks.size, movements: this.plan.report.movements }; + } + dispose() { this.renderer.setAnimationLoop(null); this.controls?.dispose(); diff --git a/web/style.css b/web/style.css index 883d112..3f3657e 100644 --- a/web/style.css +++ b/web/style.css @@ -190,6 +190,10 @@ button, select { font: inherit; color: inherit; } .cams button.on { background: #fff; color: #10202c; font-weight: 650; } .expert-toggle { flex: none; border: 1px solid var(--line); } .expert-toggle.on { background: var(--hold); border-color: var(--hold); color: #04121f; font-weight: 650; } +.sound-toggle { flex: none; display: grid; place-items: center; width: 32px; height: 30px; padding: 0; border: 1px solid var(--line); border-radius: 5px; background: transparent; + color: #c9d6dc; cursor: pointer; } +.sound-toggle:hover { color: #fff; background: rgba(255, 255, 255, 0.08); } +.sound-toggle.on { background: #fff; border-color: #fff; color: #10202c; } .follow-chip { position: absolute; z-index: 6; top: var(--gap); left: 50%; transform: translateX(-50%); display: flex; align-items: center; gap: 8px; padding: 6px 8px 6px 14px; border-radius: 20px; background: var(--card); border: 1px solid var(--line); box-shadow: var(--shadow); font-size: 13px; @@ -262,7 +266,7 @@ svg.tails .wtick.past { fill: var(--now); } html.recording, html.recording * { cursor: none !important; } html.recording .timeline .play, html.recording .timeline .speed, html.recording .cams, html.recording .expert-toggle, html.recording .more, -html.recording .follow-chip button { display: none; } +html.recording .follow-chip button, html.recording .sound-toggle { display: none; } html.recording .desk { overflow: hidden; } html.bare .cards, html.bare .timeline, html.bare .credit, html.bare .follow-chip, html.bare .view.corner, html.bare .expert { display: none !important; }