feat(live): optional tooling to livestream a model's storm days to X

README.md

@@ -1104,6 +1104,7 @@ AirOpsEnv v1 is a model of a day, not an airline's operations system. What it le
 | `scripts/build_stage.py`, `scripts/prebake_traffic.py`, `scripts/build_twin.py`, `scripts/prebake_radar.py` | the viewer's airport stages, traffic, hub twins and radar frames, into `cache/` |
 | `scripts/record.py` | films an episode in headless Chrome |
 | `scripts/hub_dataset.py`, `hub/` | the Hugging Face dataset and the static Space |
+| `live/` | optional, not needed to use the plugin: streaming a model's days live to X or any RTMP target ([live/README.md](live/README.md)) |
 
 ```bash
 uv sync --extra dev --extra viewer --extra build

live/.gitignore

@@ -0,0 +1,2 @@
+live.env
+runner/fetched/

live/aws/common.sh

@@ -0,0 +1,92 @@
+# Sourced by the other scripts: the settings (live/live.env, or the file LIVE_ENV names), the AWS profile and region,
+# and the S3 and SSM helpers.
+# shellcheck disable=SC2034
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIVE_ENV="${LIVE_ENV:-$(cd "$HERE/.." && pwd)/live.env}"
+[ -f "$LIVE_ENV" ] || { echo "ERROR: no $LIVE_ENV: copy live/live.env.example there and fill it in" >&2; exit 1; }
+# shellcheck source=/dev/null
+source "$LIVE_ENV"
+
+# Forced, not defaulted: a shell's own AWS_PROFILE (e.g. a production one) must never be used here.
+if [ -n "${LIVE_AWS_PROFILE:-}" ]; then
+  export AWS_PROFILE="$LIVE_AWS_PROFILE"
+  unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
+fi
+export AWS_REGION="${LIVE_AWS_REGION:-us-west-2}"
+export AWS_DEFAULT_REGION="$AWS_REGION"
+export AWS_PAGER=""
+if [ -n "${LIVE_AWS_ACCOUNT:-}" ]; then
+  [ "$(aws sts get-caller-identity --query Account --output text)" = "$LIVE_AWS_ACCOUNT" ] \
+    || { echo "ERROR: these credentials are not account $LIVE_AWS_ACCOUNT" >&2; exit 1; }
+fi
+
+REPO_URL="${LIVE_REPO_URL:-https://github.com/earakely-scale/agentenv-airops-plugin.git}"
+BASE_REF="${LIVE_BASE_REF:-v0.5.0}"
+BUCKET="${LIVE_BUCKET:-}"
+PREFIX="${LIVE_S3_PREFIX:-airops-live}"
+SUBNET="${SUBNET:-${LIVE_SUBNET:-}}"
+SG="${LIVE_SECURITY_GROUP:-}"
+PROFILE_NAME="${LIVE_INSTANCE_PROFILE:-}"
+TAG_OWNER="${LIVE_TAG_OWNER:-${USER:-airops}}"
+TAG_PURPOSE="${LIVE_TAG_PURPOSE:-airops-live}"
+
+log() { echo "[$(date +%H:%M:%S)] $*" >&2; }
+die() { echo "ERROR: $*" >&2; exit 1; }
+need() { local v; for v in "$@"; do [ -n "${!v:-}" ] || die "set $v in $LIVE_ENV"; done; }
+
+# presign get|put KEY SECONDS: a presigned URL for s3://$BUCKET/KEY (aws s3 presign only does GET).
+presign() {
+  local py=(python3)
+  python3 -c 'import boto3' 2>/dev/null || py=(uv run --quiet --no-project --with boto3 python)
+  "${py[@]}" - "$1" "$BUCKET" "$2" "$3" "$AWS_REGION" <<'PY'
+import sys, boto3
+from botocore.config import Config
+op, bucket, key, ttl, region = sys.argv[1:6]
+s3 = boto3.client("s3", region_name=region, endpoint_url=f"https://s3.{region}.amazonaws.com",
+                  config=Config(signature_version="s3v4", s3={"addressing_style": "virtual"}))
+print(s3.generate_presigned_url({"get": "get_object", "put": "put_object"}[op],
+                                Params={"Bucket": bucket, "Key": key}, ExpiresIn=int(ttl)))
+PY
+}
+
+wait_ssm() {
+  local id=$1 i
+  for i in $(seq 1 90); do
+    [ "$(aws ssm describe-instance-information --filters "Key=InstanceIds,Values=$id" \
+         --query 'InstanceInformationList[0].PingStatus' --output text 2>/dev/null)" = Online ] && return 0
+    sleep 10
+  done
+  die "$id never came online in SSM"
+}
+
+# ssm INSTANCE TIMEOUT_S SCRIPT [ARGS...]: run SCRIPT with ARGS under bash as root on the box; prints its stdout and
+# stderr (SSM keeps ~24 KB of stdout and ~8 KB of stderr) and returns its exit code. The script and its arguments
+# travel base64-encoded in the command, so presigned URLs passed as arguments sit in SSM's command history.
+ssm() {
+  local id=$1 timeout=$2 file=$3; shift 3
+  local wrapped b64 params cid status=Pending out
+  wrapped="$(printf 'set --'; printf ' %q' "$@"; printf '\n'; cat "$file")"
+  b64=$(printf '%s\n' "$wrapped" | base64 | tr -d '\n')
+  params=$(python3 -c 'import json,sys; print(json.dumps({"commands": [sys.argv[1]], "executionTimeout": [sys.argv[2]]}))' \
+    "f=\$(mktemp); echo $b64 | base64 -d > \$f; bash \$f" "$timeout")
+  cid=$(aws ssm send-command --instance-ids "$id" --document-name AWS-RunShellScript --timeout-seconds 600 \
+        --comment "airops $(basename "$file")" --parameters "$params" --query Command.CommandId --output text)
+  while :; do
+    sleep 5
+    status=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$id" --query Status --output text \
+             2>/dev/null || echo Pending)
+    case $status in Pending|InProgress|Delayed|Cancelling) ;; *) break ;; esac
+  done
+  out=$(aws ssm get-command-invocation --command-id "$cid" --instance-id "$id" --output json)
+  python3 -c '
+import json, sys
+r = json.loads(sys.stdin.read())
+sys.stdout.write(r.get("StandardOutputContent", ""))
+err = r.get("StandardErrorContent", "")
+if err.strip():
+    sys.stderr.write("--- stderr ---\n" + err)
+' <<<"$out"
+  [ "$status" = Success ] || { log "SSM command $cid ended $status"; return 1; }
+}

live/aws/launch-gpu.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# launch-gpu.sh NAME [MAX_HOURS] [ROOT_GB]: the on-air box, a GPU box on the AWS Deep Learning Base OSS NVIDIA driver
+# AMI (Ubuntu 22.04). GPU capacity moves between zones: it tries each type of LIVE_GPU_TYPES in each subnet of
+# LIVE_SUBNETS (default LIVE_SUBNET) until one comes up in SSM, and prints its instance id. Defaults: 48 h, 250 GB.
+source "$(dirname "$0")/common.sh"
+NAME=${1:?usage: launch-gpu.sh NAME [MAX_HOURS] [ROOT_GB]}
+MAX_HOURS=${2:-48}; ROOT_GB=${3:-250}
+TYPES=${LIVE_GPU_TYPES:-g6.4xlarge gr6.4xlarge g6.8xlarge}; SUBNETS=${LIVE_SUBNETS:-${LIVE_SUBNET:-}}
+[ -n "$SUBNETS" ] || die "set LIVE_SUBNET (or LIVE_SUBNETS) in $LIVE_ENV"
+AMI=$(aws ssm get-parameter --name /aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id \
+  --query Parameter.Value --output text)
+export AMI
+for t in $TYPES; do
+  for s in $SUBNETS; do
+    out=$(SUBNET=$s bash "$HERE/launch.sh" "$t" "$NAME" "$MAX_HOURS" "$ROOT_GB" 2>&1 || true)
+    log "$t $s: $(echo "$out" | tail -1 | cut -c1-150)"
+    if echo "$out" | grep -q 'online in SSM'; then echo "$out" | tail -1; exit 0; fi
+  done
+done
+die "no $NAME came up: no capacity for $TYPES in $SUBNETS"

live/aws/launch.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB]
+# Launches (or finds, when one named NAME is already pending or running) an Ubuntu 24.04 box (AMI= another image),
+# waits until SSM sees it and prints its instance id on stdout. The box powers itself off, and so terminates, MAX_HOURS
+# (default 12) after boot.
+source "$(dirname "$0")/common.sh"
+need LIVE_SECURITY_GROUP LIVE_INSTANCE_PROFILE
+[ -n "$SUBNET" ] || die "set LIVE_SUBNET in $LIVE_ENV"
+TYPE=${1:?usage: launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB]}
+NAME=${2:?usage: launch.sh TYPE NAME [MAX_HOURS] [ROOT_GB]}
+MAX_HOURS=${3:-12}
+ROOT_GB=${4:-100}
+
+id=$(aws ec2 describe-instances \
+  --filters "Name=tag:Name,Values=$NAME" "Name=tag:purpose,Values=$TAG_PURPOSE" \
+            "Name=instance-state-name,Values=pending,running" \
+  --query 'Reservations[].Instances[0].InstanceId' --output text)
+if [ -n "$id" ] && [ "$id" != None ]; then
+  log "$NAME is already up as $id"
+else
+  arch=amd64
+  [[ $TYPE =~ ^[a-z]+[0-9]+[a-z]*g[a-z]*\. ]] && arch=arm64
+  ami=${AMI:-$(aws ssm get-parameter --name "/aws/service/canonical/ubuntu/server/24.04/stable/current/$arch/hvm/ebs-gp3/ami-id" \
+        --query Parameter.Value --output text)}
+  userdata=$(printf '#!/bin/bash\nshutdown -h +%d "airops: MAX_HOURS reached"\n' $((MAX_HOURS * 60)) | base64 | tr -d '\n')
+  tags="{Key=Name,Value=$NAME},{Key=owner,Value=$TAG_OWNER},{Key=purpose,Value=$TAG_PURPOSE}"
+  id=$(aws ec2 run-instances --image-id "$ami" --instance-type "$TYPE" --count 1 \
+    --subnet-id "$SUBNET" --security-group-ids "$SG" --iam-instance-profile "Name=$PROFILE_NAME" \
+    --instance-initiated-shutdown-behavior terminate \
+    --metadata-options HttpTokens=required,HttpEndpoint=enabled \
+    --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$ROOT_GB,VolumeType=gp3,DeleteOnTermination=true}" \
+    --user-data "$userdata" \
+    --tag-specifications "ResourceType=instance,Tags=[$tags]" "ResourceType=volume,Tags=[$tags]" \
+    --query 'Instances[0].InstanceId' --output text)
+  log "launched $id ($TYPE, $arch, $ami, ${ROOT_GB} GB root, powers off after ${MAX_HOURS} h)"
+fi
+aws ec2 wait instance-running --instance-ids "$id"
+log "$id running; waiting for SSM"
+wait_ssm "$id"
+log "$id online in SSM"
+echo "$id"

live/aws/put-secret.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# put-secret.sh litellm-key|x-server|x-key: reads the value without echoing it and stores it as a SecureString at the
+# SSM parameter live.env names for it (LIVE_LITELLM_KEY_PARAM, LIVE_X_SERVER_PARAM, LIVE_X_KEY_PARAM). The value never
+# reaches the terminal, the shell history or a file.
+source "$(dirname "$0")/common.sh"
+case ${1:-} in
+  litellm-key) NAME=${LIVE_LITELLM_KEY_PARAM:-/airops-live/litellm-key} ;;
+  x-server) NAME=${LIVE_X_SERVER_PARAM:-/airops-live/x-server} ;;
+  x-key) NAME=${LIVE_X_KEY_PARAM:-/airops-live/x-key} ;;
+  *) die "usage: put-secret.sh litellm-key|x-server|x-key" ;;
+esac
+read -rsp "value for $NAME: " v; echo >&2
+[ -n "$v" ] || die "empty value; nothing stored"
+aws ssm put-parameter --name "$NAME" --type SecureString --overwrite --value "$v" --query Version --output text >/dev/null
+unset v
+log "stored $NAME ($(aws ssm get-parameter --name "$NAME" --with-decryption --query 'length(Parameter.Value)' --output text) chars)"

live/aws/ssm.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# ssm.sh ID TIMEOUT_S SCRIPT [ARGS...]: runs a box script (one under a box/ folder) as root on box ID through SSM and
+# prints its output, e.g. ssm.sh ID 300 live/onair/box/loudness.sh 16:00 17:00
+source "$(dirname "$0")/common.sh"
+[ $# -ge 3 ] || die "usage: ssm.sh ID TIMEOUT_S SCRIPT [ARGS...]"
+ssm "$@"

live/aws/teardown.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# teardown.sh INSTANCE_ID NAME
+# Terminates INSTANCE_ID only if its Name tag is NAME and its purpose tag is LIVE_TAG_PURPOSE's, waits until it is
+# terminated and prints the state. An instance already terminated is reported, not an error.
+source "$(dirname "$0")/common.sh"
+ID=${1:?usage: teardown.sh INSTANCE_ID NAME}
+NAME=${2:?usage: teardown.sh INSTANCE_ID NAME}
+read -r state name purpose < <(aws ec2 describe-instances --instance-ids "$ID" --query \
+  'Reservations[0].Instances[0].[State.Name, Tags[?Key==`Name`].Value|[0], Tags[?Key==`purpose`].Value|[0]]' \
+  --output text)
+[ "$name" = "$NAME" ] || die "$ID is named '$name', not '$NAME'; not terminating"
+[ "$purpose" = "$TAG_PURPOSE" ] || die "$ID has purpose '$purpose', not '$TAG_PURPOSE'; not terminating"
+if [ "$state" != terminated ]; then
+  log "terminating $ID ($NAME, $state)"
+  aws ec2 terminate-instances --instance-ids "$ID" --query 'TerminatingInstances[0].CurrentState.Name' --output text >&2
+  aws ec2 wait instance-terminated --instance-ids "$ID"
+fi
+aws ec2 describe-instances --instance-ids "$ID" \
+  --query 'Reservations[0].Instances[0].[InstanceId,InstanceType,State.Name,Tags[?Key==`Name`].Value|[0]]' --output text

live/live.env.example

@@ -0,0 +1,43 @@
+# The live tooling's settings. Copy to live/live.env (git-ignored), fill it in; every laptop script sources it.
+# Another file: LIVE_ENV=/path/to/file before the command. No secret goes here: keys live in SSM (aws/put-secret.sh).
+
+# AWS: the profile is forced for every call when set (a shell's own AWS_PROFILE is then ignored); empty uses the
+# shell's credentials. With LIVE_AWS_ACCOUNT set, every script first checks the credentials are that account's.
+LIVE_AWS_PROFILE=
+LIVE_AWS_ACCOUNT=
+LIVE_AWS_REGION=us-west-2
+
+# Where boxes launch: a subnet with a route to the internet (apt, PyPI, GitHub, the model endpoint, X's ingest, the
+# viewer's map tiles), a security group (no inbound rule is needed: everything goes through SSM) and an instance
+# profile whose role has AmazonSSMManagedInstanceCore (it also reads the SecureString parameters below).
+LIVE_SUBNET=
+# launch-gpu.sh tries each of these subnets in turn (GPU capacity moves between zones); default LIVE_SUBNET.
+LIVE_SUBNETS=
+LIVE_SECURITY_GROUP=
+LIVE_INSTANCE_PROFILE=
+# launch-gpu.sh's instance types, in order.
+LIVE_GPU_TYPES="g6.4xlarge gr6.4xlarge g6.8xlarge"
+# Tags on every box; teardown.sh terminates only a box with this purpose tag. Owner default: $USER.
+LIVE_TAG_OWNER=
+LIVE_TAG_PURPOSE=airops-live
+
+# Transfers: code bundles, the server's data, results, clips and recordings go through this bucket (in LIVE_AWS_REGION)
+# under LIVE_S3_PREFIX, with presigned URLs, so the boxes need no S3 rights.
+LIVE_BUCKET=
+LIVE_S3_PREFIX=airops-live
+
+# The code: the box clones LIVE_REPO_URL and checks out your branch, shipped as a git bundle over LIVE_BASE_REF (a tag
+# of LIVE_REPO_URL that the branch contains), in a venv with LIVE_PINS.
+LIVE_REPO_URL=https://github.com/earakely-scale/agentenv-airops-plugin.git
+LIVE_BASE_REF=v0.5.0
+LIVE_PINS="agentenv-framework==0.9.1339 agentenv-framework-protocol==0.1.354 highspy==1.15.1 numpy==2.5.4 pandas==3.0.6 pyarrow==26.0.0 shapely==2.2.0 airportsdata==20260905"
+
+# The model: a LiteLLM-compatible endpoint (OpenAI-style chat), its key in SSM, and the model the desks play
+# (scripts/live_runner.toml's model when empty).
+LIVE_LITELLM_BASE_URL=
+LIVE_LITELLM_KEY_PARAM=/airops-live/litellm-key
+LIVE_MODEL=openai/gpt-6-astra
+
+# X (or any RTMP target): the server URL and the stream key, each a SecureString in SSM.
+LIVE_X_SERVER_PARAM=/airops-live/x-server
+LIVE_X_KEY_PARAM=/airops-live/x-key

live/onair/box/clip.sh

@@ -0,0 +1,19 @@
+# clip.sh PUT_URL START=SECONDS...: cuts each [UTC start (HH:MM:SS today, or YYYY-MM-DDTHH:MM:SS), seconds] piece
+# from the recording, joins them with 0.4 s fades (picture and sound), encodes H.264 + AAC and uploads the MP4.
+set -euo pipefail
+URL=$1; shift
+REC=/var/lib/airops-stream/rec/airops; D=$(mktemp -d); i=0; : > $D/list.txt
+for spec in "$@"; do
+  t=${spec%=*}; dur=${spec#*=}
+  if [ ${#t} -gt 8 ]; then want=$(date -u -d "$t" +%s); else want=$(date -u -d "${CLIPDAY:-$(date -u +%F)} $t" +%s); fi
+  seg=; for f in $(ls $REC | sort); do s=$(echo $f | sed -E 's/^([0-9-]+)_([0-9]+)-([0-9]+)-([0-9]+)-.*/\1 \2:\3:\4/'); st=$(date -u -d "$s" +%s); [ $st -le $want ] && { seg=$f; off=$((want-st)); }; done
+  i=$((i+1)); e=$(python3 -c "print($dur - 0.4)")
+  nice -n 10 ffmpeg -nostdin -loglevel error -ss $off -i $REC/$seg -t $dur \
+    -vf "fade=t=in:st=0:d=0.4,fade=t=out:st=$e:d=0.4" -af "afade=t=in:st=0:d=0.4,afade=t=out:st=$e:d=0.4,aresample=48000" \
+    -c:v h264_nvenc -preset p5 -rc vbr -cq 23 -b:v 8M -maxrate 10M -pix_fmt yuv420p -g 60 -c:a aac -b:a 192k -ar 48000 $D/p$i.mp4
+  echo "file '$D/p$i.mp4'" >> $D/list.txt; echo "piece $i: $t +${dur}s from $seg at ${off}s"
+done
+nice -n 10 ffmpeg -nostdin -loglevel error -f concat -safe 0 -i $D/list.txt -c copy -movflags +faststart $D/clip.mp4
+ffprobe -v error -show_entries format=duration,size -of compact $D/clip.mp4
+ffmpeg -nostdin -hide_banner -i $D/clip.mp4 -vn -af ebur128=peak=true -f null - 2>&1 | grep -A14 "Summary:" | grep -E "I:|Peak:"
+curl -fsS -X PUT --upload-file $D/clip.mp4 "$URL" && echo uploaded; rm -rf $D

live/onair/box/docker-pools.sh

@@ -0,0 +1,12 @@
+# docker-pools.sh: on the Deep Learning AMI (Docker preinstalled, with the NVIDIA runtime), adds Docker address pools
+# of /24s to its daemon.json, keeping the rest, so the env's networks don't run out; turns off apt's timers.
+set -euo pipefail
+f=/etc/docker/daemon.json; [ -f $f ] || echo '{}' > $f
+if ! jq -e '."default-address-pools"' $f >/dev/null; then
+  jq '. + {"default-address-pools":[{"base":"172.20.0.0/14","size":24}]}' $f > $f.new && mv $f.new $f
+  [ -z "$(docker ps -q)" ] || { echo "containers running: not restarting docker"; exit 1; }
+  systemctl restart docker
+fi
+cat $f; docker buildx version; docker compose version --short
+systemctl stop unattended-upgrades apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
+systemctl disable unattended-upgrades apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true

live/onair/box/fresh-install.sh

@@ -0,0 +1,18 @@
+set -euo pipefail
+mkdir -p /opt/airops-onair
+echo "$1" | base64 -d > /opt/airops-onair/fresh.py
+cat > /etc/systemd/system/airops-fresh.service <<'U'
+[Unit]
+Description=AirOps stream: the picture against the page; restarts a stale encoder
+After=airops-capture.service
+[Service]
+ExecStart=/usr/bin/python3 -u /opt/airops-onair/fresh.py /var/log/airops-stream/fresh.jsonl
+Restart=always
+RestartSec=10
+CPUWeight=20
+Nice=10
+[Install]
+WantedBy=multi-user.target
+U
+systemctl daemon-reload; systemctl enable -q --now airops-fresh; systemctl restart airops-fresh
+sleep 75; systemctl is-active airops-fresh; tail -2 /var/log/airops-stream/fresh.jsonl

live/onair/box/fresh.py

@@ -0,0 +1,70 @@
+"""The stream's picture against the page: each minute Chrome's screenshot of the page and the relay's next seconds of
+frames, at 96x54 grey; the closest frame's mean difference stays a few levels while the encoder sees the page. Three
+minutes in a row past FAR restart the encoder (the capture supervisor starts it again): x11grab can keep encoding a
+stale screen at full rate, which no frame count catches."""
+import base64
+import json
+import subprocess
+import sys
+import time
+import urllib.request
+
+import websocket
+
+OUT, EVERY, FAR, TIMES = sys.argv[1], 60.0, 12.0, 3
+W, H = 96, 54
+RELAY = "rtmp://127.0.0.1:1935/airops"
+
+
+def page() -> bytes:
+    pages = [x for x in json.load(urllib.request.urlopen("http://127.0.0.1:9222/json", timeout=10))
+             if x.get("type") == "page"]
+    ws = websocket.create_connection(pages[0]["webSocketDebuggerUrl"], timeout=20, suppress_origin=True)
+    try:
+        ws.send(json.dumps({"id": 1, "method": "Page.captureScreenshot", "params": {"format": "png"}}))
+        while True:
+            m = json.loads(ws.recv())
+            if m.get("id") == 1:
+                break
+    finally:
+        ws.close()
+    return grey(["-f", "png_pipe", "-i", "-"], base64.b64decode(m["result"]["data"]))
+
+
+def grey(args: list[str], stdin: bytes | None = None) -> bytes:
+    return subprocess.run(["ffmpeg", "-nostdin" if stdin is None else "-hide_banner", "-loglevel", "error", *args,
+                           "-vf", f"scale={W}:{H},format=gray", "-f", "rawvideo", "-"],
+                          input=stdin, capture_output=True, timeout=60).stdout
+
+
+def closest(shot: bytes, frames: bytes) -> float | None:
+    n = W * H
+    diffs = [sum(abs(a - b) for a, b in zip(shot, frames[i:i + n], strict=False)) / n
+             for i in range(0, len(frames) - n + 1, n)]
+    return round(min(diffs), 1) if diffs and len(shot) == n else None
+
+
+def main() -> None:
+    far = 0
+    while True:
+        t0 = time.time()
+        row = {"t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
+        try:
+            shot = page()
+            frames = grey(["-i", RELAY, "-t", "6", "-r", "10"])
+            row |= {"diff": closest(shot, frames), "frames": len(frames) // (W * H)}
+            far = far + 1 if row["diff"] is None or row["diff"] > FAR else 0
+            if far >= TIMES:
+                subprocess.run(["pkill", "-f", "x11grab -framerate"], timeout=10)
+                row["action"] = "restarted the encoder"
+                far = 0
+        except Exception as e:
+            row["error"] = f"{type(e).__name__}: {e}"[:200]
+        row["far"] = far
+        with open(OUT, "a") as f:
+            f.write(json.dumps(row) + "\n")
+        time.sleep(max(1.0, EVERY - (time.time() - t0)))
+
+
+if __name__ == "__main__":
+    main()