diff --git a/.github/workflows/node-fleet-check.yml b/.github/workflows/node-fleet-check.yml index 15cbed8..2bf7f9e 100644 --- a/.github/workflows/node-fleet-check.yml +++ b/.github/workflows/node-fleet-check.yml @@ -1,5 +1,7 @@ -name: Node fleet setup +name: Node maintenance setup check 'on': + schedule: + - cron: '0 3 * * 1' workflow_call: inputs: inventory-repository: @@ -51,7 +53,7 @@ jobs: GH_TOKEN: ${{ secrets.read-token || github.token }} with: inventory-repository: ${{ inputs.inventory-repository }} - inventory-path: ${{ inputs.inventory-path }} + inventory-path: ${{ inputs.inventory-path || 'defaults/node-repositories.json' }} inventory-ref: ${{ inputs.inventory-ref }} - name: Retain the setup report if: ${{ always() && !cancelled() }} diff --git a/README.md b/README.md index f257a99..957dd89 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ commit. Supports GitHub.com. | Workflow | Use it to | Get started | | --- | --- | --- | | Node dependency maintenance | Attempt compatible security fixes, optionally propose routine upgrades, validate the project, and maintain one dependency PR. | [Setup guide](docs/node-maintenance.md) | -| Node fleet setup | Check a list of maintenance installations for broken setup or configuration drift, and maintain one central issue per list. | [Fleet setup guide](docs/node-fleet-setup.md) | +| Node maintenance setup check | Check a list of maintenance installations for broken setup or configuration drift, and maintain one central issue per list. | [Fleet setup guide](docs/node-fleet-setup.md) | The workflows are independent. Install maintenance in each Node repository you want to maintain. Run the fleet checker from one repository to monitor those @@ -46,7 +46,10 @@ that should follow it. You do not need to register either file here. The [fleet checker](docs/node-fleet-setup.md) inspects each default branch, honors local overrides, and reports failed or unverifiable setup in the host repository. It reuses one issue per inventory, closes it when every static check passes, and -reopens it on recurrence. Run it manually or add your own caller schedule. +reopens it on recurrence. This repository checks ten public repositories weekly +on Mondays at 03:00 UTC. Keep private inventories and reports in a private host; +other hosts choose their own caller schedule. Checks run five repositories at a time in one +runner, with one combined report and issue. ## Documentation diff --git a/defaults/node-repositories.json b/defaults/node-repositories.json index 30f0bbf..0886e84 100644 --- a/defaults/node-repositories.json +++ b/defaults/node-repositories.json @@ -6,6 +6,14 @@ }, "repositories": [ "hydra-ecosystem/hydra", - "hydra-ecosystem/omegaconf" + "hydra-ecosystem/omegaconf", + "omry/arbiter", + "omry/cli.dev", + "omry/hoodlefinance", + "omry/omegaflow", + "omry/reploy", + "omry/yadan.net", + "omry/airport-time-space", + "omry/stale-buffer-guard" ] } diff --git a/docs/node-fleet-setup.md b/docs/node-fleet-setup.md index 88d5ef5..0fc9029 100644 --- a/docs/node-fleet-setup.md +++ b/docs/node-fleet-setup.md @@ -46,13 +46,15 @@ Use a branch ref for live defaults or a full commit SHA for fixed policy. The inventory ref used to fetch this list is independent of the defaults ref inside it. See [shared defaults](shared-defaults.md) for inheritance rules. -This repository's [inventory](../defaults/node-repositories.json) lists Hydra -and OmegaConf. Other maintainers can own their own files in their own repositories; -no registration here is required. +This repository's [inventory](../defaults/node-repositories.json) lists ten public +repositories: Hydra, OmegaConf, and eight personal Node projects. Repositories +that are not installed yet remain listed so their missing setup stays visible. +Other maintainers can keep their own inventories in their own repositories; no +registration here is required. ## 2. Choose where the checker and issue run -To check this repository's inventory, use **Actions → Node fleet setup → Run +To check this repository's inventory, use **Actions → Node maintenance setup check → Run workflow** after the workflow is merged. Its default inputs select `defaults/node-repositories.json` on this repository's default branch. Set `inventory-repository`, `inventory-path`, and `inventory-ref` to read another list. @@ -63,8 +65,10 @@ To host the check elsewhere, add `.github/workflows/node-fleet-setup.yml` to tha repository's default branch: ```yaml -name: Node fleet setup +name: Node maintenance setup check 'on': + schedule: + - cron: '0 3 * * 1' workflow_dispatch: permissions: contents: read @@ -85,7 +89,11 @@ Replace `REVIEWED_FULL_COMMIT_SHA` with a reviewed full SHA that contains `node-fleet-check.yml`. Use your inventory location. Enable issues in the host repository. Only the host needs issue-write permission; listed repositories do not need a new workflow, extra write permission, or an issue opened in them. -Reports have the host repository's visibility. +Reports have the host repository's visibility. Keep private-repository inventories +and their checker in a host that will remain private if names, checked commits, +and diagnostics must stay private. A private inventory alone does not make +issues or run summaries in a public host private. Changing the host to public +also requires reviewing its inventory history and existing reports. For public repositories, the built-in token is sufficient. To read private listed repositories or a private inventory in another repository, add a host @@ -103,7 +111,7 @@ maintenance resolver. No private key or secret content is inspected. ## 3. Run and inspect results -Open **Actions → Node fleet setup → Run workflow** in the host. From a checkout +Open **Actions → Node maintenance setup check → Run workflow** in the host. From a checkout of that repository, you can also run: ```sh @@ -122,13 +130,19 @@ its exact checked default-branch commit, and an actionable diagnostic: | `failed` | Setup violates a checked requirement, such as mismatched Node policy or caller schedules. | | `unverified` | A required observation was unavailable, such as inaccessible repository data, API limits, or an incomplete tree. It is not a pass. | +The scanner checks up to five repositories concurrently in one runner, while +keeping results in inventory order. Each repository has isolated temporary files. +Shared defaults and pinned repository snapshots are reused across checks. The scan continues after an individual repository fails. Any failed or unverified -result fails the scanner job and opens or updates one **Node fleet setup** issue +result fails the scanner job and opens or updates one **Node maintenance setup check** issue in the host. The reporter still runs after scan failures. A fully passing scan closes that issue; recurrence reopens it. Repeated failures update the body without per-run comments. An invalid or unreadable inventory also reports against that list when its repository/path identity is valid. +The issue starts with a results table and links to each repository. Exact commit +identities are in the expandable **Checked revisions** section. + Issue identity uses the inventory repository and path, so changing its contents or ref keeps the same issue. Different paths have separate issues. Runs are serialized in the host to avoid competing issue creation. Existing ambiguous @@ -173,10 +187,11 @@ versions, credential contents, effective PR-creation permissions, dependency health, and project builds are labeled **not tested**. Use a manual maintenance run to exercise installation, project checks, and publication. -## Optional: schedule the fleet check +## Change the weekly schedule -There is no automatic fleet-check schedule by default. To check weekly, add a -schedule alongside `workflow_dispatch` in the host caller: +This repository checks its inventory every Monday at 03:00 UTC (11:00 in +Singapore). The caller example above uses the same weekly schedule. +To change it, edit the `schedule` block in the host workflow: ```yaml 'on': @@ -185,6 +200,8 @@ schedule alongside `workflow_dispatch` in the host caller: workflow_dispatch: ``` -This runs Mondays at 03:00 UTC. It is independent of the security and upgrade -cadences in each maintenance installation; do not add the fleet cron to those -installations' maintenance configuration. +Remove the `schedule` block to run only manually. Keep `workflow_dispatch` for +on-demand checks. A reusable workflow uses its caller's schedule; it does not +inherit this repository's cron. This schedule is independent of the security +and upgrade cadences in each maintenance installation; do not add the fleet +cron to those installations' maintenance configuration. diff --git a/node/fleet-check/action.yml b/node/fleet-check/action.yml index 16c7b23..36e73e6 100644 --- a/node/fleet-check/action.yml +++ b/node/fleet-check/action.yml @@ -1,4 +1,4 @@ -name: Check Node fleet setup +name: Check Node maintenance setup description: Inspect listed installations as data without running project commands. inputs: inventory-repository: diff --git a/node/fleet-report/action.yml b/node/fleet-report/action.yml index 95df3da..37c7b55 100644 --- a/node/fleet-report/action.yml +++ b/node/fleet-report/action.yml @@ -1,4 +1,4 @@ -name: Report Node fleet setup +name: Report Node maintenance setup description: Maintain one setup issue per inventory in the calling repository. runs: using: composite diff --git a/node/runtime/fleet.py b/node/runtime/fleet.py index 4b9bd67..e51b500 100644 --- a/node/runtime/fleet.py +++ b/node/runtime/fleet.py @@ -12,14 +12,17 @@ import urllib.error import urllib.parse import urllib.request +from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePosixPath from tempfile import TemporaryDirectory +from threading import Lock import yaml PROJECT = runpy.run_path(str(Path(__file__).with_name("project.py"))) CONFIGURATION = PROJECT["CONFIGURATION"] MAX_REPOSITORIES = 25 +MAX_CONCURRENT_REPOSITORIES = 5 MAX_FILE_BYTES = 2_000_000 CONFIG_PATH = ".github/node-maintenance.json" CALLER_PATH = ".github/workflows/dependency-audit.yml" @@ -58,8 +61,16 @@ class GitHub: def __init__(self, token=""): self.token = token self.snapshots = {} + self.snapshot_locks = {} + self.snapshot_locks_guard = Lock() + self.defaults_lock = Lock() # Defaults remain public and unauthenticated, as in the individual audit. - self.read_defaults = functools.cache(CONFIGURATION["public_read"]) + self.cached_defaults = functools.cache(CONFIGURATION["public_read"]) + + def read_defaults(self, url, accept="application/json"): + # Resolve each shared ref once, even when several repositories need it. + with self.defaults_lock: + return self.cached_defaults(url, accept) @functools.cache def get(self, path): @@ -99,9 +110,12 @@ def get(self, path): def snapshot(self, repository, ref=""): repository = repository.lower() key = (repository, ref) - if key not in self.snapshots: - self.snapshots[key] = Repository(self, repository, ref) - return self.snapshots[key] + with self.snapshot_locks_guard: + lock = self.snapshot_locks.setdefault(key, Lock()) + with lock: + if key not in self.snapshots: + self.snapshots[key] = Repository(self, repository, ref) + return self.snapshots[key] class Repository: @@ -564,9 +578,14 @@ def check_fleet(github, source): repository.read(source["path"]), "fleet inventory" ) ) - for name in settings["repositories"]: - report["results"].append( - check_repository(github, name, settings["defaults"]) + with ThreadPoolExecutor( + max_workers=min(MAX_CONCURRENT_REPOSITORIES, len(settings["repositories"])) + ) as executor: + report["results"] = list( + executor.map( + lambda name: check_repository(github, name, settings["defaults"]), + settings["repositories"], + ) ) report["status"] = ( "ok" @@ -585,7 +604,7 @@ def escape(value): return html.escape(str(value)).replace("|", "|").replace("\n", " ") lines = [ - "## Node fleet setup", + "## Node maintenance setup check", "", "| Repository | Static setup | Detail |", "| --- | --- | --- |", @@ -627,7 +646,7 @@ def main(): output.write(text) print(text) if report["status"] != "ok": - parser.exit(1, "Fleet setup needs attention; inspect the report.\n") + parser.exit(1, "Node maintenance setup needs attention; inspect the report.\n") if __name__ == "__main__": diff --git a/node/runtime/fleet_issue.py b/node/runtime/fleet_issue.py index 76c7a2a..a1d0a49 100644 --- a/node/runtime/fleet_issue.py +++ b/node/runtime/fleet_issue.py @@ -115,27 +115,73 @@ def diagnostic(value): truncated = True return html.escape(value[:limit] + TRUNCATION_SUFFIX) + def cell(value): + return ( + re.sub( + r"[\\`*_\[\]~!|]", + lambda match: f"&#{ord(match[0])};", + value, + ) + .replace("\r\n", "\n") + .replace("\r", "\n") + .replace("\n", "
") + ) + + counts = { + status: sum(item["status"] == status for item in report["results"]) + for status in ("ok", "failed", "unverified") + } + statuses = { + "ok": "✅ Passed", + "failed": "⚠️ Needs attention", + "unverified": "❔ Unverified", + } + headline = ( + " · ".join( + f"{counts[status]} {label}" + for status, label in ( + ("failed", "need attention"), + ("ok", "passed"), + ("unverified", "unverified"), + ) + if counts[status] + ) + or "No repositories verified" + ) lines = [ marker(source), "", - "## Node fleet setup", + "## Node maintenance setup check", + "", + f"**{headline}**", "", f"Inventory: {text(source['repository'])}/{text(source['path'])}", - f"Revision: {text(source['ref'])} @ {text(source['commit'] or 'unverified')}", "", ] if report["diagnostic"]: - lines.append("
" + diagnostic(report["diagnostic"]) + "
") + lines.extend(["
" + diagnostic(report["diagnostic"]) + "
", ""]) + lines.extend(["| Repository | Setup | Detail |", "| --- | --- | --- |"]) for item in report["results"]: - lines.extend( - [ - f"### {text(item['repository'])}: {text(item['status'])}", - "", - "
" + diagnostic(item["diagnostic"]) + "
", - f"Checked commit: {text(item['commit'] or 'unverified')}", - "", - ] + lines.append( + f"| [{cell(text(item['repository']))}](https://github.com/{item['repository']}) | {statuses[item['status']]} | {cell(diagnostic(item['diagnostic']))} |" + ) + lines.extend( + [ + "", + "
", + "Checked revisions", + "", + f"Inventory: {text(source['ref'])} @ {text(source['commit'] or 'unverified')}", + "", + "| Repository | Commit |", + "| --- | --- |", + ] + ) + for item in report["results"]: + lines.append( + f"| {cell(text(item['repository']))} | {text(item['commit'] or 'unverified')} |" ) + lines.extend(["", "
", ""]) if truncated: lines.extend( [ @@ -174,7 +220,7 @@ def diagnostic(value): return ISSUES["maintain_issue"]( repository, marker=marker(source), - title="Node fleet setup: " + source["path"][:180], + title="Node maintenance setup check: " + source["path"][:180], body=body, failing=report["status"] != "ok", request=request, diff --git a/node/tests/test_fleet_setup.py b/node/tests/test_fleet_setup.py index 9d7fecf..06e53e0 100644 --- a/node/tests/test_fleet_setup.py +++ b/node/tests/test_fleet_setup.py @@ -6,7 +6,9 @@ import json import runpy import subprocess +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from threading import Barrier, Event, Lock import pytest import yaml @@ -168,6 +170,113 @@ def forbidden(*args, **kwargs): assert all(call.startswith("repos/") for call in github.calls) +def test_fleet_checks_five_at_a_time_and_retains_order_and_failures(monkeypatch): + github = GitHub() + names = [f"owner/project-{number}" for number in range(9)] + github.repos["owner/settings"]["defaults/repos.json"] = data( + {"defaults": SOURCE, "repositories": names} + ) + ready = Barrier(6) + release = Event() + guard = Lock() + active = maximum = 0 + + def check(github, name, expected): + nonlocal active, maximum + assert expected == SOURCE + with guard: + active += 1 + maximum = max(maximum, active) + try: + if names.index(name) < 5: + ready.wait(timeout=5) + assert release.wait(timeout=5) + return { + "repository": name, + "commit": SHA, + "status": "failed" if name == names[2] else "ok", + "diagnostic": "example failure" if name == names[2] else "passed", + } + finally: + with guard: + active -= 1 + + monkeypatch.setitem(FLEET["check_fleet"].__globals__, "check_repository", check) + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + FLEET["check_fleet"], + github, + { + "repository": "owner/settings", + "path": "defaults/repos.json", + "ref": "main", + }, + ) + try: + ready.wait(timeout=5) + finally: + release.set() + report = future.result(timeout=5) + assert maximum == 5 + assert [item["repository"] for item in report["results"]] == names + assert report["status"] == "failed" + assert report["results"][2]["diagnostic"] == "example failure" + + +def test_concurrent_checks_share_one_pinned_repository_snapshot(monkeypatch): + github = FLEET["GitHub"]() + entered = Event() + release = Event() + ready = Barrier(6) + constructions = [] + + def repository(client, name, ref): + constructions.append((name, ref)) + entered.set() + assert release.wait(timeout=5) + return object() + + monkeypatch.setitem(FLEET["GitHub"].snapshot.__globals__, "Repository", repository) + + def snapshot(): + ready.wait(timeout=5) + return github.snapshot("OWNER/PROJECT", "main") + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(snapshot) for _ in range(5)] + try: + ready.wait(timeout=5) + assert entered.wait(timeout=5) + finally: + release.set() + snapshots = [future.result(timeout=5) for future in futures] + assert constructions == [("owner/project", "main")] + assert all(item is snapshots[0] for item in snapshots) + + +def test_concurrent_defaults_reads_resolve_shared_ref_once(monkeypatch): + calls = [] + + def public(url, accept): + calls.append((url, accept)) + return b"immutable result" + + monkeypatch.setitem(FLEET["CONFIGURATION"], "public_read", public) + github = FLEET["GitHub"]() + ready = Barrier(5) + + def read(): + ready.wait(timeout=5) + return github.read_defaults( + "https://api.github.com/repos/owner/settings/commits/main" + ) + + with ThreadPoolExecutor(max_workers=5) as executor: + results = list(executor.map(lambda _: read(), range(5))) + assert results == [b"immutable result"] * 5 + assert len(calls) == 1 + + @pytest.mark.parametrize( "case", [ @@ -323,7 +432,8 @@ def test_one_issue_per_inventory_updates_closes_and_reopens(): ) report["status"] = "failed" report["results"][0].update( - status="failed", diagnostic="Node mismatch " + status="failed", + diagnostic="Node mismatch | extra\n[link]", ) def update(): @@ -337,6 +447,13 @@ def update(): assert update()["operation"] == "created" assert update()["operation"] == "unchanged" assert "