From ce759e1f5bd83bd4aa191612f206f6e581a1ad7d Mon Sep 17 00:00:00 2001 From: Owen Price Skelly <21372141+OwenPriceSkelly@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:44:20 -0500 Subject: [PATCH 1/2] Add usage-stats collection endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a proxy-authed `usage` POST endpoint (parallel to `manifest`) that ingests the monthly rollup rows produced by `rootstock usage report` (Garden-AI/rootstock#47), plus a public GET /usage on the dashboard app. Rollups are stored per cluster per month (/data/usage//.json) so a push replaces only the months it contains — server-side history survives clusters whose local spool is purged (e.g. Perlmutter PSCRATCH). User hash lists are stripped on ingest: only derived counts leave a cluster. Co-Authored-By: Claude Fable 5 --- README.md | 9 +++++++++ modal_app.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/README.md b/README.md index a389425..899a9bb 100644 --- a/README.md +++ b/README.md @@ -12,19 +12,28 @@ Rootstock Admin provides a central dashboard for monitoring Rootstock installati |-------------|----------|-----| | Dev | Dashboard | https://garden-ai-dev--rootstock-admin-dashboard.modal.run | | Dev | Manifest API | https://garden-ai-dev--rootstock-admin-manifest.modal.run | +| Dev | Usage API | https://garden-ai-dev--rootstock-admin-usage.modal.run | | Prod | Dashboard | https://garden-ai-prod--rootstock-admin-dashboard.modal.run | | Prod | Manifest API | https://garden-ai-prod--rootstock-admin-manifest.modal.run | +| Prod | Usage API | https://garden-ai-prod--rootstock-admin-usage.modal.run | ### Manifest API `POST` Receives and stores environment manifests from Rootstock clusters. Requires proxy authentication. **Payload:** JSON manifest containing cluster name, rootstock version, Python version, maintainer info, and environment details. +### Usage API `POST` +Receives anonymous usage rollups from Rootstock clusters ([rootstock#47](https://github.com/Garden-AI/rootstock/issues/47)). Requires proxy authentication. + +**Payload:** `{"cluster": "", "rows": [...]}` where each row is a monthly rollup row as produced by `rootstock usage report` — keyed by `month`/`cluster`/`env`/`checkpoint`/`device`/`client` with `sessions`, `n_calculations`, `duration_s`, and `unique_users` counts. Rollups are stored per cluster per month; a push replaces only the months it contains, so previously reported months persist even if the cluster's local spool has been purged. User hashes never leave the cluster — only derived counts — and any `users` field sent by mistake is dropped on ingest. + ### Dashboard `GET` Public dashboard displaying all registered clusters and their environments. Shows: - Cluster metadata (name, version, root path, maintainer) - Environment status, dependencies, source code, and checkpoints +`GET /usage` returns every stored usage rollup: `{"usage": [{"cluster": ..., "month": ..., "rows": [...]}, ...]}`. + ## Development ### Prerequisites diff --git a/modal_app.py b/modal_app.py index 0797ac0..81d572d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -22,6 +22,50 @@ def manifest(manifest: dict): return manifest +@app.function(volumes={"/data": vol}) +@modal.fastapi_endpoint(method="POST", requires_proxy_auth=True) +def usage(payload: dict): + """Ingest a cluster's usage rollups: {"cluster": str, "rows": [...]}. + + Rows are the aggregated rollup rows produced by `rootstock usage report` + (keyed by month/cluster/env/checkpoint/device/client, with sessions, + n_calculations, duration_s, unique_users). Rows are stored per month and + a push only replaces the months it contains, so history survives even if + a cluster's spool loses old months (e.g. scratch purge policies). + """ + import json + import os + import re + from collections import defaultdict + + from fastapi import HTTPException + + cluster = payload.get("cluster") + rows = payload.get("rows") + if not isinstance(cluster, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", cluster): + raise HTTPException(422, "invalid or missing 'cluster'") + if not isinstance(rows, list): + raise HTTPException(422, "'rows' must be a list of rollup rows") + + by_month = defaultdict(list) + for row in rows: + if not isinstance(row, dict) or not re.fullmatch(r"\d{4}-\d{2}", str(row.get("month"))): + raise HTTPException(422, "every row needs a 'month' of the form YYYY-MM") + # Only derived counts ever leave a cluster; drop user hashes if a + # client ever sends them by mistake. + row.pop("users", None) + by_month[row["month"]].append(row) + + vol.reload() + os.makedirs(f"/data/usage/{cluster}", exist_ok=True) + for month, month_rows in by_month.items(): + with open(f"/data/usage/{cluster}/{month}.json", "w") as f: + json.dump({"cluster": cluster, "month": month, "rows": month_rows}, f) + + vol.commit() + return {"cluster": cluster, "months": sorted(by_month)} + + @app.function(volumes={"/data": vol}, min_containers=1) @modal.asgi_app() def dashboard(): @@ -45,4 +89,15 @@ def get_manifests(): manifests = [json.loads(p.read_text()) for p in Path("/data").glob("*.json")] return {"manifests": manifests} + @web_app.get("/usage") + def get_usage(): + import json + from pathlib import Path + + vol.reload() + rollups = [ + json.loads(p.read_text()) for p in sorted(Path("/data/usage").glob("*/*.json")) + ] + return {"usage": rollups} + return web_app From ec4f4f0368d62c923a1f33e1976aedc951204ad2 Mon Sep 17 00:00:00 2001 From: Owen Price Skelly <21372141+OwenPriceSkelly@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:12:58 -0500 Subject: [PATCH 2/2] Sort imports and format to satisfy ruff CI Co-Authored-By: Claude Fable 5 --- modal_app.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modal_app.py b/modal_app.py index 81d572d..1a61606 100644 --- a/modal_app.py +++ b/modal_app.py @@ -1,6 +1,7 @@ -import modal import os +import modal + app = modal.App("rootstock-admin") app.image = modal.Image.debian_slim().pip_install("fastapi[standard]") @@ -49,7 +50,9 @@ def usage(payload: dict): by_month = defaultdict(list) for row in rows: - if not isinstance(row, dict) or not re.fullmatch(r"\d{4}-\d{2}", str(row.get("month"))): + if not isinstance(row, dict) or not re.fullmatch( + r"\d{4}-\d{2}", str(row.get("month")) + ): raise HTTPException(422, "every row needs a 'month' of the form YYYY-MM") # Only derived counts ever leave a cluster; drop user hashes if a # client ever sends them by mistake. @@ -96,7 +99,8 @@ def get_usage(): vol.reload() rollups = [ - json.loads(p.read_text()) for p in sorted(Path("/data/usage").glob("*/*.json")) + json.loads(p.read_text()) + for p in sorted(Path("/data/usage").glob("*/*.json")) ] return {"usage": rollups}