diff --git a/.github/workflows/cao.json b/.github/workflows/cao.json index b85363b44c..87aec1d6bd 100644 --- a/.github/workflows/cao.json +++ b/.github/workflows/cao.json @@ -422,6 +422,30 @@ }, "repository": "github/gh-aw", "workflow": "daily-file-diet" + }, + { + "api-requests": 111, + "acquisition": { + "requirements": [ + { + "cache": "invocation", + "pool": "core", + "primitive": "github.dependabot-alert-lifecycle.v1", + "scope": "github/gh-aw", + "source": "github" + }, + { + "cache": "invocation", + "pool": "search", + "primitive": "github.dependabot-pull-request-lifecycle.v1", + "scope": "github/gh-aw", + "source": "github" + } + ] + }, + "collector": "dependabot/operational-value/dependabot-update-planner.mjs", + "repository": "github/gh-aw", + "workflow": "dependabot-update-planner" } ] } diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8cc2905b10..9e753f7654 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1926,24 +1926,16 @@ not authorize cross-repository evidence access. Host-neutral source builds use `server/Dockerfile` with explicit version and revision build arguments; deployment promotion and rollback belong to the application host. The source build - produces a lean admission-only dashboard target and a collector target with - Node, GitHub CLI, a checksum-verified policy-pinned gh-aw binary, and the - Activity projection catalog. The collector target also generates a - deterministic control-plane inventory from the exact checked-out source and - packages only that inventory plus the shared policy resolver required at - runtime; it does not restore a second editable campaign catalog. Before each - canonical ingestion replacement, a private collection role resolves the reviewed deployment - policy, overlays the exact operational enrollment set, and uses a short-lived - installation token for the control repository to enrich and retain - `control-settings.json`, `control-plane-inventory.json`, and - `inventory-sources.json` in the evidence lake. The first activation fails - closed without valid campaign inventory; a later transient refresh failure - may retain the previous valid inventory. The internet-facing dashboard holds - the webhook secret but no collection private key. Unexposed collector and - idempotent backfill services hold the read-only GitHub App key and share a - persistent evidence volume. The collection profile leaves - `CAO_SOURCE_DIRECTORY` unset, while one verified activity artifact remains an - external read-only recovery volume for rollback to the snapshot profile. + produces a lean admission-only dashboard target and a private Go collector + target with Git for the reviewed Value Git transport, but no Node, GitHub CLI, + gh-aw or Activity projection catalog. PostgreSQL retains the exact reviewed + inventory configuration and source-epoch-bound evidence; collector and + backfill roles share these durable stores rather than an evidence volume. + The internet-facing dashboard holds the webhook secret but no collection + private key. Unexposed collector and idempotent backfill services hold the + read-only GitHub App key. The collection profile leaves + `CAO_SOURCE_DIRECTORY` unset; a separately selected Actions snapshot profile + can still mount a verified activity artifact read-only. Locally source-built images are not admitted through the package workflow's vulnerability scans, hardening check, SBOM, or artifact attestations; those guarantees continue to apply only to published packages. @@ -1976,15 +1968,19 @@ not authorize cross-repository evidence access. Webhook/control deliveries carry source epochs and fresh queue receipts. Independent safe-output dispositions survive run reprojection, while durable Value invalidations remain hints, not acquired Value evidence. - The separately selected `NativeValues` lane reuses reviewed Node Value - programs and therefore still requires its catalog and policy assets. Its + The separately selected `NativeValues` lane evaluates the reviewed Value + policy and supported collector in Go, using Git only for its declared Git + acquisition requirements. Optimization uses the canonical PostgreSQL query + boundary; the separately registered Dependabot collector acquires bounded + alert and pull-request lifecycles through the shared native quota bridge + and publishes three atomic cutoff packets. The local evaluator consumes + their canonical facts; incomplete cutoffs remain missing, never zero. Its private invocation scratch is disposable: PostgreSQL restores the original source-epoch-bound packet/proof snapshot, fences publication against that starting packet, and erases private proof/publication ownership atomically on withdrawal. Those selected-feature dependencies are not dependencies of native - run acquisition, projection or trace admission. The existing remote Value - transport remains Linux-only; local query execution has its separate owned - canonical PostgreSQL boundary. + run acquisition, projection or trace admission; local query execution has + its separate owned canonical PostgreSQL boundary. Native archive admission binds the original dispatch epoch before lossy normalization. Its PostgreSQL metadata-only privacy ledger withdraws visibility with source ownership and durably retries epoch-specific Blob retirement; @@ -2003,8 +1999,10 @@ not authorize cross-repository evidence access. exercise real PostgreSQL, replica restoration and worker execution without an executable PATH or legacy files. Doctor shares native setting and selected Value dependency validation, and does not probe legacy lakes or projection - locks for native deployments. Existing hosted/container defaults remain - the separate legacy producer. + locks for native deployments. App-backed collection without native selection + now fails at startup; only the no-App Actions snapshot profile remains the + default. Unsupported Value collector registrations fail closed rather than + launching the former JavaScript runtime. See `specs/server-ingestion.md` section 2.4. - Browsers and external clients never receive Redis or Postgres endpoints or credentials. Dashboard source replacement is transactional in Postgres; a diff --git a/dependabot/operational-value/dependabot-update-planner.mjs b/dependabot/operational-value/dependabot-update-planner.mjs index 2e82c2ee32..c82bcf9ba0 100644 --- a/dependabot/operational-value/dependabot-update-planner.mjs +++ b/dependabot/operational-value/dependabot-update-planner.mjs @@ -39,6 +39,24 @@ export const definition = { zeroRule: "Complete lifecycle evidence with no alert or pull request open at the cutoff records zero.", missingRule: "Inaccessible, truncated, malformed, or incomplete lifecycle evidence records missing rather than zero.", }, + acquisition: { + requirements: [ + { + source: "github", + pool: "core", + primitive: "github.dependabot-alert-lifecycle.v1", + scope: "github/gh-aw", + cache: "invocation", + }, + { + source: "github", + pool: "search", + primitive: "github.dependabot-pull-request-lifecycle.v1", + scope: "github/gh-aw", + cache: "invocation", + }, + ], + }, model: { architecture: "Problem-first repository-risk measurement independent of gh-aw outputs, recommendations, runs, and engagement.", recommendation: "Use total open Dependabot security alerts as the primary outcome and retain critical/high alerts and the Dependabot pull-request queue as separate diagnostics. Count improvement regardless of actor and describe adoption-timed movement as association.", diff --git a/docs/deployment-azure.md b/docs/deployment-azure.md index a6d90b8d1b..9a114a58bb 100644 --- a/docs/deployment-azure.md +++ b/docs/deployment-azure.md @@ -87,9 +87,9 @@ digests, and locks the version-plus-config-hash tags against writes and deletes. It then deploys the full template with separate Functions, provisioning, and ingestion digest parameters, not those tags. After deployment it reads back the configured digest references for all three targets before starting either job; -missing, mismatched, or additional job containers fail closed. Regenerate -`server/azure/main.json` from the current Bicep source before planning: apply -rejects templates without the three digest parameters before any Azure call. +missing, mismatched, or additional job containers fail closed. Regenerate `server/azure/main.json` from the reviewed Bicep sources before +planning, then review the exact compiled template. Apply rejects templates +without the three image digest parameters before any Azure call. New managed-identity Key Vault grants can take time to propagate. Apply retries the exact reviewed ARM request at most three times only when every nested failure is the recognized job secret-fetch error in the ingestion or @@ -988,23 +988,44 @@ You configure traces for orchestrators and workers separately, in the control re ## Using the optional collection profile -By default, the server serves the data that the Activity workflow publishes and doesn't collect anything itself. To have Azure collect the data instead, set the `collectorImage` parameter. The `server/azure/collector.bicep` template then adds: +By default, the server serves the data that the Activity workflow publishes and +doesn't collect anything itself. To have Azure collect instead, set +`collectorImage` to an image built from the reviewed `collector-runtime` Docker +target and supply `collectorNativeInventoryDigest` at deployment. The +authoritative `server/azure/main.bicep` and `server/azure/collector.bicep` +sources compile to `server/azure/main.json`. The optional module adds: - Workers and backfill in the same private Container Apps environment as ingestion. -- A fixed number of collection workers sharing the PostgreSQL operational queue. Set `collectorWorkerReplicas` to the required replica count; the default is `1`. +- A fixed number of native Go collection workers sharing the same deployment-scoped PostgreSQL operational queue as the Function App. Set `collectorWorkerReplicas` to the required replica count; the default is `1`. - A backfill job. - A user-assigned managed identity with access to Key Vault and `AcrPull` on this deployment's registry. Build the collector image into that registry. -- An Azure Files share for collected evidence. The SKU is set by `collectorLakeStorageSku` (default `Premium_LRS`). -The optional evidence lake requires a separate file-storage account with shared -access across workers and evidence-retention settings; Functions runtime storage -has different access requirements. With collection disabled, the baseline -creates only one storage account. It never creates a second Insights component +No evidence lake or Azure Files mount is used by native acquisition. The +baseline and optional collector do not create an additional Insights component or workspace. -The collection profile requires these parameters: `collectorGithubAppId`, `collectorPrivateKey`, `githubWebhookSecret`, `githubAdminUsers`, and `collectorControlRepository`. - -In this profile, the Function App only admits webhooks (`CAO_COLLECT_ADMIT_ONLY=true`). It verifies and deduplicates deliveries to `POST /api/github/webhook`, then queues the work. It never holds the app's private key. +The collection profile requires `collectorGithubAppId`, `collectorPrivateKey`, +`githubWebhookSecret`, `githubAdminUsers`, `collectorControlRepository`, and +`collectorNativeInventoryDigest`. Provision the exact immutable reviewed +inventory bytes using `Acquisition.PinInventoryConfiguration` in the shared +canonical PostgreSQL dataset namespace (`azure-dashboard` by default) +**before starting workers**; the Azure template does not generate or +provision that snapshot, and this guide does not provide an automated +inventory-provisioning command. +Supply its lowercase SHA-256 digest, not a local inventory path or an +automatically discovered inventory. A missing digest fails optional-module +parameter validation; an invalid, unprovisioned, or mismatched digest fails +worker startup rather than using another snapshot or falling back to Actions. +The image must include `/app/cao-dashboard` and +`/app/queries/database.json`. The separate Value lane is not enabled by +this template (`CAO_COLLECT_NATIVE_VALUES` is unset). + +In this profile the Function App selects `CAO_COLLECT_NATIVE=true` but only +admits webhooks (`CAO_COLLECT_ADMIT_ONLY=true`). It verifies and deduplicates +deliveries to `POST /api/github/webhook`, then queues the work in PostgreSQL. +It never holds the app's private key or requires inventory bytes. Workers and +backfill select native acquisition with the same operational namespace and +the exact reviewed digest. The approved ingress path must also permit GitHub webhook delivery. Disabled public access or client-only allow rules can block it; do not widen access just diff --git a/docs/operational-value.md b/docs/operational-value.md index 7211f31d02..528c44be1c 100644 --- a/docs/operational-value.md +++ b/docs/operational-value.md @@ -147,16 +147,16 @@ capture. Ordinary Activity projection and bounded local Value remain independent. The default Actions profile has no inbound webhook receiver and continues its existing scheduled collection. -The hosted worker keeps the installation token outside the collector process. -Its repository-confined API bridge admits REST/core, search and GraphQL +The hosted Go worker keeps the installation token out of policy, packets and +private proof. Its repository-confined API bridge admits REST/core, search and GraphQL requests against the same shared installation quota used by Activity; GraphQL is limited to the reviewed File Diet query shape and its points are accounted separately from the reviewed API request count. Unsupported query shapes fail closed rather than extending a registration's authority. Missing quota evidence, exhausted capacity, changed policy/enrollment, incomplete collection or superseded accepted evidence defers publication. On success -only the validated original packet is atomically published to PostgreSQL. -Private per-pair proof retains the last 30 attempts' API request count, wall +only validated original packets are atomically published to PostgreSQL. +File Diet's private per-pair proof retains the last 30 attempts' API request count, wall time and received/output bytes, including failures, plus cumulative observed GraphQL points and the number of requests whose point cost could not be verified. An unknown point cost is not reported as zero; quota preflight is @@ -174,7 +174,7 @@ invalid result leaves previous evidence and its capture/cutoff timestamps intact it is not a fresh success or zero. Explicit operator evaluation and one-time publication remain separate, subject to their existing authority and budgets. -Collector subprocess failures cross runtime boundaries only as bounded categories: +Collector failures cross runtime boundaries only as bounded categories: `api-budget`, `budget-unavailable`, `timeout`, `output-budget`, or `collector-failed`. Worker markers cannot supply `output-budget`; that category is owned by the host enforcing its output limits. Unknown markers and raw stderr @@ -218,12 +218,34 @@ retime either clock. The control policy explicitly lists the same six repositories previously collected from `scope.allowed-repositories`; this pins Activity enrollment -without adding targets or activating Daily File Diet itself. Only the exact -File Diet pair is registered for recurring Value acquisition. +without adding targets or activating Daily File Diet itself. Only the File Diet and Dependabot Update Planner pairs in `github/gh-aw` are +registered for recurring Value acquisition. Registration is visible after the collector and policy are reviewed and deployed together. Its reviewed `api-requests: 700` ceiling allows Activity to refresh it automatically only when that full grant fits above ordinary collection's protected reserve. + +### Dependabot lifecycle collector + +The separately reviewed `dependabot-update-planner` registration covers only +`github/gh-aw`. Its Go collector follows bounded alert `after` cursors for +closed and open Dependabot security-alert lifecycles (at most 100 pages) and Dependabot-authored pull-request search +results (at most ten pages) through the installation's shared Core and Search +quotas. At 100 records per page, the reviewed ceiling is 111 metered +requests (100 alert pages, ten search pages, and one optional Search-quota +preflight); exceeding either bound, the search-result cap, or the timeout is +missing evidence, not an empty queue. The installation needs Dependabot-alert +and pull-request read permissions for that repository; lacking either defers +collection without changing accepted observations. + +Three packets, one for each frozen metric, commit atomically with the private +proof after complete acquisition. The local Go evaluator queries only those +canonical packets for the aligned seven-day cutoff. It emits numeric open +alert, critical/high-alert, and Dependabot PR counts when all three are +complete, including genuine zeroes; otherwise it emits missing measurements +and preserves earlier accepted captures. This registration does not change +the Dependabot campaign's rollout mode or grant another repository access. + It does not override `campaigns.dashboard.deploy: false`: this repository intentionally disables standalone Pages deployment because Documentation Pages owns the combined site. The existing publication chain is diff --git a/go.mod b/go.mod index 3f5254eec4..cd1d271351 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/githubnext/gh-cao/internal/dashboardquery v0.0.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 + golang.org/x/sys v0.49.0 golang.org/x/term v0.47.0 golang.org/x/text v0.43.0 modernc.org/sqlite v1.60.1 @@ -73,7 +74,6 @@ require ( golang.org/x/net v0.61.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.24.0 // indirect - golang.org/x/sys v0.49.0 // indirect golang.org/x/time v0.15.0 // indirect gonum.org/v1/gonum v0.17.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/server/Dockerfile b/server/Dockerfile index b41b9e7835..41cf3294fe 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -143,22 +143,11 @@ WORKDIR /app USER 65532:65532 ENTRYPOINT ["/bin/sh", "/app/azure/provision.sh"] -FROM dashboard-build AS control-plane-inventory - -RUN REPORT_ROOT=/workspace \ - REPORT_INVENTORY=/tmp/control-plane-inventory.json \ - REPORT_INVENTORY_STATIC=true \ - node activity/inventory.mjs - -FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS collector-runtime - -SHELL ["/bin/ash", "-o", "pipefail", "-c"] +FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 AS collector-runtime ARG VERSION=dev ARG REVISION=unknown ARG CREATED=unknown -ARG GH_AW_VERSION=v1.0.1 -ARG TARGETARCH LABEL org.opencontainers.image.title="Central Agentic Ops collector" \ org.opencontainers.image.description="Hosted CAO collection worker and backfill runtime" \ org.opencontainers.image.version="${VERSION}" \ @@ -166,48 +155,25 @@ LABEL org.opencontainers.image.title="Central Agentic Ops collector" \ org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.source="https://github.com/githubnext/gh-cao" -# Keep Alpine packages, CA roots, and the GitHub CLI current on rebuilds. # hadolint ignore=DL3018 RUN apk upgrade --no-cache \ - && apk add --no-cache ca-certificates github-cli \ + && apk add --no-cache ca-certificates git \ && addgroup -S -g 65532 cao \ && adduser -S -D -H -u 65532 -G cao cao \ - && mkdir -p /app/catalog /app/evidence /app/.local/share/gh/extensions/gh-aw \ - && case "${TARGETARCH}" in \ - amd64) checksum="1888d9faa40e102cbaa784ecb2e646c0e81ac3f1003d1cb3460d3a23908d4938" ;; \ - arm64) checksum="2bc9a5e212cab7fb02efa91ea362a27c381b1760af37f0effa3e5b3c5e8a6393" ;; \ - *) echo "Unsupported collector architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac \ - && wget -q -O /usr/local/bin/gh-aw \ - "https://github.com/github/gh-aw/releases/download/${GH_AW_VERSION}/linux-${TARGETARCH}" \ - && echo "${checksum} /usr/local/bin/gh-aw" | sha256sum -c - \ - && chmod 0755 /usr/local/bin/gh-aw \ - && ln -s /usr/local/bin/gh-aw /app/.local/share/gh/extensions/gh-aw/gh-aw \ + && mkdir -p /app/queries /app/.github/workflows /app/.github/value-evidence \ && chown -R cao:cao /app -COPY --from=dashboard-runtime --chown=cao:cao /app/ /app/ -COPY --from=dashboard-build --chown=cao:cao /workspace/activity/ /app/catalog/activity/ -COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/shared/control.mjs /app/catalog/.github/workflows/shared/control.mjs -COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/shared/policy.mjs /app/catalog/.github/workflows/shared/policy.mjs -COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/shared/authority.mjs /app/catalog/.github/workflows/shared/authority.mjs -COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/shared/operations.mjs /app/catalog/.github/workflows/shared/operations.mjs -COPY --from=control-plane-inventory --chown=cao:cao /tmp/control-plane-inventory.json /app/catalog/control-plane-inventory.json -# gh aw logs discovers agentic workflow names from local compiled workflows -# before it queries runs, so the collection catalog must retain the lock files. -COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/*.lock.yml /app/catalog/.github/workflows/ -COPY --from=dashboard-build --chown=cao:cao /workspace/dashboard/report/ /app/catalog/dashboard/report/ -COPY --from=dashboard-build --chown=cao:cao /workspace/dashboard/site/package.json /app/catalog/dashboard/site/package.json -COPY --from=dashboard-build --chown=cao:cao /workspace/dashboard/site/src/ /app/catalog/dashboard/site/src/ -COPY --from=dashboard-build --chown=cao:cao /workspace/dashboard/site/node_modules/yaml/ /app/catalog/node_modules/yaml/ -COPY --from=dashboard-build --chown=cao:cao /workspace/dependabot/ /app/catalog/dependabot/ -COPY --from=dashboard-build --chown=cao:cao /workspace/optimization/ /app/catalog/optimization/ +COPY --from=server-build --chown=cao:cao /out/cao-dashboard /app/cao-dashboard +COPY --from=dashboard-build --chown=cao:cao /workspace/dashboard/site/src/data/queries/database.json /app/queries/database.json +COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/cao.json /app/.github/workflows/cao.json +COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/cao.azure.json /app/.github/workflows/cao.azure.json +COPY --from=dashboard-build --chown=cao:cao /workspace/.github/value-evidence/daily-file-diet.jsonl.gz /app/.github/value-evidence/daily-file-diet.jsonl.gz ENV HOME=/app \ XDG_CACHE_HOME=/tmp/.cache \ XDG_CONFIG_HOME=/tmp/.config WORKDIR /app USER 65532:65532 -VOLUME ["/app/evidence"] ENTRYPOINT ["/app/cao-dashboard"] CMD ["collect", "--database-queries", "/app/queries/database.json"] diff --git a/server/README.md b/server/README.md index 5da87d87b8..7984e49fdf 100644 --- a/server/README.md +++ b/server/README.md @@ -2734,12 +2734,13 @@ The Dockerfile includes the base `cao.json` policy. Configure a reviewed credentials through the host's secret manager. The `dashboard-runtime` target contains the public server without collection -tools. The `collector-runtime` target adds Node, the GitHub CLI, a -checksum-verified policy-pinned gh-aw binary, the Activity projection catalog, -a deterministic source-bound control-plane inventory, and the shared policy -resolver. Deploy standalone collection roles without public ingress, keep the -collection App private key out of admission-only web replicas, and share a -persistent evidence volume between collector and backfill roles. +tools. The `collector-runtime` target adds Git for the reviewed Value Git +transport, not Node, the GitHub CLI, gh-aw, or the Activity projection catalog. +It packages the reviewed policy and SHA-256-pinned Value evidence; selected +Value deployments can use `/app` as `CAO_COLLECT_CATALOG_ROOT` and +`/app/.github/workflows/cao.json` as `CAO_POLICY_PATH`. +Deploy collectors without public ingress; keep the App key out of admission-only +replicas. Collection and backfill share PostgreSQL, not a filesystem volume. The external artifact volume is authoritative input, not checked-in deployment data. Before the first start, populate an unattached staging volume with a @@ -3377,18 +3378,21 @@ span ID. ### Collection settings -Legacy Activity-backed collection remains the default. Explicitly select -distributed Go collection with `CAO_COLLECT_NATIVE=true` and the exact reviewed -inventory digest provisioned in PostgreSQL. Unsupported audit/intelligence -requests reject, never fall back to Activity. Optional Value dependencies are -validated only when selected. - -Native run acquisition, projection and approved trace destinations use Go and -shared PostgreSQL fences, without Node, Activity files or filesystem coordination. -Optional `NativeValues` retains reviewed Node programs/catalog/composed policy, -bounded disposable scratch and PostgreSQL-authoritative packet/proof recovery. -Publication rechecks composed policy and source ownership; withdrawal atomically -erases private proof. Remote Value retains its Linux-only socket transport. +The Actions-published snapshot remains the default when `CAO_COLLECT_APP_ID` +is unset. App-backed collection requires `CAO_COLLECT_NATIVE=true`, an exact +reviewed inventory digest provisioned in PostgreSQL, and a durable +deployment-scoped operational store. The old filesystem collector is retired; +an App identifier without native selection fails at startup. Unsupported +audit/intelligence requests reject rather than fall back to Activity. Optional +Value dependencies are validated only when selected. + +Native run acquisition, projection, approved trace destinations and optional +`NativeValues` use Go and shared PostgreSQL fences, without Node, Activity +files or filesystem coordination. The selected Value lane evaluates reviewed +policy and supported collectors in Go; its bounded disposable Git workspace +is not an evidence store. PostgreSQL owns packet/proof recovery, publication +rechecks composed policy and source ownership, and withdrawal atomically erases +private proof. Unsupported collector registrations fail closed. Inventory digests select exact provisioned reviewed bytes, not broader authority. Native mode rejects legacy run-count/storage/request-minute options rather than ignoring them; its discovery contract is bounded and `CAO_COLLECT_TIMEOUT` @@ -3399,11 +3403,9 @@ See `ARCHITECTURE.md` and `specs/value.md` for the feature-specific boundaries. |---|---| | `CAO_COLLECT_APP_ID` | GitHub App identifier; unset selects the default profile | | `CAO_COLLECT_PRIVATE_KEY` / `CAO_COLLECT_PRIVATE_KEY_BASE64` / `CAO_COLLECT_PRIVATE_KEY_FILE` | App private key as inline PEM, single-line standard base64, or a mounted PEM file | -| `CAO_COLLECT_LAKE_DIRECTORY` | evidence lake directory, shared by workers | -| `CAO_COLLECT_CATALOG_ROOT` | directory containing `activity/cao.mjs` | -| `CAO_COLLECT_CONTROL_REPOSITORY` | control repository used for inventory discovery | -| `CAO_COLLECT_STATIC_INVENTORY` | source-bound control-plane inventory packaged with the collector | -| `CAO_POLICY_PATH` | reviewed base policy or deployment profile resolved before inventory discovery | +| `CAO_COLLECT_CATALOG_ROOT` | reviewed catalog root for the separately selected Go Value lane | +| `CAO_COLLECT_CONTROL_REPOSITORY` | reviewed inventory's control repository | +| `CAO_POLICY_PATH` | reviewed policy for the separately selected Go Value lane | | `CAO_COLLECT_WORKERS` | in-process workers; zero when workers scale separately | | `CAO_COLLECT_RATE_LIMIT_FLOOR` | requests reserved per installation | | `CAO_COLLECT_PROJECTION_INTERVAL` | minimum interval between projections (default 5 minutes) | @@ -3412,10 +3414,14 @@ See `ARCHITECTURE.md` and `specs/value.md` for the feature-specific boundaries. | `CAO_COLLECT_RECOVER_DELIVERIES` | replay failed webhook deliveries to close gaps | | `CAO_COLLECT_QUEUE_MAX_LENGTH` | admission backpressure limit for outstanding collection tasks (default 200 000); tasks are never trimmed | | `CAO_COLLECT_ADMIT_ONLY` | admit deliveries without collecting; requires no private key | -| `CAO_COLLECT_NATIVE` | explicitly selects distributed Go collection; legacy remains the default | -| `CAO_COLLECT_NATIVE_VALUES` | separately selects existing reviewed Value execution; requires native selection and its explicit Node/catalog/policy dependencies | +| `CAO_COLLECT_NATIVE` | required for App-backed collection; unset with no App identifier preserves the Actions snapshot profile | +| `CAO_COLLECT_NATIVE_VALUES` | separately selects Go Value execution; requires native selection and its explicit catalog/policy dependencies | | `CAO_COLLECT_NATIVE_INVENTORY_DIGEST` | exact digest of reviewed inventory bytes already provisioned in the shared PostgreSQL namespace; requires native selection | +With App-backed collection, the retired `CAO_COLLECT_LAKE_DIRECTORY`, +`CAO_COLLECT_STATIC_INVENTORY`, `CAO_COLLECT_GH_BINARY`, and +`CAO_COLLECT_NODE_BINARY` settings are rejected rather than silently ignored. + ### Admission-only front ends A process that only receives webhooks does not need collection credentials. diff --git a/server/SECURITY.md b/server/SECURITY.md index c8c029fb06..989b9b8c31 100644 --- a/server/SECURITY.md +++ b/server/SECURITY.md @@ -474,7 +474,9 @@ read-only. Never populate a live attached artifact volume or treat a placeholder as deployment data. Use the private `collector-runtime` target for standalone collection and -backfill roles, with a persistent evidence volume and no public ingress. +backfill roles, with PostgreSQL-backed evidence and no public ingress. Its +Git executable is confined to the reviewed Value Git acquisition transport; +the Go server does not launch Node. Admission-only web replicas must not receive the collection App private key. Official package scans, hardening checks, SBOMs, and attestations apply to diff --git a/server/azure/collector.bicep b/server/azure/collector.bicep index ff935b9226..dcae0b31ad 100644 --- a/server/azure/collector.bicep +++ b/server/azure/collector.bicep @@ -4,8 +4,8 @@ // default deployment stays exactly as it is: a Function App serving the // dashboard from snapshots published by the Activity workflow. // -// The workers run the same binary as the server in the `collect` role and -// share the durable PostgreSQL operational queue with the Function App. +// The workers run native Go acquisition in the `collect` role and share the +// durable PostgreSQL operational queue with the Function App. @description('Location for the collection workers.') param location string @@ -39,10 +39,6 @@ param keyVaultName string @description('Key Vault secret that stores the PostgreSQL URL.') param postgresUrlSecretName string = 'cao-postgres-runtime-url' -@description('Size of the evidence lake file share, in gibibytes.') -@minValue(100) -param lakeQuotaGigabytes int = 1024 - @minValue(1) @description('Maximum bytes retained in the disposable operational cache.') param operationalCacheMaxBytes int = 33554432 @@ -66,9 +62,14 @@ param operationalProtectedMaxEntries int = 200000 @description('Operational PostgreSQL namespace shared with the Function App.') param operationalNamespace string -@description('Control repository used for logical source discovery.') +@description('Control repository named by the reviewed inventory.') param controlRepository string +@minLength(64) +@maxLength(64) +@description('Exact provisioned lowercase SHA-256 digest of immutable reviewed inventory bytes. The workers fail startup if this digest is not in their shared PostgreSQL namespace.') +param nativeInventoryDigest string + @description('GitHub App identifier whose installations define ingestion scope.') param githubAppId string @@ -88,25 +89,10 @@ param otlpLogsEnabled bool = false @description('Whether the parent provisioned the cao-otel-headers Key Vault secret.') param otlpHeadersEnabled bool = false -@description('Evidence lake file share tier. Premium is provisioned SSD; Standard is IOPS-throttled by share size.') -@allowed([ - 'Premium_LRS' - 'Premium_ZRS' - 'Standard_LRS' - 'Standard_ZRS' -]) -param lakeStorageSku string = 'Premium_LRS' - @description('Bound on the collection task and dead-letter streams.') @minValue(1000) param queueMaxLength int = 200000 -// Premium file shares are provisioned SSD and live in a FileStorage account; -// Standard shares live in a general-purpose account. The lake is many small -// per-repository shards, so it is metadata-operation bound rather than -// throughput bound, and Standard share IOPS scale only with provisioned size. -var lakePremium = startsWith(lakeStorageSku, 'Premium') -var lakeStorageAccountName = '${toLower(take(replace(namePrefix, '-', ''), 7))}lake${uniqueString(resourceGroup().id, namePrefix)}' resource collectorIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: '${namePrefix}-collector-identity' location: location @@ -145,78 +131,6 @@ var registries = [ } ] -// The evidence lake is retained collected evidence, not a cache: cold start -// replays it without contacting GitHub. It is shared by every worker and by -// the projector, so it lives on a file share rather than in container storage. -resource lakeAccount 'Microsoft.Storage/storageAccounts@2023-05-01' = { - name: lakeStorageAccountName - location: location - tags: tags - kind: lakePremium ? 'FileStorage' : 'StorageV2' - sku: { - name: lakeStorageSku - } - properties: { - accessTier: lakePremium ? 'Premium' : 'Hot' - minimumTlsVersion: 'TLS1_2' - supportsHttpsTrafficOnly: true - allowBlobPublicAccess: false - allowCrossTenantReplication: false - publicNetworkAccess: 'Enabled' - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Allow' - } - encryption: { - requireInfrastructureEncryption: true - keySource: 'Microsoft.Storage' - services: { - file: { - enabled: true - keyType: 'Account' - } - } - } - } -} - -resource lakeFileService 'Microsoft.Storage/storageAccounts/fileServices@2023-05-01' = { - parent: lakeAccount - name: 'default' - properties: { - // The lake is retained evidence that cold start replays, so an accidental - // share deletion must be recoverable. - shareDeleteRetentionPolicy: { - enabled: true - days: 30 - } - } -} - -resource lakeShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-05-01' = { - parent: lakeFileService - name: 'evidence-lake' - properties: { - // A premium share provisions IOPS and throughput from its quota, so the - // quota is a performance setting there rather than only a ceiling. - shareQuota: lakePremium ? max(lakeQuotaGigabytes, 100) : lakeQuotaGigabytes - enabledProtocols: 'SMB' - } -} - -resource lakeStorage 'Microsoft.App/managedEnvironments/storages@2024-03-01' = { - parent: environment - name: 'evidence-lake' - properties: { - azureFile: { - accountName: lakeAccount.name - shareName: lakeShare.name - accessMode: 'ReadWrite' - accountKey: lakeAccount.listKeys().keys[0].value - } - } -} - var collectionBaseEnvironment = [ { name: 'CAO_POSTGRES_URL' @@ -267,16 +181,16 @@ var collectionBaseEnvironment = [ value: githubAppId } { - name: 'CAO_COLLECT_PRIVATE_KEY' - secretRef: 'cao-collect-private-key' + name: 'CAO_COLLECT_NATIVE' + value: 'true' } { - name: 'CAO_COLLECT_LAKE_DIRECTORY' - value: '/evidence' + name: 'CAO_COLLECT_NATIVE_INVENTORY_DIGEST' + value: nativeInventoryDigest } { - name: 'CAO_COLLECT_CATALOG_ROOT' - value: '/app/catalog' + name: 'CAO_COLLECT_PRIVATE_KEY' + secretRef: 'cao-collect-private-key' } { name: 'CAO_COLLECT_CONTROL_REPOSITORY' @@ -359,19 +273,6 @@ resource collectors 'Microsoft.App/containerApps@2024-03-01' = { cpu: json('1.0') memory: '2Gi' } - volumeMounts: [ - { - volumeName: 'evidence' - mountPath: '/evidence' - } - ] - } - ] - volumes: [ - { - name: 'evidence' - storageType: 'AzureFile' - storageName: lakeStorage.name } ] scale: { @@ -420,19 +321,6 @@ resource backfill 'Microsoft.App/jobs@2024-03-01' = { cpu: json('1.0') memory: '2Gi' } - volumeMounts: [ - { - volumeName: 'evidence' - mountPath: '/evidence' - } - ] - } - ] - volumes: [ - { - name: 'evidence' - storageType: 'AzureFile' - storageName: lakeStorage.name } ] } diff --git a/server/azure/main.bicep b/server/azure/main.bicep index 525fe438bb..202588c438 100644 --- a/server/azure/main.bicep +++ b/server/azure/main.bicep @@ -226,22 +226,16 @@ param githubWebhookSecret string = '' @description('GitHub logins permitted to run administrative operations.') param githubAdminUsers array = [] -@description('Control repository used for logical source discovery, in OWNER/REPOSITORY form.') +@description('Control repository named by the reviewed inventory, in OWNER/REPOSITORY form.') param collectorControlRepository string = '' +@description('Exact lowercase SHA-256 digest of immutable reviewed inventory bytes already provisioned in the shared PostgreSQL namespace. Required when collectorImage is set; an absent or unprovisioned digest prevents the workers from starting.') +param collectorNativeInventoryDigest string = '' + @minValue(1) @description('Fixed number of PostgreSQL-backed collection workers.') param collectorWorkerReplicas int = 1 -@description('Evidence lake file share tier. Premium is provisioned SSD; Standard is IOPS-throttled by share size.') -@allowed([ - 'Premium_LRS' - 'Premium_ZRS' - 'Standard_LRS' - 'Standard_ZRS' -]) -param collectorLakeStorageSku string = 'Premium_LRS' - @description('Provision an optional paid ADX destination only. Does not enable export, traces, or mining; Azure Functions is not a supported exporter host.') param adxEnabled bool = false @@ -686,10 +680,14 @@ resource functionApp 'Microsoft.Web/sites@2023-12-01' = { name: 'CAO_COLLECT_APP_ID' value: collectorGithubAppId } + { + name: 'CAO_COLLECT_NATIVE' + value: 'true' + } { // The Function App verifies deliveries and enqueues work. It never // calls GitHub and never projects, so it is given no App private key - // and no evidence lake: the internet-facing front end holds no + // or inventory bytes: the internet-facing front end holds no // credential it cannot use. name: 'CAO_COLLECT_ADMIT_ONLY' value: 'true' @@ -745,9 +743,9 @@ module collection 'collector.bicep' = if (collectionEnabled) { operationalProtectedMaxBytes: operationalProtectedMaxBytes operationalProtectedMaxEntries: operationalProtectedMaxEntries controlRepository: collectorControlRepository + nativeInventoryDigest: collectorNativeInventoryDigest githubAppId: collectorGithubAppId workerReplicas: collectorWorkerReplicas - lakeStorageSku: collectorLakeStorageSku applicationInsightsConnectionString: insights.properties.ConnectionString otlpEnvironment: otlpEnvironment otlpLogsEnabled: otlpLogsEnabled diff --git a/server/azure/main.json b/server/azure/main.json index 7ed1eb830f..8a4bc34768 100644 --- a/server/azure/main.json +++ b/server/azure/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.48.1.52935", - "templateHash": "10401505667316468141" + "templateHash": "17295245798372953389" } }, "parameters": { @@ -395,7 +395,14 @@ "type": "string", "defaultValue": "", "metadata": { - "description": "Control repository used for logical source discovery, in OWNER/REPOSITORY form." + "description": "Control repository named by the reviewed inventory, in OWNER/REPOSITORY form." + } + }, + "collectorNativeInventoryDigest": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Exact lowercase SHA-256 digest of immutable reviewed inventory bytes already provisioned in the shared PostgreSQL namespace. Required when collectorImage is set; an absent or unprovisioned digest prevents the workers from starting." } }, "collectorWorkerReplicas": { @@ -406,19 +413,6 @@ "description": "Fixed number of PostgreSQL-backed collection workers." } }, - "collectorLakeStorageSku": { - "type": "string", - "defaultValue": "Premium_LRS", - "allowedValues": [ - "Premium_LRS", - "Premium_ZRS", - "Standard_LRS", - "Standard_ZRS" - ], - "metadata": { - "description": "Evidence lake file share tier. Premium is provisioned SSD; Standard is IOPS-throttled by share size." - } - }, "adxEnabled": { "type": "bool", "defaultValue": false, @@ -782,7 +776,7 @@ "linuxFxVersion": "[format('DOCKER|{0}/{1}{2}', reference(resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName')), '2023-07-01').loginServer, parameters('functionImageRepository'), variables('functionImageSelector'))]", "appCommandLine": "azure-functions", "acrUseManagedIdentityCreds": true, - "appSettings": "[concat(createArray(createObject('name', 'FUNCTIONS_EXTENSION_VERSION', 'value', '~4'), createObject('name', 'FUNCTIONS_WORKER_RUNTIME', 'value', 'custom'), createObject('name', 'WEBSITES_PORT', 'value', '8080'), createObject('name', 'WEBSITES_ENABLE_APP_SERVICE_STORAGE', 'value', 'false'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', parameters('applicationInsightsName')), '2020-02-02').ConnectionString), createObject('name', 'AzureWebJobsStorage__accountName', 'value', parameters('storageAccountName')), createObject('name', 'AzureWebJobsStorage__credential', 'value', 'managedidentity'), createObject('name', 'CAO_DASHBOARD_HOSTING', 'value', 'azure-functions'), createObject('name', 'CAO_POLICY_PATH', 'value', '.github/workflows/cao.azure.json'), createObject('name', 'CAO_DATABASE_QUERIES', 'value', '/app/queries/database.json'), createObject('name', 'CAO_AZURE_ALLOWED_HOSTS', 'value', join(parameters('allowedHosts'), ',')), createObject('name', 'CAO_AZURE_REQUIRE_HTTPS', 'value', 'true'), createObject('name', 'REDIS_NAMESPACE', 'value', 'azure-dashboard'), createObject('name', 'CAO_OPERATIONAL_NAMESPACE', 'value', 'azure-dashboard-operational'), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_BYTES', 'value', string(parameters('operationalCacheMaxBytes'))), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_VALUE_BYTES', 'value', string(parameters('operationalCacheMaxValueBytes'))), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_ENTRIES', 'value', string(parameters('operationalCacheMaxEntries'))), createObject('name', 'CAO_OPERATIONAL_PROTECTED_MAX_BYTES', 'value', string(parameters('operationalProtectedMaxBytes'))), createObject('name', 'CAO_OPERATIONAL_PROTECTED_MAX_ENTRIES', 'value', string(parameters('operationalProtectedMaxEntries'))), createObject('name', 'CAO_POSTGRES_URL', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/{1})', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri, parameters('postgresUrlSecretName'))), createObject('name', 'CAO_POSTGRES_RUN_RETENTION_DAYS', 'value', string(parameters('postgresRunRetentionDays'))), createObject('name', 'CAO_POSTGRES_LINKED_RETENTION_DAYS', 'value', string(parameters('postgresRunRetentionDays'))), createObject('name', 'CAO_GITHUB_CLIENT_ID', 'value', parameters('githubClientId')), createObject('name', 'CAO_GITHUB_CLIENT_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/github-oauth-client-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)), createObject('name', 'CAO_GITHUB_REDIRECT_URL', 'value', variables('githubRedirectUri')), createObject('name', 'CAO_GITHUB_ALLOWED_ORGS', 'value', join(parameters('githubAllowedOrganizations'), ',')), createObject('name', 'CAO_GITHUB_ALLOWED_TEAMS', 'value', join(parameters('githubAllowedTeams'), ',')), createObject('name', 'CAO_GITHUB_ALLOWED_USERS', 'value', join(parameters('githubAllowedUsers'), ',')), createObject('name', 'CAO_SESSION_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-session-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri))), variables('otlpEnvironment'), if(not(parameters('otlpLogsEnabled')), createArray(), createArray(createObject('name', 'DEBUG', 'value', 'cao:server,cao:telemetry'))), if(not(variables('otlpHeadersEnabled')), createArray(), createArray(createObject('name', 'OTEL_EXPORTER_OTLP_HEADERS', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-otel-headers)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)))), if(empty(parameters('previousSessionSecret')), createArray(), createArray(createObject('name', 'CAO_SESSION_SECRET_PREVIOUS', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-session-secret-previous)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)))), if(not(variables('collectionEnabled')), createArray(), createArray(createObject('name', 'CAO_GITHUB_WEBHOOK_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-github-webhook-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)), createObject('name', 'CAO_GITHUB_ADMIN_USERS', 'value', join(parameters('githubAdminUsers'), ',')), createObject('name', 'CAO_COLLECT_APP_ID', 'value', parameters('collectorGithubAppId')), createObject('name', 'CAO_COLLECT_ADMIT_ONLY', 'value', 'true'), createObject('name', 'CAO_COLLECT_CONTROL_REPOSITORY', 'value', parameters('collectorControlRepository')))))]" + "appSettings": "[concat(createArray(createObject('name', 'FUNCTIONS_EXTENSION_VERSION', 'value', '~4'), createObject('name', 'FUNCTIONS_WORKER_RUNTIME', 'value', 'custom'), createObject('name', 'WEBSITES_PORT', 'value', '8080'), createObject('name', 'WEBSITES_ENABLE_APP_SERVICE_STORAGE', 'value', 'false'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', parameters('applicationInsightsName')), '2020-02-02').ConnectionString), createObject('name', 'AzureWebJobsStorage__accountName', 'value', parameters('storageAccountName')), createObject('name', 'AzureWebJobsStorage__credential', 'value', 'managedidentity'), createObject('name', 'CAO_DASHBOARD_HOSTING', 'value', 'azure-functions'), createObject('name', 'CAO_POLICY_PATH', 'value', '.github/workflows/cao.azure.json'), createObject('name', 'CAO_DATABASE_QUERIES', 'value', '/app/queries/database.json'), createObject('name', 'CAO_AZURE_ALLOWED_HOSTS', 'value', join(parameters('allowedHosts'), ',')), createObject('name', 'CAO_AZURE_REQUIRE_HTTPS', 'value', 'true'), createObject('name', 'REDIS_NAMESPACE', 'value', 'azure-dashboard'), createObject('name', 'CAO_OPERATIONAL_NAMESPACE', 'value', 'azure-dashboard-operational'), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_BYTES', 'value', string(parameters('operationalCacheMaxBytes'))), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_VALUE_BYTES', 'value', string(parameters('operationalCacheMaxValueBytes'))), createObject('name', 'CAO_OPERATIONAL_CACHE_MAX_ENTRIES', 'value', string(parameters('operationalCacheMaxEntries'))), createObject('name', 'CAO_OPERATIONAL_PROTECTED_MAX_BYTES', 'value', string(parameters('operationalProtectedMaxBytes'))), createObject('name', 'CAO_OPERATIONAL_PROTECTED_MAX_ENTRIES', 'value', string(parameters('operationalProtectedMaxEntries'))), createObject('name', 'CAO_POSTGRES_URL', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/{1})', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri, parameters('postgresUrlSecretName'))), createObject('name', 'CAO_POSTGRES_RUN_RETENTION_DAYS', 'value', string(parameters('postgresRunRetentionDays'))), createObject('name', 'CAO_POSTGRES_LINKED_RETENTION_DAYS', 'value', string(parameters('postgresRunRetentionDays'))), createObject('name', 'CAO_GITHUB_CLIENT_ID', 'value', parameters('githubClientId')), createObject('name', 'CAO_GITHUB_CLIENT_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/github-oauth-client-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)), createObject('name', 'CAO_GITHUB_REDIRECT_URL', 'value', variables('githubRedirectUri')), createObject('name', 'CAO_GITHUB_ALLOWED_ORGS', 'value', join(parameters('githubAllowedOrganizations'), ',')), createObject('name', 'CAO_GITHUB_ALLOWED_TEAMS', 'value', join(parameters('githubAllowedTeams'), ',')), createObject('name', 'CAO_GITHUB_ALLOWED_USERS', 'value', join(parameters('githubAllowedUsers'), ',')), createObject('name', 'CAO_SESSION_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-session-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri))), variables('otlpEnvironment'), if(not(parameters('otlpLogsEnabled')), createArray(), createArray(createObject('name', 'DEBUG', 'value', 'cao:server,cao:telemetry'))), if(not(variables('otlpHeadersEnabled')), createArray(), createArray(createObject('name', 'OTEL_EXPORTER_OTLP_HEADERS', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-otel-headers)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)))), if(empty(parameters('previousSessionSecret')), createArray(), createArray(createObject('name', 'CAO_SESSION_SECRET_PREVIOUS', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-session-secret-previous)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)))), if(not(variables('collectionEnabled')), createArray(), createArray(createObject('name', 'CAO_GITHUB_WEBHOOK_SECRET', 'value', format('@Microsoft.KeyVault(SecretUri={0}secrets/cao-github-webhook-secret)', reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2023-07-01').vaultUri)), createObject('name', 'CAO_GITHUB_ADMIN_USERS', 'value', join(parameters('githubAdminUsers'), ',')), createObject('name', 'CAO_COLLECT_APP_ID', 'value', parameters('collectorGithubAppId')), createObject('name', 'CAO_COLLECT_NATIVE', 'value', 'true'), createObject('name', 'CAO_COLLECT_ADMIT_ONLY', 'value', 'true'), createObject('name', 'CAO_COLLECT_CONTROL_REPOSITORY', 'value', parameters('collectorControlRepository')))))]" } }, "dependsOn": [ @@ -1411,15 +1405,15 @@ "controlRepository": { "value": "[parameters('collectorControlRepository')]" }, + "nativeInventoryDigest": { + "value": "[parameters('collectorNativeInventoryDigest')]" + }, "githubAppId": { "value": "[parameters('collectorGithubAppId')]" }, "workerReplicas": { "value": "[parameters('collectorWorkerReplicas')]" }, - "lakeStorageSku": { - "value": "[parameters('collectorLakeStorageSku')]" - }, "applicationInsightsConnectionString": { "value": "[reference(resourceId('Microsoft.Insights/components', parameters('applicationInsightsName')), '2020-02-02').ConnectionString]" }, @@ -1440,7 +1434,7 @@ "_generator": { "name": "bicep", "version": "0.48.1.52935", - "templateHash": "6349869892450546943" + "templateHash": "2327320549105080117" } }, "parameters": { @@ -1504,14 +1498,6 @@ "description": "Key Vault secret that stores the PostgreSQL URL." } }, - "lakeQuotaGigabytes": { - "type": "int", - "defaultValue": 1024, - "minValue": 100, - "metadata": { - "description": "Size of the evidence lake file share, in gibibytes." - } - }, "operationalCacheMaxBytes": { "type": "int", "defaultValue": 33554432, @@ -1561,7 +1547,15 @@ "controlRepository": { "type": "string", "metadata": { - "description": "Control repository used for logical source discovery." + "description": "Control repository named by the reviewed inventory." + } + }, + "nativeInventoryDigest": { + "type": "string", + "minLength": 64, + "maxLength": 64, + "metadata": { + "description": "Exact provisioned lowercase SHA-256 digest of immutable reviewed inventory bytes. The workers fail startup if this digest is not in their shared PostgreSQL namespace." } }, "githubAppId": { @@ -1602,19 +1596,6 @@ "description": "Whether the parent provisioned the cao-otel-headers Key Vault secret." } }, - "lakeStorageSku": { - "type": "string", - "defaultValue": "Premium_LRS", - "allowedValues": [ - "Premium_LRS", - "Premium_ZRS", - "Standard_LRS", - "Standard_ZRS" - ], - "metadata": { - "description": "Evidence lake file share tier. Premium is provisioned SSD; Standard is IOPS-throttled by share size." - } - }, "queueMaxLength": { "type": "int", "defaultValue": 200000, @@ -1625,8 +1606,6 @@ } }, "variables": { - "lakePremium": "[startsWith(parameters('lakeStorageSku'), 'Premium')]", - "lakeStorageAccountName": "[format('{0}lake{1}', toLower(take(replace(parameters('namePrefix'), '-', ''), 7)), uniqueString(resourceGroup().id, parameters('namePrefix')))]", "collectionBaseEnvironment": [ { "name": "CAO_POSTGRES_URL", @@ -1677,16 +1656,16 @@ "value": "[parameters('githubAppId')]" }, { - "name": "CAO_COLLECT_PRIVATE_KEY", - "secretRef": "cao-collect-private-key" + "name": "CAO_COLLECT_NATIVE", + "value": "true" }, { - "name": "CAO_COLLECT_LAKE_DIRECTORY", - "value": "/evidence" + "name": "CAO_COLLECT_NATIVE_INVENTORY_DIGEST", + "value": "[parameters('nativeInventoryDigest')]" }, { - "name": "CAO_COLLECT_CATALOG_ROOT", - "value": "/app/catalog" + "name": "CAO_COLLECT_PRIVATE_KEY", + "secretRef": "cao-collect-private-key" }, { "name": "CAO_COLLECT_CONTROL_REPOSITORY", @@ -1720,82 +1699,6 @@ "location": "[parameters('location')]", "tags": "[parameters('tags')]" }, - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2023-05-01", - "name": "[variables('lakeStorageAccountName')]", - "location": "[parameters('location')]", - "tags": "[parameters('tags')]", - "kind": "[if(variables('lakePremium'), 'FileStorage', 'StorageV2')]", - "sku": { - "name": "[parameters('lakeStorageSku')]" - }, - "properties": { - "accessTier": "[if(variables('lakePremium'), 'Premium', 'Hot')]", - "minimumTlsVersion": "TLS1_2", - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowCrossTenantReplication": false, - "publicNetworkAccess": "Enabled", - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Allow" - }, - "encryption": { - "requireInfrastructureEncryption": true, - "keySource": "Microsoft.Storage", - "services": { - "file": { - "enabled": true, - "keyType": "Account" - } - } - } - } - }, - { - "type": "Microsoft.Storage/storageAccounts/fileServices", - "apiVersion": "2023-05-01", - "name": "[format('{0}/{1}', variables('lakeStorageAccountName'), 'default')]", - "properties": { - "shareDeleteRetentionPolicy": { - "enabled": true, - "days": 30 - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('lakeStorageAccountName'))]" - ] - }, - { - "type": "Microsoft.Storage/storageAccounts/fileServices/shares", - "apiVersion": "2023-05-01", - "name": "[format('{0}/{1}/{2}', variables('lakeStorageAccountName'), 'default', 'evidence-lake')]", - "properties": { - "shareQuota": "[if(variables('lakePremium'), max(parameters('lakeQuotaGigabytes'), 100), parameters('lakeQuotaGigabytes'))]", - "enabledProtocols": "SMB" - }, - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('lakeStorageAccountName'), 'default')]" - ] - }, - { - "type": "Microsoft.App/managedEnvironments/storages", - "apiVersion": "2024-03-01", - "name": "[format('{0}/{1}', parameters('environmentName'), 'evidence-lake')]", - "properties": { - "azureFile": { - "accountName": "[variables('lakeStorageAccountName')]", - "shareName": "evidence-lake", - "accessMode": "ReadWrite", - "accountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('lakeStorageAccountName')), '2023-05-01').keys[0].value]" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('lakeStorageAccountName'))]", - "[resourceId('Microsoft.Storage/storageAccounts/fileServices/shares', variables('lakeStorageAccountName'), 'default', 'evidence-lake')]" - ] - }, { "type": "Microsoft.App/containerApps", "apiVersion": "2024-03-01", @@ -1837,20 +1740,7 @@ "resources": { "cpu": "[json('1.0')]", "memory": "2Gi" - }, - "volumeMounts": [ - { - "volumeName": "evidence", - "mountPath": "/evidence" - } - ] - } - ], - "volumes": [ - { - "name": "evidence", - "storageType": "AzureFile", - "storageName": "evidence-lake" + } } ], "scale": { @@ -1861,7 +1751,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-collector-identity', parameters('namePrefix')))]", - "[resourceId('Microsoft.App/managedEnvironments/storages', parameters('environmentName'), 'evidence-lake')]", "[resourceId('Microsoft.Resources/deployments', 'collector-roles')]" ] }, @@ -1912,27 +1801,13 @@ "resources": { "cpu": "[json('1.0')]", "memory": "2Gi" - }, - "volumeMounts": [ - { - "volumeName": "evidence", - "mountPath": "/evidence" - } - ] - } - ], - "volumes": [ - { - "name": "evidence", - "storageType": "AzureFile", - "storageName": "evidence-lake" + } } ] } }, "dependsOn": [ "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-collector-identity', parameters('namePrefix')))]", - "[resourceId('Microsoft.App/managedEnvironments/storages', parameters('environmentName'), 'evidence-lake')]", "[resourceId('Microsoft.Resources/deployments', 'collector-roles')]" ] }, diff --git a/server/internal/collect/native_local_value_attempt.go b/server/internal/collect/native_local_value_attempt.go new file mode 100644 index 0000000000..790d3f2c5d --- /dev/null +++ b/server/internal/collect/native_local_value_attempt.go @@ -0,0 +1,92 @@ +package collect + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/githubnext/gh-cao/server/internal/collectnative" + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/operationalvalue" +) + +// RecordNativeValueAttempt publishes a real collector outcome independently +// of cost accounting. The canonical writer checks the exact enrollment epoch +// and revision, so a withdrawn or changed target cannot regain health. +func (p Projector) RecordNativeValueAttempt( + ctx context.Context, repository, workflow string, attemptedAt time.Time, state, reason string, +) error { + if p.Data == nil || !p.Data.Open() || p.Enrollment.Authority == nil || p.PolicyPath == "" { + return collectnative.ErrUnsupported + } + repository = strings.ToLower(repository) + policy, err := p.resolvedNativeValuePolicy(ctx) + if err != nil { + return err + } + root, err := p.nativeLocalValueRoot() + if err != nil { + return err + } + snapshot, err := operationalvalue.ResolveWorkflowPolicy(ctx, + operationalvalue.Config{Root: root}, p.PolicyPath) + if err != nil { + return err + } + var selected *operationalvalue.WorkflowRegistration + for i := range snapshot.Registrations { + registration := &snapshot.Registrations[i] + if strings.EqualFold(registration.Repository, repository) && registration.Workflow == workflow { + selected = registration + break + } + } + if selected == nil || selected.Collector == "" { + return errors.New("value attempt lacks an enabled reviewed registration") + } + scope, err := p.Enrollment.Authority.CurrentScope(ctx, repository) + if err != nil { + return err + } + inputs, err := p.Data.ReadNativeValueInputs(ctx, []string{repository}) + if err != nil { + return err + } + digest := sha256.Sum256(policy) + record := model.Row{ + "schemaVersion": 1, + "id": "workflow-value-collection:" + url.QueryEscape(repository+":"+workflow), + "repository": repository, "workflow": workflow, + "sourcePath": ".github/workflows/" + workflow + ".md", + "policyDigest": fmt.Sprintf("%x", digest), + "observedAt": attemptedAt.UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z"), + "collector": selected.Collector, "registration": "enabled", + "state": "deferred", "reasonCode": "shared-budget-unavailable", + "reason": "Automatic remote acquisition is deferred until shared API request budgets can be enforced. Activity retains priority.", + } + for _, previous := range inputs.Collections { + if previous["id"] == record["id"] { + if previous["policyDigest"] == record["policyDigest"] { + record = previous + } else if previous["lastAttemptAt"] != nil { + record["lastAttemptAt"], record["lastAttemptOutcome"] = + previous["lastAttemptAt"], previous["lastAttemptOutcome"] + } + break + } + } + record, err = operationalvalue.CollectionAttempt(record, attemptedAt, state, reason) + if err != nil { + return err + } + if err := p.nativeLocalPolicyCurrent(ctx, policy); err != nil { + return err + } + _, err = p.Data.PublishNativeValuePreparation(ctx, inputs.Revision, []collectnative.EnrollmentScope{scope}, + []model.Row{record}, nil) + return err +} diff --git a/server/internal/collect/native_local_value_attempt_test.go b/server/internal/collect/native_local_value_attempt_test.go new file mode 100644 index 0000000000..e756eb1e07 --- /dev/null +++ b/server/internal/collect/native_local_value_attempt_test.go @@ -0,0 +1,49 @@ +package collect + +import ( + "testing" + "time" + + "github.com/githubnext/gh-cao/server/internal/query" +) + +func TestNativeRemoteAttemptHealthSurvivesLocalPreparation(t *testing.T) { + p, _ := nativeLocalValueFixture(t) + if err := p.CollectNativeLocalValues(t.Context()); err != nil { + t.Fatal(err) + } + attempted := time.Now().UTC().Truncate(time.Millisecond) + if err := p.RecordNativeValueAttempt(t.Context(), "octo/example", "verify-bugs", attempted, + "failed", "collector-failed"); err != nil { + t.Fatal(err) + } + read := func() map[string]any { + t.Helper() + result, _, err := p.Data.ExecuteSQLPlan(t.Context(), []query.Definition{ + {Name: "health", From: "$workflowValueCollections"}, + }, []string{"health"}) + if err != nil { + t.Fatal(err) + } + for _, row := range result["health"].Rows { + if row["repository"] == "octo/example" && row["workflow"] == "verify-bugs" { + return row + } + } + t.Fatal("reviewed collection health missing") + return nil + } + first := read() + if first["state"] != "failed" || first["reasonCode"] != "collector-failed" || + first["lastAttemptOutcome"] != "failed" || first["lastAttemptAt"] == nil { + t.Fatalf("real remote attempt was not recorded: %+v", first) + } + if err := p.CollectNativeLocalValues(t.Context()); err != nil { + t.Fatal(err) + } + later := read() + if later["state"] != "failed" || later["reasonCode"] != "collector-failed" || + later["lastAttemptAt"] != first["lastAttemptAt"] { + t.Fatalf("local preparation concealed actual collector attempt: %+v", later) + } +} diff --git a/server/internal/collect/native_local_value_policy.go b/server/internal/collect/native_local_value_policy.go index 0e812e9167..1de4115185 100644 --- a/server/internal/collect/native_local_value_policy.go +++ b/server/internal/collect/native_local_value_policy.go @@ -3,7 +3,6 @@ package collect import ( "context" "path/filepath" - "time" "github.com/githubnext/gh-cao/server/internal/operationalvalue" ) @@ -17,9 +16,8 @@ func (p Projector) resolvedNativeValuePolicy(ctx context.Context) ([]byte, error if err != nil { return nil, err } - snapshot, err := operationalvalue.ResolveWorkflowPolicy(ctx, operationalvalue.Config{ - Root: root, NodeBinary: p.node(), Environment: collectionEnvironment(""), WorkerTimeout: 30 * time.Second, - }, policy) + snapshot, err := operationalvalue.ResolveWorkflowPolicy(ctx, + operationalvalue.Config{Root: root}, policy) if err != nil { return nil, err } @@ -31,7 +29,6 @@ func (p Projector) nativeLocalValueRoot() (string, error) { if err != nil { return "", err } - // Node resolves the module's real path before its argv/import.meta entrypoint - // check. Preserve that identity through catalog aliases such as macOS /var. + // Canonicalize aliases before resolving confined reviewed evidence files. return filepath.EvalSymlinks(root) } diff --git a/server/internal/collect/native_local_value_policy_test.go b/server/internal/collect/native_local_value_policy_test.go index be4662b8ff..420c65b9c9 100644 --- a/server/internal/collect/native_local_value_policy_test.go +++ b/server/internal/collect/native_local_value_policy_test.go @@ -68,7 +68,11 @@ func TestNativeLocalValuesTracksComposedBasePolicyAuthority(t *testing.T) { } result, _, err := p.Data.ExecuteSQLPlan(t.Context(), []query.Definition{{Name: "health", From: "$workflowValueCollections"}}, []string{"health"}) - if err != nil || len(result["health"].Rows) != 1 || result["health"].Rows[0]["reasonCode"] != "activity-disabled" { + disabled := false + for _, row := range result["health"].Rows { + disabled = disabled || row["repository"] == "octo/example" && row["reasonCode"] == "activity-disabled" + } + if err != nil || len(result["health"].Rows) != 2 || !disabled { t.Fatalf("changed composed authority did not refresh collection health: %+v %v", result, err) } } @@ -89,13 +93,13 @@ func TestNativeLocalValuesCanonicalizesCatalogAliases(t *testing.T) { t.Fatalf("native runtime did not canonicalize the configured catalog: %q %v", canonical, err) } if err := p.CollectNativeLocalValues(t.Context()); err != nil { - t.Fatalf("catalog alias skipped a real Node entrypoint: %v", err) + t.Fatalf("catalog alias skipped Go-native Value evaluation: %v", err) } results, _, err := p.Data.ExecuteSQLPlan(t.Context(), []query.Definition{ {Name: "values", From: "$operationalValues"}, {Name: "health", From: "$workflowValueCollections"}, }, []string{"values", "health"}) - if err != nil || len(results["values"].Rows) == 0 || len(results["health"].Rows) != 1 { + if err != nil || len(results["values"].Rows) == 0 || len(results["health"].Rows) != 2 { t.Fatalf("aliased runtime returned success without computing and publishing: %+v %v", results, err) } if _, err := p.Data.ReadOperationalValue(t.Context(), "octo/example", "verify-bugs"); err != nil { diff --git a/server/internal/collect/native_local_values.go b/server/internal/collect/native_local_values.go index 8b0b7eed6d..f4dbedc9a9 100644 --- a/server/internal/collect/native_local_values.go +++ b/server/internal/collect/native_local_values.go @@ -27,7 +27,7 @@ type nativeLocalValueProgress struct { CompletedAt time.Time `json:"completedAt"` } -// CollectNativeLocalValues is the optional Node/catalog Value coexistence lane. +// CollectNativeLocalValues is the Go-native hosted Value preparation lane. // PostgreSQL owns input, accepted evidence and publication. Scratch contains // only bounded per-invocation transport files and is never a recovery source. func (p Projector) CollectNativeLocalValues(ctx context.Context) error { @@ -92,7 +92,11 @@ func (p Projector) CollectNativeLocalValues(ctx context.Context) error { if err != nil { return err } - workspace, err := os.MkdirTemp(".", ".native-local-value-") + root, err := p.nativeLocalValueRoot() + if err != nil { + return err + } + workspace, err := os.MkdirTemp(root, ".native-local-value-") if err != nil { return err } @@ -108,12 +112,7 @@ func (p Projector) CollectNativeLocalValues(ctx context.Context) error { if err := writeNativeValueInputs(workspace, inputs); err != nil { return err } - root, err := p.nativeLocalValueRoot() - if err != nil { - return err - } - config := operationalvalue.Config{Root: root, NodeBinary: p.node(), Repositories: repositories, - Environment: collectionEnvironment(p.GitHubAPIURL), WorkerTimeout: 30 * time.Second} + config := operationalvalue.Config{Root: root, Repositories: repositories, WorkerTimeout: 30 * time.Second} if err := operationalvalue.WriteCollectionHealth(ctx, config, policyPath, workspace); err != nil { return err } @@ -228,12 +227,6 @@ func (p Projector) computeNativeLocalValues( output := filepath.Join(workspace, "operational-values.jsonl") config.WorkerTimeout = 2 * time.Minute var retained []model.Row - retainedScope := make([]string, 0, len(enrolled)) - for _, repository := range enrolled { - if !slices.Contains(selected, repository) { - retainedScope = append(retainedScope, repository) - } - } err := p.Data.WithReadTransaction(ctx, func(readCtx context.Context, reader postgresx.NativeReader) error { state, err := reader.State(readCtx) if err != nil { @@ -242,14 +235,14 @@ func (p Projector) computeNativeLocalValues( if state.Revision != expectedRevision { return postgresx.ErrOperationalValueRevisionStale } - // Disabled/out-of-policy repositories keep accepted history. The runtime - // receives only its reviewed plan scope, never all ingestion enrollment. - if len(retainedScope) != 0 { + // The evaluator writes only new observations. Preserve previous accepted + // history for all enrolled repositories, including the selected scope. + if len(enrolled) != 0 { cutoff := time.Now().UTC().Add(-operationalvalue.DefaultRetention).Format(time.RFC3339Nano) sources, _, err := reader.ExecuteSQLPlanWithOptions(readCtx, []query.Definition{{ Name: "retained", From: "$operationalValues", Filter: &query.Filter{Predicates: []query.Predicate{{Field: "timestamp", GTE: cutoff}}}, - }}, []string{"retained"}, postgresx.SQLExecutionOptions{Repositories: retainedScope}) + }}, []string{"retained"}, postgresx.SQLExecutionOptions{Repositories: enrolled}) if err != nil { return err } @@ -287,13 +280,85 @@ func (p Projector) computeNativeLocalValues( return 0, errors.New("native local Value combined history exceeds its row budget") } } + kept := nativeLocalValueRetainedRows(retained, rows) if err := p.nativeLocalPolicyCurrent(ctx, policy); err != nil { return 0, err } - state, err := p.Data.ReplaceOperationalValues(ctx, expectedRevision, append(retained, rows...)) + state, err := p.Data.ReplaceOperationalValues(ctx, expectedRevision, append(kept, rows...)) return state.Revision, err } +// A repeated evaluation replaces the same frozen interval. The legacy +// Dependabot missing placeholder predates its evaluator digest; only a +// complete numeric observation at that exact cutoff may supersede it. +func nativeLocalValueRetainedRows(retained, rows []model.Row) []model.Row { + updated := make(map[string]bool, len(rows)) + key := func(row model.Row) string { + return strings.Join([]string{textValue(row["campaign"]), textValue(row["repository"]), + textValue(row["valueId"]), textValue(row["evaluator-digest"]), + textValue(row["window-start"]), textValue(row["window-end"])}, "\x00") + } + for _, row := range rows { + updated[key(row)] = true + } + legacyReplacements := map[string]bool{} + for _, row := range rows { + if nativeLocalValueCompleteDependabot(row) { + legacyReplacements[nativeLocalValueWindowKey(row)] = true + } + } + kept := retained[:0] + for _, row := range retained { + if !updated[key(row)] && (!nativeLocalValueLegacyDependabotMissing(row) || + !legacyReplacements[nativeLocalValueWindowKey(row)]) { + kept = append(kept, row) + } + } + return kept +} + +func nativeLocalValueWindowKey(row model.Row) string { + cutoff := func(field string) string { + value := textValue(row[field]) + at, err := time.Parse(time.RFC3339Nano, value) + if err != nil || !strings.HasSuffix(value, "Z") { + return value + } + return at.UTC().Format(time.RFC3339Nano) + } + return strings.Join([]string{textValue(row["campaign"]), textValue(row["repository"]), + textValue(row["valueId"]), cutoff("window-start"), cutoff("window-end")}, "\x00") +} + +func nativeLocalValueDependabotMetric(row model.Row) bool { + if row["campaign"] != "dependabot" || row["repository"] != dependabotValueRepository || + row["measurement-kind"] != "snapshot" { + return false + } + for _, metric := range dependabotValueMetrics { + if row["valueId"] == dependabotValueWorkflow+"."+metric.id { + return true + } + } + return false +} + +func nativeLocalValueCompleteDependabot(row model.Row) bool { + _, numeric := row["value"].(json.Number) + return nativeLocalValueDependabotMetric(row) && numeric && + row["coverage-status"] == "complete" && row["maturity-status"] == "matured" && + row["missing-reason"] == nil && textValue(row["evaluator-digest"]) != "" && + textValue(row["window-start"]) != "" && textValue(row["window-end"]) != "" +} + +func nativeLocalValueLegacyDependabotMissing(row model.Row) bool { + return nativeLocalValueDependabotMetric(row) && row["value"] == nil && + row["coverage-status"] == "missing" && row["maturity-status"] == "unavailable" && + row["missing-reason"] == "remote-acquisition-unavailable" && + textValue(row["evaluator-digest"]) == "" && + textValue(row["window-start"]) != "" && textValue(row["window-end"]) != "" +} + func textValue(value any) string { text, _ := value.(string) return text diff --git a/server/internal/collect/native_local_values_test.go b/server/internal/collect/native_local_values_test.go index 6602d7ea0b..c837f5dee5 100644 --- a/server/internal/collect/native_local_values_test.go +++ b/server/internal/collect/native_local_values_test.go @@ -8,13 +8,13 @@ import ( "fmt" "os" "path/filepath" - "strings" "testing" "time" "github.com/githubnext/gh-cao/server/internal/collectnative" "github.com/githubnext/gh-cao/server/internal/model" "github.com/githubnext/gh-cao/server/internal/operational" + "github.com/githubnext/gh-cao/server/internal/operationalvalue" "github.com/githubnext/gh-cao/server/internal/postgresx" "github.com/githubnext/gh-cao/server/internal/query" "github.com/githubnext/gh-cao/server/internal/testutil" @@ -25,16 +25,12 @@ func nativeLocalValueFixture(t *testing.T) (Projector, string) { data := testutil.Postgres(t, t.Context(), "CAO_TEST_POSTGRES_URL", "POSTGRES_URL") queue, _, _ := nativeRunQueue(t) enrollment := nativeEnrollmentFixture(t, data, queue) - if err := enrollment.AddRepositories(t.Context(), 7, []string{"octo/example", "octo/secret"}); err != nil { + if err := enrollment.AddRepositories(t.Context(), 7, []string{"octo/example", "octo/secret", "githubnext/gh-cao"}); err != nil { t.Fatal(err) } services := queue.Tasks.(operational.Store).OperationalServices() root := t.TempDir() - repository, err := filepath.Abs("../../..") - if err != nil { - t.Fatal(err) - } - for _, directory := range []string{"activity", "optimization/operational-value", ".github/value-evidence"} { + for _, directory := range []string{"optimization/operational-value", ".github/value-evidence"} { if err := os.MkdirAll(filepath.Join(root, directory), 0o700); err != nil { t.Fatal(err) } @@ -45,62 +41,22 @@ func nativeLocalValueFixture(t *testing.T) (Projector, string) { t.Fatal(err) } } - for _, name := range []string{"dashboard", ".github/workflows"} { - if err := os.Symlink(filepath.Join(repository, name), filepath.Join(root, name)); err != nil { - t.Fatal(err) - } - } - files, err := os.ReadDir(filepath.Join(repository, "activity")) + repository, err := filepath.Abs("../../..") if err != nil { t.Fatal(err) } - for _, file := range files { - source := filepath.Join(repository, "activity", file.Name()) - if file.Name() == "local-values.mjs" || file.Name() == "workflow-values.mjs" { - content, err := os.ReadFile(source) // #nosec G304 -- entries enumerated from the checked-out Activity runtime. - if err != nil { - t.Fatal(err) - } - write("activity/"+file.Name(), content) - } else if err := os.Symlink(source, filepath.Join(root, "activity", file.Name())); err != nil { - t.Fatal(err) - } - } - adapter, err := os.ReadFile(filepath.Join(repository, "optimization", "operational-value.mjs")) // #nosec G304 -- fixed checked-out pinned adapter. + adapter, err := os.ReadFile(filepath.Join(repository, "optimization", "operational-value.mjs")) // #nosec G304 -- checked-in local adapter fingerprint. if err != nil { t.Fatal(err) } write("optimization/operational-value.mjs", adapter) - // Uses the production pinned adapter and canonical socket client. An - // intentionally scope-free count must still see only the reviewed target. - write("optimization/operational-value/fixture.mjs", []byte(` -import { executeCanonicalQuery } from "../../activity/canonical-query.mjs"; -export const localCollection = true; -export const definition = { - slug:"fixture", workflowName:"Fixture", adoption:{adoptedAt:"2026-10-01T00:00:00.000Z"}, - evaluation:{mode:"attainment-only"}, - evidence:{repositories:["octo/example"],measurementKind:"interval", - window:{durationDays:1,maturationDays:0,cadenceDays:365}}, - metrics:[{id:"count",role:"primary",name:"Count",unit:"runs",direction:"increase"}] -}; -export async function collectBatch(windows, options) { - if (!options.localOnly || options.database) throw new Error("local canonical execution required"); - const result = await executeCanonicalQuery({name:"count",from:"runs", - aggregate:{values:[{field:"id",as:"count",reducer:"count"}]}}); - const count = result.rows[0].count; - if (count !== 1) throw new Error("reviewed canonical scope leaked"); - return windows.map(() => ({evidence:{count,coverageStatus:"partial", - missingReason:"fixture-partial-coverage"},provenance:{}})); -} -export const scoreMetric = (_metric, evidence) => evidence.count; -`)) evidence, err := os.ReadFile(filepath.Join(repository, "dashboard/site/test/fixtures/operational-value-evidence.jsonl")) // #nosec G304 -- fixed checked-out contract fixture. if err != nil { t.Fatal(err) } write(".github/value-evidence/fixture.jsonl", evidence) policy := fmt.Sprintf(`{"version":1,"gh-aw-version":"v0.93.1","control-plane":{ - "scope":{"allowed-repositories":["octo/example"]},"activity":{"repositories":["octo/example"]}, + "scope":{"allowed-repositories":["octo/example","githubnext/gh-cao"]},"activity":{"repositories":["octo/example","githubnext/gh-cao"]}, "campaigns":{"activity":{"workflow-values":[{"repository":"octo/example","workflow":"verify-bugs", "collector":"not-present.mjs","evidence":{"path":".github/value-evidence/fixture.jsonl","sha256":"%x"}}]}}}}`, sha256.Sum256(evidence)) @@ -126,6 +82,27 @@ export const scoreMetric = (_metric, evidence) => evidence.count; } } } + coordinate := "githubnext/gh-cao" + repositoryID := "repository:" + model.EncodeCoordinate(coordinate) + workflowID := "workflow:" + model.EncodeCoordinate(coordinate+":.github/workflows/worker.md") + for source, row := range map[string]model.Row{ + "$repositories": {"id": repositoryID, "owner": "githubnext", "name": "gh-cao"}, + "$workflows": {"id": workflowID, "repositoryId": repositoryID, "path": ".github/workflows/worker.md"}, + } { + if err := writer.Append(t.Context(), source, row); err != nil { + t.Fatal(err) + } + } + for i := 1; i <= 3; i++ { + if err := writer.Append(t.Context(), "$runs", model.Row{ + "id": fmt.Sprintf("run:optimizer-%d", i), "repositoryId": repositoryID, "workflowId": workflowID, + "owner": "githubnext", "repository": "gh-cao", "githubRunId": fmt.Sprint(i), + "startedAt": time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339Nano), + "status": "completed", "conclusion": "success", "aicTotal": 1, + }); err != nil { + t.Fatal(err) + } + } if err := writer.Append(t.Context(), "$campaigns", model.Row{ "id": "campaign:optimization", "slug": "optimization", "name": "Optimization", }); err != nil { @@ -156,12 +133,21 @@ func TestNativeLocalValuesProductionCanonicalQueriesHealthAndReviewedImports(t * t.Fatal(err) } values, health := results["values"].Rows, results["health"].Rows - if len(values) == 0 || fmt.Sprint(values[0]["value"]) != "1" || - values[0]["coverage-status"] != "partial" || values[0]["repository"] != "octo/example" { + if len(values) != 3 || fmt.Sprint(values[0]["value"]) != "1" || + values[0]["coverage-status"] != "unknown" || values[0]["repository"] != "githubnext/gh-cao" { t.Fatalf("canonical local computation lost scope/partial outputs: %+v", values) } - if len(health) != 1 || health[0]["registration"] != "enabled" || - health[0]["reasonCode"] != "shared-budget-unavailable" || health[0]["lastAttemptAt"] != nil { + if len(health) != 2 { + t.Fatalf("native collection health inventory disappeared: %+v", health) + } + var reviewed model.Row + for _, row := range health { + if row["repository"] == "octo/example" && row["workflow"] == "verify-bugs" { + reviewed = row + } + } + if reviewed == nil || reviewed["registration"] != "enabled" || + reviewed["reasonCode"] != "shared-budget-unavailable" || reviewed["lastAttemptAt"] != nil { t.Fatalf("native collection health disappeared or invented an attempt: %+v", health) } if results["health"].Metadata["availability"] != "available" || @@ -193,18 +179,24 @@ func TestNativeLocalValuesProductionCanonicalQueriesHealthAndReviewedImports(t * } func TestNativeLocalValuesFailureDoesNotConsumeInputs(t *testing.T) { - p, _ := nativeLocalValueFixture(t) + p, root := nativeLocalValueFixture(t) p.NodeBinary = "/nonexistent-node" + unsupported := filepath.Join(root, "optimization", "operational-value", "unreviewed.mjs") + if err := os.WriteFile(unsupported, []byte(`export const localCollection = true`), 0o600); err != nil { + t.Fatal(err) + } if err := p.CollectNativeLocalValues(t.Context()); err == nil { - t.Fatal("missing selected Node runtime was silently disabled") + t.Fatal("unreviewed JS-only local adapter was silently disabled") } completed, err := p.State.OperationalState(t.Context(), nativeLocalValueCompletedKey) if err != nil || len(completed) != 0 { t.Fatalf("failed execution consumed input: %q %v", completed, err) } - p.NodeBinary = "node" + if err := os.Remove(unsupported); err != nil { + t.Fatal(err) + } if err := p.CollectNativeLocalValues(t.Context()); err != nil { - t.Fatalf("retry required an Activity projection or file recovery: %v", err) + t.Fatalf("retry required a Node subprocess or Activity projection: %v", err) } } @@ -218,59 +210,26 @@ func TestNativeLocalValuesUnselectedAndMisconfigured(t *testing.T) { } func TestNativeLocalValuesWithdrawalDuringEvaluationCannotResurrectMeasurements(t *testing.T) { - p, root := nativeLocalValueFixture(t) - module := filepath.Join(root, "optimization/operational-value/fixture.mjs") - source, err := os.ReadFile(module) // #nosec G304 -- fixed module in the test-owned catalog. + p, _ := nativeLocalValueFixture(t) + before, err := p.Data.State(t.Context()) if err != nil { t.Fatal(err) } - started, release := filepath.Join(root, "started"), filepath.Join(root, "release") - startedJSON, _ := json.Marshal(started) - releaseJSON, _ := json.Marshal(release) - wait := `const fs = await import("node:fs/promises"); - await fs.writeFile(` + string(startedJSON) + `,"ready"); - for (;;) { - try { await fs.stat(` + string(releaseJSON) + `); break; } catch {} - await new Promise(resolve => setTimeout(resolve,10)); - } - const count = result.rows[0].count;` - if err := os.WriteFile(module, []byte(strings.Replace(string(source), "const count = result.rows[0].count;", wait, 1)), 0o600); err != nil { // #nosec G703 -- fixed module in the test-owned catalog. - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) - defer cancel() - done := make(chan error, 1) - go func() { done <- p.CollectNativeLocalValues(ctx) }() - deadline := time.Now().Add(15 * time.Second) - for { - if _, err := os.Stat(started); err == nil { - break - } - select { - case err := <-done: - t.Fatalf("local evaluation did not reach the canonical query: %v", err) - default: - } - if time.Now().After(deadline) { - t.Fatal("local evaluation did not start before its deadline") - } - time.Sleep(10 * time.Millisecond) - } - if _, err := p.Enrollment.RemoveRepositories(ctx, 7, []string{"octo/example"}); err != nil { + if _, err := p.Enrollment.RemoveRepositories(t.Context(), 7, []string{"octo/example"}); err != nil { t.Fatal(err) } - if err := os.WriteFile(release, []byte("ready"), 0o600); err != nil { - t.Fatal(err) - } - if err := <-done; !errors.Is(err, postgresx.ErrOperationalValueRevisionStale) { + config := operationalvalue.Config{Root: p.CatalogRoot} + _, err = p.computeNativeLocalValues(t.Context(), config, p.PolicyPath, t.TempDir(), nil, + []string{"octo/example", "githubnext/gh-cao"}, []string{"githubnext/gh-cao"}, before.Revision) + if !errors.Is(err, postgresx.ErrOperationalValueRevisionStale) { t.Fatalf("withdrawn source published evaluated measurements: %v", err) } nativeCanonicalCount(t, p.Data, "$operationalValues", 0) nativeCanonicalCount(t, p.Data, "$workflowValueCollections", 0) - if _, err := p.Data.ReadOperationalValue(ctx, "octo/example", "verify-bugs"); !errors.Is(err, postgresx.ErrValueParentMissing) { + if _, err := p.Data.ReadOperationalValue(t.Context(), "octo/example", "verify-bugs"); !errors.Is(err, postgresx.ErrValueParentMissing) { t.Fatalf("withdrawn source retained imported evidence: %v", err) } - completed, err := p.State.OperationalState(ctx, nativeLocalValueCompletedKey) + completed, err := p.State.OperationalState(t.Context(), nativeLocalValueCompletedKey) if err != nil || len(completed) != 0 { t.Fatalf("stale computation consumed its durable input: %q %v", completed, err) } @@ -317,6 +276,67 @@ func TestNativeLocalValueDefinitionsOnlyTranslateCanonicalRoots(t *testing.T) { } } +func TestNativeLocalValueNumericDependabotReplacesOnlyMatchingLegacyMissing(t *testing.T) { + windowStart, windowEnd := "2026-09-17T08:48:00Z", "2026-09-24T08:48:00Z" + newRow := model.Row{ + "campaign": "dependabot", "repository": dependabotValueRepository, + "valueId": dependabotValueWorkflow + ".open-security-alert-count", + "measurement-kind": "snapshot", "coverage-status": "complete", + "maturity-status": "matured", "evaluator-digest": "frozen-v1", + "window-start": windowStart, "window-end": windowEnd, + "value": json.Number("0"), "timestamp": "2026-09-25T00:00:00Z", + } + old := model.Row{} + for field, value := range newRow { + old[field] = value + } + old["value"], old["coverage-status"], old["maturity-status"] = nil, "missing", "unavailable" + old["missing-reason"], old["evaluator-digest"] = "remote-acquisition-unavailable", nil + old["timestamp"] = "2026-09-24T10:00:00Z" + if got := nativeLocalValueRetainedRows([]model.Row{old}, []model.Row{newRow}); len(got) != 0 { + t.Fatalf("legacy missing remained beside numeric same-cutoff result: %+v", got) + } + old["window-start"], old["window-end"] = "2026-09-17T08:48:00.000000Z", "2026-09-24T08:48:00.000000Z" + if got := nativeLocalValueRetainedRows([]model.Row{old}, []model.Row{newRow}); len(got) != 0 { + t.Fatalf("Postgres timestamp precision prevented same-cutoff replacement: %+v", got) + } + for _, changed := range []struct { + field string + value any + }{ + {"window-end", "2026-10-01T08:48:00Z"}, + {"valueId", dependabotValueWorkflow + ".open-high-critical-alert-count"}, + {"campaign", "optimization"}, + {"repository", "octo/other"}, + {"missing-reason", "incomplete-search"}, + {"evaluator-digest", "older-frozen-evaluator"}, + } { + mismatch := model.Row{} + for field, value := range old { + mismatch[field] = value + } + mismatch[changed.field] = changed.value + if got := nativeLocalValueRetainedRows([]model.Row{mismatch}, []model.Row{newRow}); len(got) != 1 { + t.Fatalf("unrelated missing assessment removed on %s mismatch: %+v", changed.field, got) + } + } + for _, changed := range []struct { + field string + value any + }{ + {"value", nil}, {"coverage-status", "partial"}, {"maturity-status", "unavailable"}, + } { + incomplete := model.Row{} + for field, value := range newRow { + incomplete[field] = value + } + incomplete[changed.field] = changed.value + if got := nativeLocalValueRetainedRows([]model.Row{old}, []model.Row{incomplete}); len(got) != 1 { + t.Fatalf("non-numeric/incomplete observation replaced legacy evidence: %+v", got) + } + } +} + func TestNativeLocalValueTransportRejectsMalformedEnvelope(t *testing.T) { filename := filepath.Join(t.TempDir(), "evidence.jsonl") content, err := json.Marshal(map[string]any{"schema_version": 2, "kind": "workflow_value_collection", @@ -371,7 +391,12 @@ func TestNativeLocalValuesActivityDisableRetainsEvidenceAndMeasurements(t *testi t.Fatalf("disabled Activity dropped retained measurements: %+v %v", after, err) } health := after["health"].Rows - if len(health) != 1 || health[0]["state"] != "disabled" || health[0]["reasonCode"] != "activity-disabled" { + disabled := false + for _, row := range health { + disabled = disabled || row["repository"] == "octo/example" && + row["state"] == "disabled" && row["reasonCode"] == "activity-disabled" + } + if len(health) != 2 || !disabled { t.Fatalf("disabled collection decision was lost: %+v", health) } if _, err := p.Data.ReadOperationalValue(t.Context(), "octo/example", "verify-bugs"); err != nil { diff --git a/server/internal/collect/native_values.go b/server/internal/collect/native_values.go index 120dfe2e5a..732af0f42c 100644 --- a/server/internal/collect/native_values.go +++ b/server/internal/collect/native_values.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "os/exec" "path/filepath" "strconv" "strings" @@ -14,9 +13,8 @@ import ( "github.com/githubnext/gh-cao/server/internal/collectnative" ) -// NativeValueService is a separately selected execution lane for existing -// reviewed Value programs. Only that lane needs Node/catalog/policy assets; -// native acquisition never stages an Activity lake for it. +// NativeValueService is a separately selected execution lane for reviewed +// Value programs. Native acquisition never stages an Activity lake for it. type NativeValueService struct { Projector Projector } @@ -42,29 +40,26 @@ func (s NativeValueService) Validate() error { return s.ValidateRuntime() } -// ValidateRuntime checks only the selected Value lane's deployed dependencies, -// without opening credentials, executing programs or requiring backend handles. +// ValidateRuntime checks only the selected Value lane's deployed policy, +// without opening credentials or requiring backend handles. func (s NativeValueService) ValidateRuntime() error { p := s.Projector if p.CatalogRoot == "" || p.PolicyPath == "" { return errors.New("selected native Value execution requires its reviewed catalog and policy") } - if _, err := exec.LookPath(p.node()); err != nil { - return fmt.Errorf("selected native Value execution requires Node: %w", err) + root, err := os.Stat(p.CatalogRoot) + if err != nil { + return fmt.Errorf("selected native Value execution requires its reviewed catalog: %w", err) } - for _, path := range []string{ - p.PolicyPath, - filepath.Join(p.CatalogRoot, "activity", "local-values.mjs"), - filepath.Join(p.CatalogRoot, "activity", "workflow-values.mjs"), - filepath.Join(p.CatalogRoot, ".github", "workflows", "shared", "policy.mjs"), - } { - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("selected native Value execution requires deployed policy/catalog assets: %w", err) - } - if !info.Mode().IsRegular() { - return errors.New("selected native Value policy/catalog asset is not a regular file") - } + if !root.IsDir() { + return errors.New("selected native Value catalog is not a directory") + } + info, err := os.Stat(p.PolicyPath) + if err != nil { + return fmt.Errorf("selected native Value execution requires its reviewed policy: %w", err) + } + if !info.Mode().IsRegular() { + return errors.New("selected native Value policy is not a regular file") } return nil } diff --git a/server/internal/collect/native_values_test.go b/server/internal/collect/native_values_test.go index 8da9503e1d..3d84833869 100644 --- a/server/internal/collect/native_values_test.go +++ b/server/internal/collect/native_values_test.go @@ -7,9 +7,7 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" - "strings" "testing" "github.com/githubnext/gh-cao/server/internal/collectnative" @@ -97,7 +95,7 @@ func TestNativeValuePlanTracksComposedPolicyAuthority(t *testing.T) { } } -func TestNativeValueCatalogAliasRunsTheProductionWorkerEntrypoint(t *testing.T) { +func TestNativeValueCatalogAliasResolvesToReviewedRoot(t *testing.T) { root, err := filepath.Abs("../../..") if err != nil { t.Fatal(err) @@ -110,10 +108,7 @@ func TestNativeValueCatalogAliasRunsTheProductionWorkerEntrypoint(t *testing.T) if err != nil { t.Fatal(err) } - command := exec.CommandContext(t.Context(), "node", filepath.Join(resolved, "activity", "workflow-values.mjs"), "--worker") // #nosec G204 -- fixed reviewed runtime and invalid-input fixture. - command.Stdin = strings.NewReader("{}") - output, err := command.CombinedOutput() - if err == nil || !strings.Contains(string(output), remoteValueFailurePrefix) { - t.Fatalf("production worker silently skipped its entrypoint through a catalog alias: %v: %s", err, output) + if resolved != root { + t.Fatalf("catalog alias must resolve to reviewed root: got %q want %q", resolved, root) } } diff --git a/server/internal/collect/projector_remote_value.go b/server/internal/collect/projector_remote_value.go index 0139bcbbd0..31d69169cc 100644 --- a/server/internal/collect/projector_remote_value.go +++ b/server/internal/collect/projector_remote_value.go @@ -13,7 +13,6 @@ import ( "net/http" "net/url" "os" - "os/exec" "path/filepath" "strings" "time" @@ -25,10 +24,8 @@ import ( var ErrRemoteValueDeferred = errors.New("remote Value acquisition deferred") -const remoteValueFailurePrefix = "CAO_WORKFLOW_VALUE_FAILURE " - -// CollectScopedRemoteValues preserves the separately selected reviewed Value -// runtime without giving native run acquisition a Node or evidence-lake dependency. +// CollectScopedRemoteValues uses the reviewed Go acquisition under one exact +// enrollment epoch, independently of native run acquisition. func (p Projector) CollectScopedRemoteValues(ctx context.Context, scope collectnative.EnrollmentScope) error { if p.Enrollment.Authority == nil || p.Data == nil { return collectnative.ErrUnsupported @@ -60,8 +57,7 @@ func (p Projector) nativeValueCurrent(ctx context.Context) error { } // CollectRemoteValues works on one exact enrolled repository. The caller owns -// the per-repository lease and durable dirty hint. This method never holds the -// projection lease or an App token in its child process. +// the per-repository lease and durable dirty hint. func (p Projector) CollectRemoteValues(ctx context.Context, repository string) error { name, err := NormalizeRepository(repository) if err != nil { @@ -71,6 +67,9 @@ func (p Projector) CollectRemoteValues(ctx context.Context, repository string) e p.Tokens == nil || p.Leases == nil || p.Enrollment.Metadata == nil { return fmt.Errorf("%w: hosted Postgres, shared quota and App enrollment are required", ErrRemoteValueDeferred) } + if p.nativeValueScope == nil { + return fmt.Errorf("%w: remote Value requires an exact native enrollment epoch", ErrRemoteValueDeferred) + } entries, policyDigest, err := p.remoteValuePlan(ctx) if err != nil { return err @@ -84,6 +83,10 @@ func (p Projector) CollectRemoteValues(ctx context.Context, repository string) e if len(selected) == 0 { return nil } + valid, attempts := supportedRemoteValueEntries(selected) + if len(valid) == 0 { + return attempts + } enrolled, err := p.Enrollment.Enrolled(ctx, name) if err != nil { return err @@ -95,7 +98,7 @@ func (p Projector) CollectRemoteValues(ctx context.Context, repository string) e if !enrolled || installation <= 0 { return fmt.Errorf("%w: remote Value repository is no longer enrolled", ErrRemoteValueDeferred) } - plan := compileRemoteValueAcquisitionPlan(selected) + plan := compileRemoteValueAcquisitionPlan(valid) var shared *remoteValueResponseCache if len(plan.Shared) > 0 { shared = newRemoteValueResponseCache() @@ -104,7 +107,7 @@ func (p Projector) CollectRemoteValues(ctx context.Context, repository string) e if err != nil { return err } - for _, entry := range selected { + for _, entry := range valid { cache := (*remoteValueResponseCache)(nil) requirements := plan.Shared[entry.Workflow] if len(requirements) > 0 { @@ -113,10 +116,23 @@ func (p Projector) CollectRemoteValues(ctx context.Context, repository string) e if err := p.collectRemoteValue( ctx, name, installation, entry, policyDigest, apiBase, cache, requirements, ); err != nil { - return err + attempts = errors.Join(attempts, err) } } - return nil + return attempts +} + +func supportedRemoteValueEntries(entries []remoteValueEntry) ([]remoteValueEntry, error) { + valid := make([]remoteValueEntry, 0, len(entries)) + var failures error + for _, entry := range entries { + if err := reviewedRemoteValue(entry); err != nil { + failures = errors.Join(failures, fmt.Errorf("%s: %w", entry.Workflow, err)) + continue + } + valid = append(valid, entry) + } + return valid, failures } func (p Projector) collectRemoteValue( @@ -124,16 +140,18 @@ func (p Projector) collectRemoteValue( policyDigest [sha256.Size]byte, base *url.URL, cache *remoteValueResponseCache, sharedRequirements map[string]remoteValueRequirement, ) (resultErr error) { + if entry.Workflow == dependabotValueWorkflow { + return p.collectDependabotValue(ctx, repository, installation, entry, policyDigest, base, cache, sharedRequirements) + } ctx, cancel := context.WithTimeout(ctx, 8*time.Minute) defer cancel() - var previous model.Row - var restored []postgresx.ValueProofFile - var err error - if p.nativeValueScope != nil { - previous, restored, err = p.Data.Acquisition().ReadValue(ctx, *p.nativeValueScope, entry.Workflow) - } else { - previous, err = p.Data.ReadOperationalValue(ctx, repository, entry.Workflow) + if err := reviewedRemoteValue(entry); err != nil { + return err } + if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { + return err + } + previous, restored, err := p.Data.Acquisition().ReadValue(ctx, *p.nativeValueScope, entry.Workflow) switch { case errors.Is(err, postgresx.ErrValueParentMissing): return fmt.Errorf("%w: canonical workflow parent has not been projected", ErrRemoteValueDeferred) @@ -142,38 +160,8 @@ func (p Projector) collectRemoteValue( case err != nil: return err } - var root string - if p.nativeValueScope == nil { - root, err = p.remoteValueWorkspace(ctx, repository, entry.Workflow) - if err != nil { - return err - } - } - work, err := os.MkdirTemp(root, "cao-value-invocation-") - if err != nil { - return err - } - defer func() { _ = os.RemoveAll(work) }() - if p.nativeValueScope != nil { - if err := writeRemoteValueProof(restored, filepath.Join(work, "latest")); err != nil { - return err - } - } else { - if err := restoreRemoteValueProof(ctx, p.Data, repository, entry.Workflow, filepath.Join(work, "latest")); err != nil { - return err - } - } - bin := filepath.Join(work, "bin") - if err := os.Mkdir(bin, 0o700); err != nil { - return err - } - if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(remoteValueGH), 0o700); err != nil { // #nosec G306 -- owner-only executable bridge in a private invocation directory. - return err - } - token, err := p.Tokens.InstallationToken(ctx, installation) - if err != nil { - return err - } + started := time.Now() + failureReason, outputBytes := "", int64(0) transport := p.RemoteHTTP if transport == nil { transport = &http.Client{Timeout: 65 * time.Second, @@ -182,44 +170,13 @@ func (p Projector) collectRemoteValue( api := &remoteValueAPI{ client: transport, quota: p.RemoteQuota, searchQuota: p.RemoteSearchQuota, app: p.QuotaApp, installation: installation, repository: repository, collector: entry.Collector, - base: base, token: token, max: entry.APIRequests, cache: cache, + base: base, max: entry.APIRequests, cache: cache, sharedRequirements: sharedRequirements, authorize: p.nativeValueCurrent, } - socket, closeAPI, err := api.listen(ctx) - if err != nil { - return err - } - defer closeAPI() - request := struct { - Root string `json:"root"` - Entry json.RawMessage `json:"entry"` - Previous model.Row `json:"previous,omitempty"` - ObservedAt string `json:"observedAt"` - Workspace string `json:"workspace"` - }{ - Root: p.CatalogRoot, Entry: entry.Raw, Previous: previous, - ObservedAt: time.Now().UTC().Format(time.RFC3339Nano), Workspace: work, - } - input, err := json.Marshal(request) - if err != nil || len(input) > 8<<20 { - return errors.New("remote Value invocation exceeds its input budget") - } - started := time.Now() - command := exec.CommandContext(ctx, p.node(), filepath.Join(p.CatalogRoot, "activity", "workflow-values.mjs"), "--worker") // #nosec G204 -- reviewed runtime, private environment. - command.Dir = p.CatalogRoot - command.Env = append(collectionEnvironment(""), - "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), - "HOME="+work, "GH_TOKEN=remote-value-proxy-only", "GITHUB_TOKEN=remote-value-proxy-only", - "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", - "CAO_REMOTE_VALUE_SOCKET="+socket) - command.Stdin = bytes.NewReader(input) - stdout := &remoteValueOutput{limit: 8 << 20} - stderr := &remoteValueOutput{limit: 16 << 10} - failureReason := "" defer func() { if resultErr != nil && failureReason == "" { - failureReason = "collector-failed" + failureReason = nativeRemoteValueFailure(ctx, resultErr) } apiCalls, coreRequests, searchRequests, graphQLRequests, cacheHits, preflights, points, unverified, receivedBytes := api.costUsage() @@ -231,128 +188,72 @@ func (p Projector) collectRemoteValue( CoreRequests: coreRequests, SearchRequests: searchRequests, GraphQLRequests: graphQLRequests, WallMS: time.Since(started).Milliseconds(), PreflightRequests: preflights, GraphQLPointsObserved: points, - GraphQLRequestsUnverified: unverified, OutputBytes: stdout.bytes, TransportBytes: receivedBytes, + GraphQLRequestsUnverified: unverified, OutputBytes: outputBytes, TransportBytes: receivedBytes, Succeeded: resultErr == nil, FailureReason: failureReason, }); costErr != nil { resultErr = errors.Join(resultErr, costErr) } + if p.nativeValueScope != nil { + state, reason := "available", "captured" + if resultErr != nil { + state, reason = "failed", nativeRemoteValueFailure(ctx, resultErr) + } else if outputBytes == 4 { + reason = "not-due" + } + if healthErr := p.RecordNativeValueAttempt(costCtx, repository, entry.Workflow, + started, state, reason); healthErr != nil { + resultErr = errors.Join(resultErr, healthErr) + } + } }() - command.Stdout, command.Stderr = stdout, stderr - remoteValueProcessGroup(command) - command.WaitDelay = 5 * time.Second - if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { + api.token, err = p.Tokens.InstallationToken(ctx, installation) + if err != nil { return err } - runErr := command.Run() - stopRemoteValueProcessGroup(command) - if runErr != nil || stdout.exceeded || stderr.exceeded { - switch { - case errors.Is(ctx.Err(), context.DeadlineExceeded): - failureReason = "timeout" - case stdout.exceeded || stderr.exceeded: - failureReason = "output-budget" - default: - failureReason = remoteValueFailureReason(stderr.buffer.Bytes()) - } - return remoteValueFailureError(failureReason) + observedAt := time.Now().UTC().Truncate(time.Millisecond) + packet, proof, skipped, err := p.acquireNativeDailyFileDiet(ctx, entry, api, previous, restored, observedAt) + if err != nil { + return err } - if bytes.Equal(bytes.TrimSpace(stdout.buffer.Bytes()), []byte("null")) { + if skipped { + outputBytes = 4 return nil } - var envelope struct { + output, err := json.Marshal(struct { SchemaVersion int `json:"schema_version"` Kind string `json:"kind"` Packet model.Row `json:"operational_value_evidence"` - } - decoder := json.NewDecoder(bytes.NewReader(stdout.buffer.Bytes())) - decoder.UseNumber() - if err := decoder.Decode(&envelope); err != nil || envelope.SchemaVersion != 2 || - envelope.Kind != "operational_value_evidence" || envelope.Packet == nil { - return errors.New("remote Value collector returned an invalid evidence envelope") - } - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - return errors.New("remote Value collector returned trailing output") - } - if !strings.EqualFold(fmt.Sprint(envelope.Packet["repository"]), repository) || - envelope.Packet["workflow"] != entry.Workflow { - return errors.New("remote Value evidence exceeds the reviewed repository or workflow") - } - proof, err := readRemoteValueProof(filepath.Join(work, "latest")) + }{2, "operational_value_evidence", packet}) if err != nil { - return err + return fmt.Errorf("encode File Diet evidence: %w", err) } - if p.nativeValueScope != nil { - if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { - return err - } - _, err := p.Data.Acquisition().PublishValue(ctx, *p.nativeValueScope, previous, envelope.Packet, proof, time.Now().UTC()) - return err + outputBytes = int64(len(output)) + if outputBytes > 8<<20 { + return errors.New("file Diet output exceeds its evidence byte budget") } - return p.Enrollment.withMutationLock(ctx, func(locked context.Context) error { - if err := p.remoteValueRegistrationCurrent(locked, repository, installation, entry, policyDigest); err != nil { - return err - } - current, err := p.Data.ReadOperationalValue(locked, repository, entry.Workflow) - if errors.Is(err, postgresx.ErrValueNotFound) { - current = nil - } else if err != nil { - return err - } - if !remoteValuePacketCurrent(previous, current) { - return fmt.Errorf("%w: canonical remote Value proof changed before publication", ErrRemoteValueDeferred) - } - if proof == nil { - _, err := p.Data.PublishOperationalValue(locked, envelope.Packet) - return err - } - _, err = p.Data.PublishOperationalValueWithProof( - locked, envelope.Packet, repository, entry.Workflow, proof, time.Now().UTC()) + if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { return err - }) -} - -func remoteValueFailureReason(stderr []byte) string { - for _, line := range bytes.Split(stderr, []byte{'\n'}) { - if !bytes.HasPrefix(line, []byte(remoteValueFailurePrefix)) { - continue - } - var diagnostic struct { - ReasonCode string `json:"reasonCode"` - } - if err := json.Unmarshal(bytes.TrimPrefix(line, []byte(remoteValueFailurePrefix)), &diagnostic); err != nil { - continue - } - switch diagnostic.ReasonCode { - case "api-budget", "budget-unavailable", "timeout", "collector-failed": - return diagnostic.ReasonCode - } } - return "collector-failed" + _, err = p.Data.Acquisition().PublishValue(ctx, *p.nativeValueScope, previous, packet, proof, time.Now().UTC()) + return err } -func remoteValueFailureError(reason string) error { - switch reason { - case "api-budget": - return errors.New("remote Value collector exhausted its reviewed API request grant; previous evidence retained") - case "budget-unavailable": - return errors.New("remote Value collector API budget state was unavailable; previous evidence retained") - case "timeout": - return errors.New("remote Value collector command timed out; previous evidence retained") - case "output-budget": - return errors.New("remote Value collector exceeded its output budget; previous evidence retained") +func nativeRemoteValueFailure(ctx context.Context, err error) string { + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + return "timeout" + case strings.Contains(err.Error(), "request limit exhausted"), + strings.Contains(err.Error(), "budget exhausted"): + return "api-budget" + case strings.Contains(err.Error(), "quota unavailable"), + strings.Contains(err.Error(), "shared remote Value quota"): + return "batch-budget" + case errors.Is(err, postgresx.ErrValueInvalid), errors.Is(err, postgresx.ErrValueConflict), + strings.Contains(err.Error(), "evidence packet"), strings.Contains(err.Error(), "File Diet proof"): + return "invalid-evidence" default: - return errors.New("remote Value collector failed; previous evidence retained") - } -} - -func remoteValuePacketCurrent(start, current model.Row) bool { - if (start == nil) != (current == nil) { - return false + return "collector-failed" } - before, beforeErr := json.Marshal(start) - after, afterErr := json.Marshal(current) - return beforeErr == nil && afterErr == nil && bytes.Equal(before, after) } func (p Projector) remoteValueRegistrationCurrent( @@ -384,22 +285,6 @@ func (p Projector) remoteValueRegistrationCurrent( return fmt.Errorf("%w: reviewed registration or enrollment changed", ErrRemoteValueDeferred) } -type remoteValueOutput struct { - buffer bytes.Buffer - limit int - bytes int64 - exceeded bool -} - -func (w *remoteValueOutput) Write(content []byte) (int, error) { - w.bytes += int64(len(content)) - if w.bytes > int64(w.limit) { - w.exceeded = true - return 0, errors.New("remote Value output exceeds its byte budget") - } - return w.buffer.Write(content) -} - func (p Projector) remoteValueWorkspace(ctx context.Context, repository, workflow string) (string, error) { path, err := p.remoteValueWorkspacePath(repository, workflow) if err != nil { @@ -506,42 +391,6 @@ func (p Projector) ForgetRemoteValueWorkspace(ctx context.Context, repository st return nil } -func restoreRemoteValueProof( - ctx context.Context, store *postgresx.Store, repository, workflow, destination string, -) error { - files, err := store.ReadValueProof(ctx, repository, workflow) - if err != nil { - return err - } - return writeRemoteValueProof(files, destination) -} - -func writeRemoteValueProof(files []postgresx.ValueProofFile, destination string) error { - if len(files) == 0 { - return nil - } - if err := os.Mkdir(destination, 0o700); err != nil { - return err - } - root, err := os.OpenRoot(destination) - if err != nil { - return err - } - defer func() { _ = root.Close() }() - for _, file := range files { - parent := filepath.Dir(file.Path) - if parent != "." { - if err := root.MkdirAll(parent, 0o700); err != nil { - return err - } - } - if err := root.WriteFile(file.Path, file.Content, 0o600); err != nil { - return err - } - } - return nil -} - func readRemoteValueProof(directory string) ([]postgresx.ValueProofFile, error) { info, err := os.Lstat(directory) if errors.Is(err, os.ErrNotExist) { diff --git a/server/internal/collect/projector_remote_value_acquire.go b/server/internal/collect/projector_remote_value_acquire.go new file mode 100644 index 0000000000..c22c221277 --- /dev/null +++ b/server/internal/collect/projector_remote_value_acquire.go @@ -0,0 +1,206 @@ +package collect + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "strings" + "time" + + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/postgresx" +) + +func fileDietCadenceSatisfied(previous model.Row, at time.Time) (bool, error) { + if previous == nil { + return false, nil + } + facts, ok := previous["facts"].([]any) + if !ok { + return false, errors.New("retained File Diet evidence is malformed") + } + adoption := fileDietTime("2025-11-15T13:36:21Z") + days := int(at.Sub(adoption) / (24 * time.Hour)) + due := adoption.Add(time.Duration(days) * 24 * time.Hour) + var cadence bool + var complete bool + for _, row := range facts { + fact, ok := row.(map[string]any) + if !ok { + return false, errors.New("retained File Diet fact is malformed") + } + switch fact["fact-kind"] { + case "contract": + cadence = fmt.Sprint(fact["cadence-days"]) == "1" + case "coverage": + complete = complete || fact["coverage-status"] == "complete" && + !fileDietTime(fmt.Sprint(fact["at"])).After(due.Add(-24*time.Hour)) && + !fileDietTime(fmt.Sprint(fact["end-at"])).Before(due) + } + } + if !cadence { + return false, errors.New("frozen File Diet cadence changed") + } + return !fileDietTime(fmt.Sprint(previous["windowEnd"])).Before(due) && complete, nil +} + +func fileDietHistoryPreserved(previous, next model.Row) error { + if previous == nil { + return nil + } + if previous["id"] != next["id"] || previous["contractDigest"] != next["contractDigest"] || + !fileDietTime(fmt.Sprint(next["observedAt"])).After(fileDietTime(fmt.Sprint(previous["observedAt"]))) || + fileDietTime(fmt.Sprint(next["windowStart"])).After(fileDietTime(fmt.Sprint(previous["windowStart"]))) || + fileDietTime(fmt.Sprint(next["windowEnd"])).Before(fileDietTime(fmt.Sprint(previous["windowEnd"]))) { + return errors.New("file Diet frozen contract, capture or history conflict") + } + newFacts := map[string]map[string]any{} + for _, item := range next["facts"].([]any) { + row := item.(map[string]any) + newFacts[fmt.Sprint(row["fact-kind"], ":", row["parent-id"], ":", row["fact-id"])] = row + } + for _, item := range previous["facts"].([]any) { + old := item.(map[string]any) + kind := fmt.Sprint(old["fact-kind"]) + if kind != "contract" && kind != "implementation" && + kind != "snapshot" && kind != "snapshot-item" && kind != "output" && + !strings.HasPrefix(kind, "implementation-") { + continue + } + value := newFacts[fmt.Sprint(old["fact-kind"], ":", old["parent-id"], ":", old["fact-id"])] + if value == nil { + return errors.New("accepted File Diet evidence history was removed") + } + left, right := maps.Clone(old), maps.Clone(value) + if kind == "contract" { + for _, field := range []string{"outputs-available", "repository-state-available", "checks-available"} { + if old[field] == true && value[field] != true { + return errors.New("file Diet evidence availability was revoked") + } + delete(left, field) + delete(right, field) + } + delete(left, "output-coverage") + delete(right, "output-coverage") + } + if kind == "output" || kind == "implementation" { + if old["origin"] != value["origin"] && + (old["origin"] != "corroborated" || value["origin"] != "receipt-verified") { + return errors.New("accepted File Diet origin changed") + } + delete(left, "origin") + delete(right, "origin") + } + if kind == "output" { + delete(left, "state") + delete(right, "state") + } + a, _ := json.Marshal(left) + b, _ := json.Marshal(right) + if !bytes.Equal(a, b) { + return errors.New("immutable accepted File Diet evidence changed") + } + } + return nil +} + +func fileDietDeliveriesPreserved(before, after fileDietLedger) error { + if before.Metric != after.Metric { + return errors.New("frozen File Diet measurement changed") + } + byID := map[string]fileDietEvent{} + for _, event := range after.Events { + byID[event.ID] = event + } + for _, old := range before.Events { + next, ok := byID[old.ID] + if !ok || old.At != next.At || old.Value != next.Value || old.Reference != next.Reference { + return errors.New("preserved File Diet accepted delivery changed") + } + } + return nil +} + +func (p Projector) acquireNativeDailyFileDiet( + ctx context.Context, entry remoteValueEntry, api *remoteValueAPI, + previous model.Row, restored []postgresx.ValueProofFile, observedAt time.Time, +) (model.Row, []postgresx.ValueProofFile, bool, error) { + if err := reviewedDailyFileDiet(entry); err != nil { + return nil, nil, false, err + } + satisfied, err := fileDietCadenceSatisfied(previous, observedAt) + if err != nil || satisfied { + return nil, nil, satisfied, err + } + var prior *fileDietArchive + if len(restored) != 0 { + proof, err := fileDietRestore(restored) + if err != nil { + return nil, nil, false, err + } + prior = &proof + } else { + seed, err := fileDietLoadStartingProof(ctx, p.CatalogRoot, nil) + if err != nil { + return nil, nil, false, err + } + prior = &seed + } + if err := p.nativeValueCurrent(ctx); err != nil { + return nil, nil, false, err + } + reader := &nativeFileDietReader{api: api} + until := observedAt.UTC().Truncate(time.Second).Format(time.RFC3339) + current, err := fileDietAcquireHistory(ctx, reader.read, until, &prior.proof) + if err != nil { + return nil, nil, false, err + } + if current.Adoption != prior.proof.Adoption { + return nil, nil, false, errors.New("file Diet frozen adoption changed") + } + if err := fileDietCollectReceipts(ctx, reader.read, ¤t, prior); err != nil { + return nil, nil, false, err + } + workspace, err := fileDietGitWorkspace(p.CatalogRoot) + if err != nil { + return nil, nil, false, err + } + defer func() { _ = os.RemoveAll(workspace) }() + measured, err := fileDietAcquireInventories(ctx, workspace, ¤t, prior) + if err != nil { + return nil, nil, false, err + } + oldMeasured := fileDietMeasurements{inventories: prior.inventories} + oldLedger, err := fileDietReplay(prior.proof, oldMeasured) + if err != nil { + return nil, nil, false, err + } + files, err := fileDietBuildProofFiles(¤t, measured, prior) + if err != nil { + return nil, nil, false, err + } + ledger, err := fileDietReplay(current, measured) + if err != nil { + return nil, nil, false, err + } + if err := fileDietDeliveriesPreserved(oldLedger, ledger); err != nil { + return nil, nil, false, err + } + packet, err := fileDietExport(ledger) + if err != nil { + return nil, nil, false, err + } + if !strings.EqualFold(fmt.Sprint(packet["repository"]), entry.Repository) || + packet["workflow"] != entry.Workflow || fileDietTime(fmt.Sprint(packet["observedAt"])).After(time.Now()) || + fileDietTime(fmt.Sprint(packet["windowEnd"])).After(observedAt) { + return nil, nil, false, errors.New("file Diet evidence exceeds reviewed identity or capture cutoff") + } + if err := fileDietHistoryPreserved(previous, packet); err != nil { + return nil, nil, false, err + } + return packet, files, false, nil +} diff --git a/server/internal/collect/projector_remote_value_api.go b/server/internal/collect/projector_remote_value_api.go index 8918156390..bb99eab6c7 100644 --- a/server/internal/collect/projector_remote_value_api.go +++ b/server/internal/collect/projector_remote_value_api.go @@ -7,12 +7,10 @@ import ( "errors" "fmt" "io" - "net" "net/http" "net/url" "path" "regexp" - "runtime" "strconv" "strings" "sync" @@ -87,9 +85,8 @@ func remoteValueExpectedGraphQuery(repository string, numbers []int) string { strings.Join(parts, "\n") + " } }" } -// The collector never receives an installation credential. Its confined gh -// executable speaks to this private Unix socket; the host reserves each API -// request against the shared installation buckets before forwarding it. +// The native collector calls this provider directly. The host reserves each +// API request against the shared installation buckets before forwarding it. type remoteValueAPI struct { client *http.Client quota *githubquota.Service @@ -693,62 +690,6 @@ func (a *remoteValueAPI) ServeHTTP(writer http.ResponseWriter, request *http.Req _, _ = writer.Write(content) // #nosec G705 -- only JSON or ZIP with nosniff, never upstream-selected HTML. } -func (a *remoteValueAPI) listen(ctx context.Context) (string, func(), error) { - if runtime.GOOS != "linux" { - return "", nil, errors.New("remote Value authenticated API confinement requires Linux abstract Unix sockets") - } - nonce, err := operationToken() - if err != nil { - return "", nil, err - } - name := "cao-remote-value-" + nonce - listener, err := (&net.ListenConfig{}).Listen(ctx, "unix", "@"+name) - if err != nil { - return "", nil, err - } - server := &http.Server{Handler: a, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 65 * time.Second, WriteTimeout: 70 * time.Second, - BaseContext: func(net.Listener) context.Context { return ctx }} - go func() { _ = server.Serve(listener) }() - return "@" + name, func() { - _ = server.Close() - _ = listener.Close() - }, nil -} - -// The CLI supports one REST GET or the reviewed bounded GraphQL query. -// Other flags, pagination and mutation paths cannot escape host admission. -const remoteValueGH = `#!/usr/bin/env node -const http = require('node:http'); -const args = process.argv.slice(2); -const graph = args.length === 4 && args[0] === 'api' && args[1] === 'graphql' - && args[2] === '-f' && args[3].startsWith('query='); -const rest = args.length === 2 && args[0] === 'api' && /^(?:repos\/|search\/)/.test(args[1]); -if (!graph && !rest) { - process.stderr.write('remote Value gh API command unsupported\n'); - process.exit(1); -} -const body = graph ? JSON.stringify({query: args[3].slice('query='.length)}) : ''; -const requirement = process.env.CAO_VALUE_ACQUISITION_REQUIREMENT || ''; -const headers = graph ? {'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body)} : {}; -if (/^[a-f0-9]{64}$/.test(requirement)) headers['X-CAO-Value-Acquisition-Requirement'] = requirement; -const request = http.request({socketPath: process.env.CAO_REMOTE_VALUE_SOCKET.replace(/^@/, '\0'), - path: graph ? '/graphql' : '/api?endpoint=' + encodeURIComponent(args[1]), - method: graph ? 'POST' : 'GET', timeout: 65000, - headers}, response => { - if (response.statusCode < 200 || response.statusCode >= 300) { - process.stderr.write('remote Value API failed (HTTP ' + response.statusCode + ')\n'); - response.resume(); - process.exitCode = 1; - return; - } - response.pipe(process.stdout); -}); -request.on('error', () => { process.stderr.write('remote Value API unavailable\n'); process.exitCode = 1; }); -request.on('timeout', () => request.destroy()); -request.end(body); -` - func remoteValueAPIBase(raw string) (*url.URL, error) { if raw == "" { raw = "https://api.github.com" diff --git a/server/internal/collect/projector_remote_value_dependabot.go b/server/internal/collect/projector_remote_value_dependabot.go new file mode 100644 index 0000000000..4193b6c701 --- /dev/null +++ b/server/internal/collect/projector_remote_value_dependabot.go @@ -0,0 +1,629 @@ +package collect + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/postgresx" +) + +const ( + dependabotValueRepository = "github/gh-aw" + dependabotValueWorkflow = "dependabot-update-planner" + dependabotValueCollector = "dependabot/operational-value/dependabot-update-planner.mjs" + dependabotValueAdoption = "2026-09-17T08:48:00Z" + dependabotValueVersion = "dependabot-lifecycle-v1" +) + +var dependabotValueRequirements = []remoteValueRequirement{ + {"github", "core", "github.dependabot-alert-lifecycle.v1", dependabotValueRepository, "invocation"}, + {"github", "search", "github.dependabot-pull-request-lifecycle.v1", dependabotValueRepository, "invocation"}, +} + +type dependabotValueMetric struct { + id, name, unit, formula string +} + +var dependabotValueMetrics = []dependabotValueMetric{ + {"open-security-alert-count", "Open security alerts", "alerts", "Dependabot security alerts open at the cutoff"}, + {"open-high-critical-alert-count", "Open critical/high security alerts", "alerts", "Dependabot security alerts with critical or high severity open at the cutoff"}, + {"open-dependabot-pr-count", "Open Dependabot pull requests", "pull-requests", "Dependabot-authored dependency pull requests open at the cutoff"}, +} + +func reviewedDependabotValue(entry remoteValueEntry) error { + if entry.Repository != dependabotValueRepository || entry.Workflow != dependabotValueWorkflow || + entry.Collector != dependabotValueCollector || entry.APIRequests != 111 || + entry.Acquisition == nil || len(entry.Acquisition.Requirements) != len(dependabotValueRequirements) { + return errors.New("unsupported reviewed Dependabot Value registration") + } + required := map[string]bool{} + for _, requirement := range dependabotValueRequirements { + required[remoteValueRequirementKey(requirement)] = true + } + for _, requirement := range entry.Acquisition.Requirements { + key := remoteValueRequirementKey(requirement) + if !required[key] { + return errors.New("unreviewed or duplicate Dependabot Value acquisition requirement") + } + delete(required, key) + } + return nil +} + +func reviewedRemoteValue(entry remoteValueEntry) error { + if entry.Workflow == dependabotValueWorkflow || entry.Collector == dependabotValueCollector { + return reviewedDependabotValue(entry) + } + return reviewedDailyFileDiet(entry) +} + +func dependabotValueHash(value any) string { + content, _ := json.Marshal(value) + digest := sha256.Sum256(content) + return hex.EncodeToString(digest[:]) +} + +func dependabotValueCutoff(at time.Time) time.Time { + adoption, _ := time.Parse(time.RFC3339, dependabotValueAdoption) + period := 7 * 24 * time.Hour + if at.Before(adoption.Add(period)) { + return time.Time{} + } + elapsedNanos := at.Sub(adoption).Nanoseconds() + periodNanos := period.Nanoseconds() + return adoption.Add(time.Duration(elapsedNanos / periodNanos * periodNanos)) +} + +func dependabotValueTime(value string, observed time.Time) (time.Time, error) { + if value == "" || !strings.HasSuffix(value, "Z") { + return time.Time{}, errors.New("dependabot lifecycle timestamp missing or not UTC") + } + at, err := time.Parse(time.RFC3339Nano, value) + if err != nil || at.After(observed) { + return time.Time{}, errors.New("dependabot lifecycle timestamp malformed or in the future") + } + return at, nil +} + +type dependabotValueLifecycle struct { + CreatedAt string `json:"created_at"` + FixedAt *string `json:"fixed_at"` + DismissedAt *string `json:"dismissed_at"` + AutoDismissedAt *string `json:"auto_dismissed_at"` + ClosedAt *string `json:"closed_at"` + State string `json:"state"` + Number int `json:"number"` + SecurityAdvisory *struct { + Severity string `json:"severity"` + } `json:"security_advisory"` + User *struct { + Login string `json:"login"` + } `json:"user"` + PullRequest json.RawMessage `json:"pull_request"` +} + +type dependabotValueEvent struct { + created, ended time.Time + severity string +} + +func dependabotValueRecord(record dependabotValueLifecycle, observed time.Time, pull bool) (dependabotValueEvent, error) { + created, err := dependabotValueTime(record.CreatedAt, observed) + if err != nil || record.Number <= 0 { + return dependabotValueEvent{}, errors.New("dependabot lifecycle record lacks an authoritative identity or creation") + } + event := dependabotValueEvent{created: created} + terminal := []*string{record.FixedAt, record.DismissedAt, record.AutoDismissedAt} + switch { + case pull: + if record.User == nil || record.User.Login != "dependabot[bot]" || + len(record.PullRequest) == 0 || bytes.Equal(record.PullRequest, []byte("null")) || + (record.State != "open" && record.State != "closed") { + return event, errors.New("search result is not a Dependabot-authored pull request") + } + terminal = []*string{record.ClosedAt} + case record.SecurityAdvisory == nil || + (record.SecurityAdvisory.Severity != "critical" && record.SecurityAdvisory.Severity != "high" && + record.SecurityAdvisory.Severity != "medium" && record.SecurityAdvisory.Severity != "low") || + (record.State != "open" && record.State != "fixed" && + record.State != "dismissed" && record.State != "auto_dismissed"): + return event, errors.New("dependabot alert lacks a valid severity or state") + default: + event.severity = record.SecurityAdvisory.Severity + } + for _, value := range terminal { + if value == nil { + continue + } + ended, err := dependabotValueTime(*value, observed) + if err != nil || ended.Before(created) { + return event, errors.New("dependabot lifecycle terminal timestamp is invalid") + } + if event.ended.IsZero() || ended.Before(event.ended) { + event.ended = ended + } + } + if (record.State == "open") != event.ended.IsZero() { + return event, errors.New("dependabot lifecycle state contradicts its terminal timestamp") + } + return event, nil +} + +func dependabotValueNext(headers http.Header, base *url.URL, endpoint string) (string, error) { + link := headers.Get("Link") + if link == "" { + return "", nil + } + current, _ := url.Parse(endpoint) + next := "" + parts := []string{} + start, inURL := 0, false + for index, char := range link { + switch char { + case '<': + inURL = true + case '>': + inURL = false + case ',': + if !inURL { + parts = append(parts, link[start:index]) + start = index + 1 + } + } + } + parts = append(parts, link[start:]) + for _, part := range parts { + section := strings.SplitN(strings.TrimSpace(part), ";", 2) + if len(section) != 2 { + return "", errors.New("malformed Dependabot pagination link") + } + if !strings.Contains(section[1], `rel="next"`) { + continue + } + target, err := url.Parse(strings.Trim(section[0], "<> ")) + if err != nil || next != "" || target.User != nil || target.Fragment != "" || + target.Scheme != base.Scheme || target.Host != base.Host || + target.Path != strings.TrimSuffix(base.Path, "/")+"/"+current.Path { + return "", errors.New("dependabot pagination link escaped reviewed endpoint") + } + query, err := url.ParseQuery(target.RawQuery) + if err != nil || len(query["after"]) != 1 || query.Get("after") == "" || + query.Get("after") == current.Query().Get("after") { + return "", errors.New("dependabot pagination link lacks a new cursor") + } + expected := current.Query() + expected.Del("after") + query.Del("after") + if query.Encode() != expected.Encode() { + return "", errors.New("dependabot pagination link changed reviewed selection") + } + next = current.Path + "?" + target.RawQuery + } + return next, nil +} + +func dependabotValueReadAlerts(ctx context.Context, api *remoteValueAPI, observed time.Time) ([]dependabotValueEvent, error) { + const path = "repos/github/gh-aw/dependabot/alerts" + events := []dependabotValueEvent{} + seen := map[int]bool{} + endpoint := path + "?state=auto_dismissed,dismissed,fixed,open&per_page=100" + for page := 1; page <= 100; page++ { + body, headers, status, err := api.request(ctx, endpoint) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("dependabot alerts API returned status %d", status) + } + var rows []dependabotValueLifecycle + if err := json.Unmarshal(body, &rows); err != nil || rows == nil || len(rows) > 100 { + return nil, errors.New("dependabot alert page is malformed") + } + next, err := dependabotValueNext(headers, api.base, endpoint) + if err != nil { + return nil, err + } + if next != "" && len(rows) != 100 || page == 100 && next != "" { + return nil, errors.New("dependabot alert pagination is incomplete") + } + for _, row := range rows { + if seen[row.Number] { + return nil, errors.New("duplicate Dependabot alert") + } + seen[row.Number] = true + event, err := dependabotValueRecord(row, observed, false) + if err != nil { + return nil, err + } + events = append(events, event) + } + if next == "" { + if len(rows) == 100 { + return nil, errors.New("full Dependabot alert page lacks a next cursor") + } + return events, nil + } + endpoint = next + } + return nil, errors.New("dependabot alert pagination is incomplete") +} + +func dependabotValueReadPulls(ctx context.Context, api *remoteValueAPI, observed, cutoff time.Time) ([]dependabotValueEvent, error) { + const path = "search/issues" + query := fmt.Sprintf("repo:%s is:pr author:app/dependabot created:<=%s", + dependabotValueRepository, cutoff.Format(time.RFC3339)) + events := []dependabotValueEvent{} + seen := map[int]bool{} + expected := -1 + for page := 1; page <= 10; page++ { + endpoint := fmt.Sprintf("%s?q=%s&per_page=100&page=%d", path, url.QueryEscape(query), page) + body, _, status, err := api.request(ctx, endpoint) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("dependabot PR search returned status %d", status) + } + var response struct { + TotalCount *int `json:"total_count"` + IncompleteResults *bool `json:"incomplete_results"` + Items []dependabotValueLifecycle `json:"items"` + } + if err := json.Unmarshal(body, &response); err != nil || response.TotalCount == nil || + *response.TotalCount < 0 || *response.TotalCount > 1000 || + response.IncompleteResults == nil || *response.IncompleteResults || + response.Items == nil || len(response.Items) > 100 { + return nil, errors.New("dependabot PR search is incomplete or malformed") + } + if expected < 0 { + expected = *response.TotalCount + } else if expected != *response.TotalCount { + return nil, errors.New("dependabot PR search total changed during pagination") + } + if len(response.Items) != min(100, expected-len(events)) { + return nil, errors.New("dependabot PR search page was truncated") + } + for _, row := range response.Items { + if seen[row.Number] { + return nil, errors.New("duplicate Dependabot PR search result") + } + seen[row.Number] = true + event, err := dependabotValueRecord(row, observed, true) + if err != nil || event.created.After(cutoff) { + return nil, errors.New("dependabot PR search returned an invalid author, lifecycle or cutoff") + } + events = append(events, event) + } + if len(events) == expected { + return events, nil + } + } + return nil, errors.New("dependabot PR search exceeded ten pages") +} + +func dependabotValueCounts(alerts, pulls []dependabotValueEvent, cutoff time.Time) [3]int { + var counts [3]int + for _, alert := range alerts { + if !alert.created.After(cutoff) && (alert.ended.IsZero() || alert.ended.After(cutoff)) { + counts[0]++ + if alert.severity == "high" || alert.severity == "critical" { + counts[1]++ + } + } + } + for _, pull := range pulls { + if !pull.created.After(cutoff) && (pull.ended.IsZero() || pull.ended.After(cutoff)) { + counts[2]++ + } + } + return counts +} + +func dependabotValueContract(metric dependabotValueMetric) (string, map[string]any) { + fact := map[string]any{ + "fact-kind": "contract", "fact-id": "contract", + "goal": "Reduce the open Dependabot security-alert backlog in github/gh-aw.", + "acceptance": "Complete alert and Dependabot-authored pull-request lifecycle evidence at a seven-day cutoff.", + "workflow-name": "Dependabot / Update Planner", + "metric-id": metric.id, "metric-name": metric.name, "metric-unit": metric.unit, + "metric-version": dependabotValueVersion, "metric-formula": metric.formula, + "metric-direction": "decrease", "measurement-basis": "repository-outcome", + "result-label": "open Dependabot lifecycle count", + "outputs-available": false, "repository-state-available": true, "checks-available": false, + "snapshot-id": metric.id, "snapshot-name": metric.name, "snapshot-unit": metric.unit, + "snapshot-version": dependabotValueVersion, "snapshot-formula": metric.formula, + "snapshot-direction": "decrease", "cadence-days": 7, "maturation-days": 0, + } + return dependabotValueHash(fact), fact +} + +func dependabotValuePacket(metric dependabotValueMetric, previous model.Row, cutoff, observed time.Time, count int) (model.Row, error) { + adoption, _ := time.Parse(time.RFC3339, dependabotValueAdoption) + digest, contract := dependabotValueContract(metric) + start := adoption + facts := []any{contract} + if previous != nil { + if previous["contractDigest"] != digest || + previous["id"] != "operational-value-evidence:"+url.QueryEscape(dependabotValueRepository+":"+dependabotValueWorkflow+":"+metric.id+":"+dependabotValueVersion) || + previous["windowStart"] != dependabotValueAdoption { + return nil, errors.New("retained Dependabot contract or fixed adoption was changed") + } + oldFacts, ok := previous["facts"].([]any) + if !ok { + return nil, errors.New("retained Dependabot facts are malformed") + } + for _, old := range oldFacts { + fact, ok := old.(map[string]any) + if !ok { + return nil, errors.New("retained Dependabot fact is malformed") + } + if fact["fact-kind"] != "contract" { + facts = append(facts, fact) + } + } + } + at := cutoff.UTC().Format(time.RFC3339) + snapshotID := "cutoff:" + at + for _, fact := range facts[1:] { + row, ok := fact.(map[string]any) + if !ok { + return nil, errors.New("retained Dependabot fact is malformed") + } + if row["fact-kind"] == "snapshot" && row["fact-id"] == snapshotID { + return nil, errors.New("retained Dependabot cutoff is already present") + } + } + facts = append(facts, map[string]any{ + "fact-kind": "snapshot", "fact-id": snapshotID, "at": at, + "value": count, "ref": "cutoff:" + at, "items-available": false, + }) + coverageStart := cutoff.Add(-7 * 24 * time.Hour) + if previous != nil { + previousEnd, err := time.Parse(time.RFC3339Nano, fmt.Sprint(previous["windowEnd"])) + if err != nil || !previousEnd.Before(cutoff) { + return nil, errors.New("retained Dependabot cutoff is not before new cutoff") + } + } + facts = append(facts, map[string]any{ + "fact-kind": "coverage", "fact-id": snapshotID, + "at": coverageStart.Format(time.RFC3339), "end-at": at, "coverage-status": "complete", + }) + packet := model.Row{ + "schemaVersion": 1, + "id": "operational-value-evidence:" + url.QueryEscape(dependabotValueRepository+":"+dependabotValueWorkflow+":"+metric.id+":"+dependabotValueVersion), + "repository": dependabotValueRepository, "workflow": dependabotValueWorkflow, + "sourcePath": ".github/workflows/" + dependabotValueWorkflow + ".md", + "observedAt": observed.UTC().Format(time.RFC3339Nano), + "windowStart": start.Format(time.RFC3339), "windowEnd": at, + "contractDigest": digest, "facts": facts, + } + encoded, err := json.Marshal(packet) + if err != nil || len(encoded) > (8<<20)-1024 || len(facts) > 20000 { + return nil, errors.New("dependabot evidence packet exceeds its byte budget") + } + return packet, nil +} + +func (p Projector) collectDependabotValue( + ctx context.Context, repository string, installation int64, entry remoteValueEntry, + policyDigest [sha256.Size]byte, base *url.URL, cache *remoteValueResponseCache, + sharedRequirements map[string]remoteValueRequirement, +) (resultErr error) { + ctx, cancel := context.WithTimeout(ctx, 8*time.Minute) + defer cancel() + if err := reviewedDependabotValue(entry); err != nil { + return err + } + if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { + return err + } + started := time.Now().UTC() + var outputBytes int64 + transport := p.RemoteHTTP + if transport == nil { + transport = &http.Client{Timeout: 65 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + } + api := &remoteValueAPI{ + client: transport, quota: p.RemoteQuota, searchQuota: p.RemoteSearchQuota, + app: p.QuotaApp, installation: installation, repository: repository, + collector: entry.Collector, base: base, max: entry.APIRequests, cache: cache, + sharedRequirements: sharedRequirements, authorize: p.nativeValueCurrent, + } + defer func() { + calls, core, search, graphql, hits, preflights, points, unknown, received := api.costUsage() + costCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer stop() + reason := "" + if resultErr != nil { + reason = nativeRemoteValueFailure(ctx, resultErr) + } + if err := p.Data.AppendValueCollectionCost(costCtx, postgresx.ValueCollectionCost{ + At: started, Installation: installation, Repository: repository, Workflow: entry.Workflow, + Collector: entry.Collector, APIRequests: calls, CoreRequests: core, SearchRequests: search, + GraphQLRequests: graphql, CacheHits: hits, PreflightRequests: preflights, + GraphQLPointsObserved: points, GraphQLRequestsUnverified: unknown, + TransportBytes: received, OutputBytes: outputBytes, + WallMS: time.Since(started).Milliseconds(), Succeeded: resultErr == nil, FailureReason: reason, + }); err != nil { + resultErr = errors.Join(resultErr, err) + } + state := "available" + switch { + case resultErr != nil: + state, reason = "failed", nativeRemoteValueFailure(ctx, resultErr) + case outputBytes == 4: + reason = "not-due" + default: + reason = "captured" + } + if err := p.RecordNativeValueAttempt(costCtx, repository, entry.Workflow, started, state, reason); err != nil { + resultErr = errors.Join(resultErr, err) + } + }() + previous, _, err := p.Data.Acquisition().ReadValueBatch(ctx, *p.nativeValueScope, entry.Workflow) + if errors.Is(err, postgresx.ErrValueParentMissing) { + return fmt.Errorf("%w: canonical workflow parent has not been projected", ErrRemoteValueDeferred) + } + if err != nil { + return err + } + api.token, err = p.Tokens.InstallationToken(ctx, installation) + if err != nil { + return err + } + observed := time.Now().UTC().Truncate(time.Microsecond) + packets, skipped, err := acquireNativeDependabotValue(ctx, api, previous, observed) + if err != nil { + return err + } + if skipped { + outputBytes = 4 + return nil + } + for _, packet := range packets { + content, err := json.Marshal(packet) + if err != nil { + return err + } + outputBytes += int64(len(content)) + } + if outputBytes > (8<<20)-1024 { + return errors.New("dependabot evidence batch exceeds its byte budget") + } + if err := p.remoteValueRegistrationCurrent(ctx, repository, installation, entry, policyDigest); err != nil { + return err + } + _, err = p.Data.Acquisition().PublishValueBatch(ctx, *p.nativeValueScope, previous, packets, nil, time.Now().UTC()) + return err +} + +func acquireNativeDependabotValue(ctx context.Context, api *remoteValueAPI, previous []model.Row, observed time.Time) ([]model.Row, bool, error) { + cutoff := dependabotValueCutoff(observed) + if cutoff.IsZero() { + return nil, true, nil + } + if len(previous) != 0 && len(previous) != len(dependabotValueMetrics) { + return nil, false, errors.New("incomplete retained Dependabot metric batch") + } + starting := make(map[string]model.Row, len(previous)) + complete := len(previous) == len(dependabotValueMetrics) + var retainedCutoff string + for _, packet := range previous { + if packet == nil { + return nil, false, errors.New("retained Dependabot metric is missing") + } + facts, ok := packet["facts"].([]any) + if !ok { + return nil, false, errors.New("retained Dependabot facts are malformed") + } + var metricID string + var contract map[string]any + var latest, coverage bool + for _, item := range facts { + row, ok := item.(map[string]any) + if !ok { + return nil, false, errors.New("retained Dependabot fact is malformed") + } + switch row["fact-kind"] { + case "contract": + if contract != nil { + return nil, false, errors.New("retained Dependabot contract is duplicated") + } + contract = row + metricID, _ = row["metric-id"].(string) + case "snapshot": + if row["at"] == packet["windowEnd"] && + row["fact-id"] == "cutoff:"+fmt.Sprint(packet["windowEnd"]) && + row["items-available"] == false && row["value"] != nil { + value, err := strconv.Atoi(fmt.Sprint(row["value"])) + if err != nil || value < 0 { + return nil, false, errors.New("retained Dependabot snapshot is not a native count") + } + latest = true + } + case "coverage": + coverageEnd, err := time.Parse(time.RFC3339Nano, fmt.Sprint(row["end-at"])) + coverageStart, startErr := time.Parse(time.RFC3339Nano, fmt.Sprint(row["at"])) + if err != nil || startErr != nil { + return nil, false, errors.New("retained Dependabot coverage is malformed") + } + if row["end-at"] == packet["windowEnd"] && row["coverage-status"] == "complete" && + row["fact-id"] == "cutoff:"+fmt.Sprint(packet["windowEnd"]) && + coverageEnd.Sub(coverageStart) == 7*24*time.Hour { + coverage = true + } + } + } + if metricID == "" || starting[metricID] != nil { + return nil, false, errors.New("retained Dependabot metric identity is invalid") + } + var known bool + for _, metric := range dependabotValueMetrics { + if metric.id == metricID { + digest, frozen := dependabotValueContract(metric) + frozenBytes, _ := json.Marshal(frozen) + retainedBytes, _ := json.Marshal(contract) + if packet["contractDigest"] != digest || !bytes.Equal(frozenBytes, retainedBytes) || + packet["id"] != "operational-value-evidence:"+url.QueryEscape(dependabotValueRepository+":"+dependabotValueWorkflow+":"+metric.id+":"+dependabotValueVersion) || + packet["windowStart"] != dependabotValueAdoption { + return nil, false, errors.New("retained Dependabot contract changed") + } + known = true + break + } + } + if !known { + return nil, false, errors.New("retained Dependabot metric is not frozen") + } + starting[metricID] = packet + parsed, err := time.Parse(time.RFC3339Nano, fmt.Sprint(packet["windowEnd"])) + if err != nil || parsed.After(cutoff) || + retainedCutoff != "" && retainedCutoff != fmt.Sprint(packet["windowEnd"]) { + return nil, false, errors.New("retained Dependabot cutoff is invalid") + } + retainedCutoff = fmt.Sprint(packet["windowEnd"]) + complete = complete && !parsed.Before(cutoff) && latest && coverage + } + if retainedCutoff == cutoff.Format(time.RFC3339) && !complete { + return nil, false, errors.New("retained Dependabot cutoff is incomplete; immutable evidence cannot be refreshed") + } + if complete { + return nil, true, nil + } + alerts, err := dependabotValueReadAlerts(ctx, api, observed) + if err != nil { + return nil, false, err + } + pulls, err := dependabotValueReadPulls(ctx, api, observed, cutoff) + if err != nil { + return nil, false, err + } + counts := dependabotValueCounts(alerts, pulls, cutoff) + packets := make([]model.Row, 0, len(dependabotValueMetrics)) + var size int + for i, metric := range dependabotValueMetrics { + packet, err := dependabotValuePacket(metric, starting[metric.id], cutoff, observed, counts[i]) + if err != nil { + return nil, false, err + } + content, _ := json.Marshal(packet) + size += len(content) + if size > (8<<20)-1024 { + return nil, false, errors.New("dependabot metric batch exceeds its byte budget") + } + packets = append(packets, packet) + } + return packets, false, nil +} diff --git a/server/internal/collect/projector_remote_value_dependabot_test.go b/server/internal/collect/projector_remote_value_dependabot_test.go new file mode 100644 index 0000000000..916ec02772 --- /dev/null +++ b/server/internal/collect/projector_remote_value_dependabot_test.go @@ -0,0 +1,627 @@ +package collect + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/githubnext/gh-cao/server/internal/githubquota" + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/operational" + "github.com/githubnext/gh-cao/server/internal/operationalvalue" + "github.com/githubnext/gh-cao/server/internal/query" + "github.com/githubnext/gh-cao/server/internal/testutil" +) + +func TestDependabotValuePacketsReachProductionPostgresQuery(t *testing.T) { + data := testutil.Postgres(t, t.Context(), "CAO_TEST_POSTGRES_URL", "POSTGRES_URL") + writer, err := data.BeginIngestion(t.Context()) + if err != nil { + t.Fatal(err) + } + defer writer.Abort(t.Context()) + path := ".github/workflows/" + dependabotValueWorkflow + ".md" + repositoryID := "repository:" + model.EncodeCoordinate(dependabotValueRepository) + workflowID := "workflow:" + model.EncodeCoordinate(dependabotValueRepository+":"+path) + if err := writer.Append(t.Context(), "$repositories", model.Row{ + "id": repositoryID, "owner": "github", "name": "gh-aw", + }); err != nil { + t.Fatal(err) + } + if err := writer.Append(t.Context(), "$workflows", model.Row{ + "id": workflowID, "repositoryId": repositoryID, "path": path, + }); err != nil { + t.Fatal(err) + } + if _, err := writer.Publish(t.Context(), "dependabot-parent"); err != nil { + t.Fatal(err) + } + authority := data.NativeEnrollment() + if err := authority.WithMutation(t.Context(), func(ctx context.Context) error { + return authority.Enroll(ctx, 9, []string{dependabotValueRepository}) + }); err != nil { + t.Fatal(err) + } + scope, err := authority.CurrentScope(t.Context(), dependabotValueRepository) + if err != nil { + t.Fatal(err) + } + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/repos/github/gh-aw/dependabot/alerts": + _, _ = fmt.Fprint(writer, `[]`) + case "/search/issues": + _, _ = fmt.Fprint(writer, `{"total_count":0,"incomplete_results":false,"items":[]}`) + default: + t.Errorf("unreviewed endpoint: %s", request.URL.Path) + } + }) + observed := time.Now().UTC().Truncate(time.Microsecond) + packets, skipped, err := acquireNativeDependabotValue(t.Context(), api, nil, observed) + if err != nil || skipped || len(packets) != 3 || *calls != 2 { + t.Fatalf("complete empty lifecycles were not accepted: packets=%d skipped=%v calls=%d err=%v", + len(packets), skipped, *calls, err) + } + if _, err := data.Acquisition().PublishValueBatch(t.Context(), scope, nil, packets, nil, time.Now()); err != nil { + t.Fatalf("native Dependabot packets rejected by production canonical writer: %v", err) + } + restored, _, err := data.Acquisition().ReadValueBatch(t.Context(), scope, dependabotValueWorkflow) + if err != nil || len(restored) != 3 { + t.Fatalf("native Dependabot packet batch not retained: %d %v", len(restored), err) + } + route := dependabotValueRepository + ":" + path + defs := []query.Definition{ + {Name: "operational-value-evidence", From: "$operationalValueEvidence"}, + {Name: "dependabot-cutoff", From: "operational-value-evidence", Filter: &query.Filter{ + Predicates: []query.Predicate{{Field: "workflow-route", Equals: route}, + {Field: "fact-id", In: []any{"contract", "cutoff:" + dependabotValueCutoff(observed).Format(time.RFC3339)}}, + }}}, + } + sources, _, err := data.ExecuteSQLPlan(t.Context(), defs, []string{"dependabot-cutoff"}) + if err != nil || len(sources["dependabot-cutoff"].Rows) != 9 { + t.Fatalf("production canonical cutoff query missing required facts: %d %v", + len(sources["dependabot-cutoff"].Rows), err) + } + for _, row := range sources["dependabot-cutoff"].Rows { + if row["report-id"] == nil || row["window-end"] == nil || row["observed-at"] == nil { + t.Fatalf("canonical Dependabot fact lost its packet identity or cutoff: %+v", row) + } + } + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + workspace := t.TempDir() + policy := filepath.Join(workspace, "cao.json") + content := `{"version":1,"gh-aw-version":"v1.0.1","control-plane":{ + "scope":{"allowed-repositories":["github/gh-aw"]},"activity":{"repositories":["github/gh-aw"]}, + "campaigns":{"optimization":{"enabled":false}, + "dependabot":{"targets":{"github/gh-aw":{"mode":"review"}}, + "workers":{"update-planner":{"workflow":"dependabot-update-planner"}}}, + "activity":{"workflow-values":[{"repository":"github/gh-aw","workflow":"dependabot-update-planner", + "collector":"dependabot/operational-value/dependabot-update-planner.mjs", + "api-requests":111,"acquisition":{"requirements":[ + {"source":"github","pool":"core","primitive":"github.dependabot-alert-lifecycle.v1","scope":"github/gh-aw","cache":"invocation"}, + {"source":"github","pool":"search","primitive":"github.dependabot-pull-request-lifecycle.v1","scope":"github/gh-aw","cache":"invocation"}]}}]}}}}` + if err := os.WriteFile(policy, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(workspace, "snapshot") + if err := os.Mkdir(snapshot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { + t.Fatal(err) + } + config := operationalvalue.Config{Root: root, Repositories: []string{dependabotValueRepository}, + ObservedAt: observed, NodeBinary: "/nonexistent-node"} + if err := operationalvalue.WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + output := filepath.Join(workspace, "values.jsonl") + if err := operationalvalue.ComputeHostedSnapshot(t.Context(), config, policy, snapshot, output, + func(ctx context.Context, defs []query.Definition, requested []string) (map[string]model.Source, error) { + rows, _, err := data.ExecuteSQLPlan(ctx, defs, requested) + return rows, err + }); err != nil { + t.Fatalf("native local evaluation of production Dependabot facts failed: %v", err) + } + values, err := operationalvalue.ReadMeasurements(output) + if err != nil || len(values) != 3 { + t.Fatalf("complete empty lifecycles did not produce three numeric measurements: %+v %v", values, err) + } + for _, row := range values { + if row["value"] != json.Number("0") || row["coverage-status"] != "complete" || + row["evidence-digest"] == nil { + t.Fatalf("empty complete evidence did not produce an honest zero: %+v", row) + } + } +} + +func testDependabotAPI(t *testing.T, handler http.HandlerFunc) (*remoteValueAPI, *int) { + t.Helper() + backend := volatileBackfill(t).Enrollment.Metadata.(operational.GitHubQuotaStore) + core, err := githubquota.New(backend, githubquota.Options{SafetyReserve: 1}) + if err != nil { + t.Fatal(err) + } + search, err := githubquota.New(backend, githubquota.Options{SafetyReserve: 1}) + if err != nil { + t.Fatal(err) + } + reset := time.Now().Add(time.Hour).Unix() + for _, resource := range []string{"core", "search"} { + if err := core.Observe(t.Context(), githubquota.BucketID{ + App: "dependabot-test", Installation: 9, Resource: resource, + }, githubquota.Observation{Limit: 5000, Remaining: 4000, ResetAt: time.Unix(reset, 0)}); err != nil { + t.Fatal(err) + } + } + calls := new(int) + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + *calls++ + resource := "core" + if strings.HasPrefix(request.URL.Path, "/search/") { + resource = "search" + } + writer.Header().Set("X-RateLimit-Resource", resource) + writer.Header().Set("X-RateLimit-Limit", "5000") + writer.Header().Set("X-RateLimit-Remaining", strconv.Itoa(4000-*calls)) + writer.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset, 10)) + handler(writer, request) + })) + t.Cleanup(upstream.Close) + base, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + return &remoteValueAPI{client: upstream.Client(), quota: core, searchQuota: search, + app: "dependabot-test", installation: 9, repository: dependabotValueRepository, + collector: dependabotValueCollector, base: base, token: "test-token", max: 111}, calls +} + +func testDependabotEntry() remoteValueEntry { + return remoteValueEntry{ + Repository: dependabotValueRepository, Workflow: dependabotValueWorkflow, + Collector: dependabotValueCollector, APIRequests: 111, + Acquisition: &remoteValueAcquisition{Requirements: append([]remoteValueRequirement(nil), dependabotValueRequirements...)}, + } +} + +func TestDependabotValueFrozenManifestAndDispatch(t *testing.T) { + entry := testDependabotEntry() + if err := reviewedDependabotValue(entry); err != nil { + t.Fatal(err) + } + for index, want := range []string{ + "d73a2676ebb04ebf34093873dc5eb72b8e54a461c8b5f068b7b3bd100fc2507a", + "c0bcc97030ca99e4ed5bbe1b89b64ebde02bc7b4659682058e407cf155603ed2", + "774112c80afca79b272669e30a75dc5b1216191f9345e6e14970863bb617652a", + } { + if got, _ := dependabotValueContract(dependabotValueMetrics[index]); got != want { + t.Fatalf("Dependabot contract digest changed: %s vs %s", got, want) + } + } + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + code := `import { pathToFileURL } from 'node:url'; +const {definition} = await import(pathToFileURL(process.argv[1])); +process.stdout.write(JSON.stringify(definition.acquisition?.requirements));` + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, filepath.Join(root, dependabotValueCollector)) // #nosec G204 -- fixed reviewed source. + content, err := command.CombinedOutput() + if err != nil { + t.Fatalf("read frozen acquisition manifest: %v: %s", err, content) + } + var manifest []remoteValueRequirement + if err := json.Unmarshal(content, &manifest); err != nil || len(manifest) != 2 { + t.Fatalf("Actions manifest diverged: %s %v", content, err) + } + for i := range manifest { + if manifest[i] != dependabotValueRequirements[i] { + t.Fatalf("native acquisition differs from frozen JS: %+v", manifest) + } + } + for _, mutate := range []func(*remoteValueEntry){ + func(value *remoteValueEntry) { value.APIRequests = 110 }, + func(value *remoteValueEntry) { value.Repository = "octo/other" }, + func(value *remoteValueEntry) { value.Collector = ".github/value-collectors/unreviewed.mjs" }, + func(value *remoteValueEntry) { value.Acquisition = nil }, + func(value *remoteValueEntry) { + value.Acquisition = &remoteValueAcquisition{Requirements: []remoteValueRequirement{ + dependabotValueRequirements[0], dependabotValueRequirements[0], + }} + }, + func(value *remoteValueEntry) { + value.Acquisition = &remoteValueAcquisition{Requirements: []remoteValueRequirement{ + dependabotValueRequirements[0], dependabotValueRequirements[1], + }} + value.Acquisition.Requirements[1].Scope = "octo/other" + }, + } { + invalid := entry + mutate(&invalid) + if err := reviewedRemoteValue(invalid); err == nil { + t.Fatalf("unreviewed registration accepted: %+v", invalid) + } + } + valid, failures := supportedRemoteValueEntries([]remoteValueEntry{{Workflow: dependabotValueWorkflow}, entry}) + if failures == nil || len(valid) != 1 || valid[0].Workflow != dependabotValueWorkflow { + t.Fatalf("failure suppressed reviewed registration: %+v %v", valid, failures) + } +} + +func TestDependabotValueMatchesFrozenJSOracle(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + module := filepath.Join(root, dependabotValueCollector) + code := `import { pathToFileURL } from 'node:url'; +const {buildEvidence,scoreMetric,definition} = await import(pathToFileURL(process.argv[1])); +let input = ''; for await (const chunk of process.stdin) input += chunk; +const {alerts,pullRequests,cutoff} = JSON.parse(input); +const evidence = buildEvidence({alerts,pullRequests,repository:"github/gh-aw",request:{windowEnd:cutoff}}); +process.stdout.write(JSON.stringify(definition.metrics.map(({id}) => scoreMetric(id,evidence))));` + type scenario struct { + name string + alerts []map[string]any + pulls []map[string]any + cutoff string + wantMissing bool + } + cases := []scenario{ + {name: "complete-empty", alerts: []map[string]any{}, pulls: []map[string]any{}, cutoff: "2026-09-24T08:48:00Z"}, + {name: "incomplete", cutoff: "2026-09-24T08:48:00Z", wantMissing: true}, + {name: "lifecycle", cutoff: "2026-09-24T08:48:00Z", + alerts: []map[string]any{ + {"created_at": "2026-09-01T00:00:00Z", "fixed_at": "2026-09-25T00:00:00Z", "severity": "critical"}, + {"created_at": "2026-09-01T00:00:00Z", "dismissed_at": "2026-09-24T08:48:00Z", "severity": "medium"}, + {"created_at": "2026-09-20T00:00:00Z", "severity": "high"}, + {"created_at": "2026-09-25T00:00:00Z", "severity": "low"}, + }, + pulls: []map[string]any{ + {"created_at": "2026-09-18T00:00:00Z", "closed_at": "2026-09-25T00:00:00Z"}, + {"created_at": "2026-09-20T00:00:00Z", "closed_at": "2026-09-24T08:48:00Z"}, + }}, + } + for _, testcase := range cases { + t.Run(testcase.name, func(t *testing.T) { + input, _ := json.Marshal(map[string]any{"alerts": testcase.alerts, "pullRequests": testcase.pulls, "cutoff": testcase.cutoff}) + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, module) // #nosec G204 -- fixed checked-in oracle module. + command.Stdin = bytes.NewReader(input) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("frozen oracle: %v: %s", err, output) + } + var expected []*int + if err := json.Unmarshal(output, &expected); err != nil { + t.Fatal(err) + } + if testcase.wantMissing { + for _, score := range expected { + if score != nil { + t.Fatalf("incomplete oracle was numeric: %s", output) + } + } + return + } + observed, _ := time.Parse(time.RFC3339, "2026-10-01T00:00:00Z") + cutoff, _ := time.Parse(time.RFC3339, testcase.cutoff) + alerts, pulls := []dependabotValueEvent{}, []dependabotValueEvent{} + for i, item := range testcase.alerts { + item["number"], item["state"] = i+1, "open" + if item["fixed_at"] != nil { + item["state"] = "fixed" + } + if item["dismissed_at"] != nil { + item["state"] = "dismissed" + } + item["security_advisory"] = map[string]any{"severity": item["severity"]} + content, _ := json.Marshal(item) + var record dependabotValueLifecycle + if err := json.Unmarshal(content, &record); err != nil { + t.Fatal(err) + } + event, err := dependabotValueRecord(record, observed, false) + if err != nil { + t.Fatal(err) + } + alerts = append(alerts, event) + } + for i, item := range testcase.pulls { + item["number"], item["state"] = i+1, "open" + if item["closed_at"] != nil { + item["state"] = "closed" + } + item["user"] = map[string]any{"login": "dependabot[bot]"} + item["pull_request"] = map[string]any{"url": "https://api.github.com/repos/github/gh-aw/pulls/1"} + content, _ := json.Marshal(item) + var record dependabotValueLifecycle + if err := json.Unmarshal(content, &record); err != nil { + t.Fatal(err) + } + event, err := dependabotValueRecord(record, observed, true) + if err != nil { + t.Fatal(err) + } + pulls = append(pulls, event) + } + counts := dependabotValueCounts(alerts, pulls, cutoff) + for i, score := range expected { + if score == nil || counts[i] != *score { + t.Fatalf("frozen oracle score %d = %v; Go = %d", i, score, counts[i]) + } + } + }) + } +} + +func TestDependabotValuePaginationAndMalformedEvidence(t *testing.T) { + cutoff, _ := time.Parse(time.RFC3339, "2026-09-24T08:48:00Z") + observed := cutoff.Add(time.Hour) + alert := `{"number":1,"state":"open","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}` + pull := `{"number":1,"state":"open","created_at":"2026-09-20T00:00:00Z","user":{"login":"dependabot[bot]"},"pull_request":{"url":"https://api.github.com/pulls/1"}}` + for _, testcase := range []struct { + name, alerts, pulls string + wantErr bool + }{ + {"complete-empty", `[]`, `{"total_count":0,"incomplete_results":false,"items":[]}`, false}, + {"complete", `[` + alert + `]`, `{"total_count":1,"incomplete_results":false,"items":[` + pull + `]}`, false}, + {"alert-object", `{}`, `{"total_count":0,"incomplete_results":false,"items":[]}`, true}, + {"alert-future", `[{"number":1,"state":"open","created_at":"2027-01-01T00:00:00Z","security_advisory":{"severity":"high"}}]`, `{"total_count":0,"incomplete_results":false,"items":[]}`, true}, + {"alert-terminal-missing", `[{"number":1,"state":"fixed","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}]`, `{"total_count":0,"incomplete_results":false,"items":[]}`, true}, + {"search-truncated", `[]`, `{"total_count":1,"incomplete_results":false,"items":[]}`, true}, + {"search-incomplete", `[]`, `{"total_count":0,"incomplete_results":true,"items":[]}`, true}, + {"search-overflow", `[]`, `{"total_count":1001,"incomplete_results":false,"items":[]}`, true}, + {"incorrect-author", `[]`, `{"total_count":1,"incomplete_results":false,"items":[` + strings.Replace(pull, "dependabot[bot]", "someone", 1) + `]}`, true}, + {"not-a-pr", `[]`, `{"total_count":1,"incomplete_results":false,"items":[` + strings.Replace(pull, `"pull_request":{"url":"https://api.github.com/pulls/1"}`, `"pull_request":null`, 1) + `]}`, true}, + } { + t.Run(testcase.name, func(t *testing.T) { + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/repos/github/gh-aw/dependabot/alerts": + _, _ = fmt.Fprint(writer, testcase.alerts) + case "/search/issues": + if request.URL.Query().Get("q") != + "repo:github/gh-aw is:pr author:app/dependabot created:<=2026-09-24T08:48:00Z" { + t.Error("search escaped frozen repository or cutoff") + } + _, _ = fmt.Fprint(writer, testcase.pulls) + default: + t.Errorf("unreviewed request: %s", request.URL) + } + }) + _, _, err := acquireNativeDependabotValue(t.Context(), api, nil, observed) + if (err != nil) != testcase.wantErr || *calls > 2 { + t.Fatalf("acquisition error=%v, calls=%d", err, *calls) + } + }) + } +} + +func TestDependabotValueSearchPreflightUsesReviewedBudget(t *testing.T) { + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/repos/github/gh-aw/dependabot/alerts": + _, _ = fmt.Fprint(writer, `[]`) + case "/rate_limit": + _, _ = fmt.Fprintf(writer, `{"resources":{"search":{"limit":5000,"remaining":4000,"reset":%d}}}`, + time.Now().Add(time.Hour).Unix()) + case "/search/issues": + _, _ = fmt.Fprint(writer, `{"total_count":0,"incomplete_results":false,"items":[]}`) + default: + t.Errorf("unreviewed preflight endpoint: %s", request.URL.Path) + } + }) + fresh, err := githubquota.New(volatileBackfill(t).Enrollment.Metadata.(operational.GitHubQuotaStore), + githubquota.Options{SafetyReserve: 1}) + if err != nil { + t.Fatal(err) + } + api.searchQuota = fresh + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + if packets, skipped, err := acquireNativeDependabotValue(t.Context(), api, nil, observed); err != nil || skipped || len(packets) != 3 || *calls != 3 { + t.Fatalf("Search-quota preflight was omitted from the reviewed API budget: packets=%d skipped=%v calls=%d err=%v", + len(packets), skipped, *calls, err) + } +} + +func TestDependabotValueAlertCursorPagination(t *testing.T) { + alerts := make([]string, 100) + for i := range alerts { + alerts[i] = fmt.Sprintf(`{"number":%d,"state":"open","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}`, i+1) + } + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Query().Get("after") == "" { + writer.Header().Set("Link", fmt.Sprintf(`; rel="next"`, + request.Host, request.URL.Path)) + _, _ = fmt.Fprintf(writer, "[%s]", strings.Join(alerts, ",")) + } else { + if request.URL.Query().Get("after") != "cursor-two" { + t.Errorf("unexpected Dependabot pagination cursor: %s", request.URL) + } + _, _ = fmt.Fprint(writer, `[{"number":101,"state":"open","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}]`) + } + }) + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + if events, err := dependabotValueReadAlerts(t.Context(), api, observed); err != nil || len(events) != 101 || *calls != 2 { + t.Fatalf("cursor-paginated alerts were not complete: events=%d calls=%d error=%v", len(events), *calls, err) + } +} + +func TestDependabotValueRejectsUnreviewedAlertCursors(t *testing.T) { + api, _ := testDependabotAPI(t, func(http.ResponseWriter, *http.Request) {}) + endpoint := "repos/github/gh-aw/dependabot/alerts?state=auto_dismissed,dismissed,fixed,open&per_page=100" + valid := api.base.String() + "/" + endpoint + "&after=cursor-two" + for _, testcase := range []struct { + name, link string + valid bool + }{ + {"next", "<" + valid + `>; rel="next"`, true}, + {"previous and next", "<" + valid + `>; rel="prev", <` + valid + `>; rel="next"`, true}, + {"other host", `; rel="next"`, false}, + {"other repository", "<" + api.base.String() + `/repos/octo/other/dependabot/alerts?state=auto_dismissed,dismissed,fixed,open&per_page=100&after=cursor-two>; rel="next"`, false}, + {"changed state", "<" + api.base.String() + `/repos/github/gh-aw/dependabot/alerts?state=open&per_page=100&after=cursor-two>; rel="next"`, false}, + {"no cursor", "<" + api.base.String() + "/" + endpoint + `>; rel="next"`, false}, + {"duplicate next", "<" + valid + `>; rel="next", <` + valid + `>; rel="next"`, false}, + } { + t.Run(testcase.name, func(t *testing.T) { + headers := http.Header{"Link": []string{testcase.link}} + next, err := dependabotValueNext(headers, api.base, endpoint) + if (err == nil) != testcase.valid || testcase.valid && next != endpoint+"&after=cursor-two" { + t.Fatalf("cursor next=%q error=%v", next, err) + } + }) + } +} + +func TestDependabotValueFullAlertPageWithoutCursorIsIncomplete(t *testing.T) { + alerts := make([]string, 100) + for i := range alerts { + alerts[i] = fmt.Sprintf(`{"number":%d,"state":"open","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}`, i+1) + } + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(writer, "[%s]", strings.Join(alerts, ",")) + }) + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + if _, err := dependabotValueReadAlerts(t.Context(), api, observed); err == nil || *calls != 1 { + t.Fatalf("full alert page without cursor was treated as complete: calls=%d error=%v", *calls, err) + } +} + +func TestDependabotValueAlertCapDoesNotTruncate(t *testing.T) { + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + page := 1 + if after := request.URL.Query().Get("after"); after != "" { + page, _ = strconv.Atoi(after) + } + writer.Header().Set("Link", fmt.Sprintf(`; rel="next"`, + request.Host, request.URL.Path, page+1)) + items := make([]string, 100) + for i := range items { + items[i] = fmt.Sprintf(`{"number":%d,"state":"open","created_at":"2026-09-20T00:00:00Z","security_advisory":{"severity":"high"}}`, (page-1)*100+i+1) + } + _, _ = fmt.Fprintf(writer, "[%s]", strings.Join(items, ",")) + }) + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + if _, err := dependabotValueReadAlerts(t.Context(), api, observed); err == nil || *calls != 100 { + t.Fatalf("100 full alert pages were accepted or overran bound: calls=%d error=%v", *calls, err) + } +} + +func TestDependabotValueCancellationAndScope(t *testing.T) { + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + _, _ = fmt.Fprint(writer, `[]`) + }) + for _, endpoint := range []string{ + "search/issues?q=repo:other/repo+is:pr+author:app/dependabot", + "repos/other/repo/dependabot/alerts?state=open", + } { + if _, _, _, err := api.request(t.Context(), endpoint); err == nil { + t.Fatalf("out-of-scope request forwarded: %s", endpoint) + } + } + if *calls != 0 { + t.Fatal("out-of-scope search used quota") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + if _, err := dependabotValueReadAlerts(ctx, api, observed); err == nil { + t.Fatal("cancelled acquisition succeeded") + } + if *calls != 0 { + t.Fatal("cancelled acquisition reached GitHub") + } +} + +func TestDependabotValuePacketsPreserveOnlyAcquiredWindows(t *testing.T) { + observed, _ := time.Parse(time.RFC3339, "2026-09-25T00:00:00Z") + api, calls := testDependabotAPI(t, func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/search/issues" { + _, _ = fmt.Fprint(writer, `{"total_count":0,"incomplete_results":false,"items":[]}`) + } else { + _, _ = fmt.Fprint(writer, `[]`) + } + }) + first, skipped, err := acquireNativeDependabotValue(t.Context(), api, nil, observed) + if err != nil || skipped || len(first) != 3 { + t.Fatalf("first collection: %v skip=%v packets=%d", err, skipped, len(first)) + } + for i, packet := range first { + if packet["windowStart"] != dependabotValueAdoption || packet["windowEnd"] != "2026-09-24T08:48:00Z" { + t.Fatal("first cutoff or adoption changed") + } + facts := packet["facts"].([]any) + if len(facts) != 3 || facts[0].(map[string]any)["metric-id"] != dependabotValueMetrics[i].id || + facts[1].(map[string]any)["fact-id"] != "cutoff:2026-09-24T08:48:00Z" || + facts[2].(map[string]any)["at"] != dependabotValueAdoption { + t.Fatalf("incorrect native facts: %+v", facts) + } + root, _ := filepath.Abs(filepath.Join("..", "..", "..")) + code := `import { pathToFileURL } from 'node:url'; +const {validateOperationalValueEvidence} = await import(pathToFileURL(process.argv[1])); +let input = ''; for await (const chunk of process.stdin) input += chunk; +process.stdout.write(JSON.stringify(validateOperationalValueEvidence(JSON.parse(input))));` + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, + filepath.Join(root, "dashboard/site/src/data/model/operational-value-evidence.js")) // #nosec G204 -- fixed checked-in schema validator. + encoded, _ := json.Marshal(packet) + command.Stdin = bytes.NewReader(encoded) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("native evidence rejected by independent packet validator: %v: %s", err, output) + } + } + startCalls := *calls + if _, skipped, err := acquireNativeDependabotValue(t.Context(), api, first, observed); err != nil || !skipped || *calls != startCalls { + t.Fatalf("complete same cutoff reread GitHub: %v, %v, %d", err, skipped, *calls) + } + next := observed.Add(21 * 24 * time.Hour) + second, skipped, err := acquireNativeDependabotValue(t.Context(), api, first, next) + if err != nil || skipped { + t.Fatalf("new cutoff: %v, %v", err, skipped) + } + for i, packet := range second { + facts := packet["facts"].([]any) + if len(facts) != 5 || packet["contractDigest"] != first[i]["contractDigest"] || + facts[1].(map[string]any)["fact-id"] != "cutoff:2026-09-24T08:48:00Z" || + facts[4].(map[string]any)["at"] != "2026-10-08T08:48:00Z" { + t.Fatalf("retained history or skipped-week coverage changed: %+v", facts) + } + } + partial := append([]model.Row(nil), first[:2]...) + if _, _, err := acquireNativeDependabotValue(t.Context(), api, partial, next); err == nil { + t.Fatal("incomplete prior batch was treated as zero") + } + malformed := append([]model.Row(nil), first...) + altered := model.Row{} + for key, value := range first[0] { + altered[key] = value + } + facts := append([]any(nil), first[0]["facts"].([]any)...) + snapshot := map[string]any{} + for key, value := range facts[1].(map[string]any) { + snapshot[key] = value + } + snapshot["value"] = "missing" + facts[1] = snapshot + altered["facts"] = facts + malformed[0] = altered + if _, _, err := acquireNativeDependabotValue(t.Context(), api, malformed, observed); err == nil { + t.Fatal("malformed same-cutoff snapshot was reused") + } +} diff --git a/server/internal/collect/projector_remote_value_evidence.go b/server/internal/collect/projector_remote_value_evidence.go new file mode 100644 index 0000000000..4214359750 --- /dev/null +++ b/server/internal/collect/projector_remote_value_evidence.go @@ -0,0 +1,636 @@ +package collect + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/url" + "slices" + "sort" + "strconv" + "strings" + + "github.com/githubnext/gh-cao/server/internal/model" +) + +type fileDietEvidenceRef struct { + Label string `json:"label"` + URL string `json:"url"` +} +type fileDietMetric struct { + ID string `json:"id"` + Name string `json:"name"` + Unit string `json:"unit"` + Direction string `json:"direction"` + Version string `json:"version"` + Formula string `json:"formula"` +} +type fileDietIntent struct { + Goal string `json:"goal"` + Acceptance string `json:"acceptance"` + Requirements []string `json:"requirements"` + Sources []fileDietEvidenceRef `json:"sources"` +} +type fileDietLineage struct { + OutputID string `json:"outputId"` + ImplementationID string `json:"implementationId"` + Relation string `json:"relation"` + At string `json:"at"` +} +type fileDietEvent struct { + ID string + At string + Value int + Attribution string + Label string + Title string + Detail string + Reference string + Evidence []fileDietEvidenceRef + Lineage []fileDietLineage +} +type fileDietOutput struct { + ID string + Label string + Title string + URL string + CreatedAt string + State string + Origin string + ImplementationIDs []string + Evidence []fileDietEvidenceRef +} +type fileDietObservation struct { + At string + Value int + Ref string + URL string + Items []fileDietEvidenceRef +} +type fileDietLedger struct { + Proof fileDietProof + Metric fileDietMetric + Snapshot fileDietMetric + Intent fileDietIntent + Events []fileDietEvent + Outputs []fileDietOutput + Observations []fileDietObservation + Limitations []string + Coverage string +} + +func fileDietEncodeJSON(value any) []byte { + var buffer bytes.Buffer + writer := json.NewEncoder(&buffer) + writer.SetEscapeHTML(false) + if writer.Encode(value) != nil { + return nil + } + return bytes.TrimSuffix(buffer.Bytes(), []byte{'\n'}) +} +func fileDietDigest(value any) string { + return fmt.Sprintf("%x", sha256.Sum256(fileDietEncodeJSON(value))) +} + +func fileDietFileReference(file fileDietFile, commit, prefix string) (fileDietEvidenceRef, error) { + if !validFileDietSHA(commit) || !fileDietEligible(file.Path) { + return fileDietEvidenceRef{}, errors.New("invalid immutable File Diet file reference") + } + segments := strings.Split(file.Path, "/") + for index, segment := range segments { + if segment == "" || segment == "." || segment == ".." { + return fileDietEvidenceRef{}, errors.New("unsafe immutable File Diet file reference") + } + segments[index] = url.PathEscape(segment) + } + line := max(1, min(file.Lines, 800)) + if file.Lines < 800 { + line = 1 + } + return fileDietEvidenceRef{ + Label: fmt.Sprintf("%s%s (%d lines)", prefix, file.Path, file.Lines), + URL: fmt.Sprintf("https://github.com/%s/blob/%s/%s#L%d", dailyFileDietRepository, + commit, strings.Join(segments, "/"), line), + }, nil +} + +func fileDietReplay(proof fileDietProof, measured fileDietMeasurements) (fileDietLedger, error) { + var result fileDietLedger + if proof.Adoption.Commit != "1186030a620f4113f655c156bedf70cf2c164f79" || + proof.Adoption.At != "2025-11-15T13:36:21Z" || + len(proof.Snapshots) == 0 || proof.Snapshots[0].At != proof.Adoption.At || + proof.Snapshots[len(proof.Snapshots)-1].At != proof.Until { + return result, errors.New("file Diet frozen adoption or snapshot cutoff changed") + } + root := "https://github.com/" + dailyFileDietRepository + adopted := root + "/blob/" + proof.Adoption.Commit + "/.github/workflows" + intent := fileDietIntent{ + Goal: "Identify oversized production Go files and promote behavior-preserving decomposition.", + Acceptance: "This named structural facet measures net 800+ file reductions at linked merges; full behavior-preserving acceptance remains unknown.", + Requirements: []string{ + "Inspect the largest non-test Go source file under pkg/ and propose a focused refactoring issue when the workflow's size threshold is met.", + "Preserve behavior and public API; keep every new file under 500 lines and require at least 80% test coverage for new files.", + "Require passing tests, lint and build. Creating the proposal is not the same as implementing it.", + }, + Sources: []fileDietEvidenceRef{ + {"Workflow Markdown at adoption", adopted + "/daily-file-diet.md"}, + {"Compiled workflow at adoption", adopted + "/daily-file-diet.lock.yml"}, + {"Reporting import at adoption", adopted + "/shared/reporting.md"}, + {"Serena import at adoption", adopted + "/shared/mcp/serena.md"}, + }, + } + metric := fileDietMetric{ + "oversized-files-reduced", "Workflow-enabled oversized-file reductions", "files", + "increase", "observed-800-line-net-reductions-v1", + "Count regular non-test Go files under pkg/ with at least 800 newline characters before the merge, minus the same count after it, including replacements.", + } + snapshot := fileDietMetric{ + "oversized-files", "Oversized production Go files", "files", + "decrease", "observed-800-line-snapshot-v1", + "Count regular non-test Go files under pkg/ with at least 800 newline characters at the snapshot revision.", + } + result = fileDietLedger{Proof: proof, Metric: metric, Snapshot: snapshot, Intent: intent, + Events: []fileDietEvent{}, Outputs: []fileDietOutput{}, Observations: []fileDietObservation{}} + inventory := func(commit string) ([]fileDietFile, error) { + return fileDietInventoryByRevision(measured, commit) + } + eligible := make([]fileDietIssue, 0) + receipt := map[int]bool{} + implementations := map[string][]string{} + for _, issue := range proof.Issues { + if issue.Eligible != nil && !*issue.Eligible || issue.Receipt != nil && + issue.Receipt.Status == "not-applied-to-output" { + continue + } + if issue.Receipt != nil && issue.Receipt.Status == "available" { + if issue.Reference.RunID == nil || issue.Run.Validation == nil || + strconv.FormatInt(issue.Receipt.RunID, 10) != *issue.Reference.RunID || + issue.Receipt.HeadSHA != issue.Run.Validation.HeadSHA || + fileDietRawBytesDigest([]byte(issue.Receipt.Content)) != issue.Receipt.EntrySHA256 || + issue.Receipt.Digest != "sha256:"+issue.Receipt.ArchiveSHA256 { + return result, errors.New("file Diet applied receipt is not verified") + } + var source struct { + Type string `json:"type"` + Number int `json:"number"` + Repo string `json:"repo"` + URL string `json:"url"` + Metadata *struct { + NodeID string `json:"node_id"` + } `json:"metadata"` + } + if err := json.Unmarshal([]byte(issue.Receipt.Content), &source); err != nil || + source.Type != "create_issue" || source.Number != issue.Number || + source.Repo != dailyFileDietRepository && source.Repo != "githubnext/gh-aw" || + source.URL != fmt.Sprintf("https://github.com/%s/issues/%d", source.Repo, source.Number) || + source.Metadata != nil && source.Metadata.NodeID != issue.ID { + return result, errors.New("file Diet creation receipt does not identify output") + } + receipt[issue.Number] = true + } + eligible = append(eligible, issue) + implementations[issue.ID] = []string{} + } + var undatedLinks int + for _, pull := range proof.Pulls { + if pull.MergedAt == nil || !fileDietTime(*pull.MergedAt).Before(fileDietTime(proof.Until)) { + continue + } + if pull.MergeSHA == nil || pull.PreMergeSHA == "" || + pull.Graph.ID != pull.ID || pull.Graph.State != "MERGED" || + pull.Graph.MergeCommit == nil || *pull.Graph.MergeCommit != *pull.MergeSHA || + pull.Graph.MergedAt == nil || *pull.Graph.MergedAt != *pull.MergedAt { + return result, errors.New("file Diet accepted implementation revision mismatch or not merged") + } + linked := []fileDietIssue{} + lineage := []fileDietLineage{} + for _, issue := range eligible { + if fileDietTime(issue.CreatedAt).After(fileDietTime(pull.CreatedAt)) { + continue + } + var connection *fileDietTimelineEvent + for index := range issue.Timeline.Events { + event := &issue.Timeline.Events[index] + if event.Target != nil && event.Target.Kind == "pull-request" && + event.Target.Number == pull.Number && + (event.Type == "connected" || event.Type == "disconnected") && + !fileDietTime(event.At).After(fileDietTime(*pull.MergedAt)) && + (connection == nil || !fileDietTime(event.At).Before(fileDietTime(connection.At))) { + connection = event + } + } + connected := connection != nil && connection.Type == "connected" + closing := false + var closedBy, referencedBy *fileDietEdge + for index := range pull.Graph.Edges { + edge := &pull.Graph.Edges[index] + if edge.Relation == "declares-closing" && edge.Target.ID == issue.ID { + closing = true + } + } + for index := range issue.Graph.Edges { + edge := &issue.Graph.Edges[index] + if edge.Target.ID != pull.ID { + continue + } + if edge.Relation == "closed-by" && !fileDietTime(edge.At).Before(fileDietTime(*pull.MergedAt)) && + !fileDietTime(edge.At).After(fileDietTime(proof.ObservedAt)) && closedBy == nil { + closedBy = edge + } + if edge.Relation == "referenced-by" && !fileDietTime(edge.At).After(fileDietTime(*pull.MergedAt)) && + referencedBy == nil { + referencedBy = edge + } + } + if closing && !connected && closedBy == nil { + undatedLinks++ + } + if !closing && !connected { + continue + } + linked = append(linked, issue) + implementations[issue.ID] = append(implementations[issue.ID], pull.ID) + if referencedBy != nil { + lineage = append(lineage, fileDietLineage{issue.ID, pull.ID, "referenced-by", referencedBy.At}) + } + if connected { + lineage = append(lineage, fileDietLineage{issue.ID, pull.ID, "connected", connection.At}) + } + if closedBy != nil { + lineage = append(lineage, fileDietLineage{issue.ID, pull.ID, "closed-by", closedBy.At}) + } + } + if len(linked) == 0 { + continue + } + before, err := inventory(pull.PreMergeSHA) + if err != nil { + return result, err + } + after, err := inventory(*pull.MergeSHA) + if err != nil { + return result, err + } + oversized := func(files []fileDietFile) int { + count := 0 + for _, file := range files { + if file.Lines >= 800 { + count++ + } + } + return count + } + byPath := func(files []fileDietFile, path string) *fileDietFile { + for index := range files { + if files[index].Path == path { + return &files[index] + } + } + return nil + } + targetPaths := []string{} + for _, issue := range linked { + if issue.TargetPath != nil && !slices.Contains(targetPaths, *issue.TargetPath) { + targetPaths = append(targetPaths, *issue.TargetPath) + } + } + measurement := func(file *fileDietFile) string { + if file == nil { + return "absent" + } + return strconv.Itoa(file.Lines) + } + targets := []string{} + refs := []fileDietEvidenceRef{} + for _, target := range targetPaths { + original, accepted := byPath(before, target), byPath(after, target) + targets = append(targets, fmt.Sprintf("%s: %s -> %s lines", target, measurement(original), measurement(accepted))) + for _, candidate := range []struct { + file *fileDietFile + commit string + prefix string + }{{original, pull.PreMergeSHA, "Before: "}, {accepted, *pull.MergeSHA, "After: "}} { + if candidate.file != nil { + ref, err := fileDietFileReference(*candidate.file, candidate.commit, candidate.prefix) + if err != nil { + return result, err + } + refs = append(refs, ref) + } + } + } + additions := []string{} + for _, replacement := range pull.Files { + if replacement.Status != "added" && replacement.Status != "renamed" || + !fileDietEligible(replacement.Path) { + continue + } + accepted := byPath(after, replacement.Path) + additions = append(additions, fmt.Sprintf("%s (%s)", replacement.Path, measurement(accepted))) + if accepted != nil && !slices.Contains(targetPaths, accepted.Path) { + ref, err := fileDietFileReference(*accepted, *pull.MergeSHA, "Added/renamed: ") + if err != nil { + return result, err + } + refs = append(refs, ref) + } + } + evidence := []fileDietEvidenceRef{ + {"Merged PR", fmt.Sprintf("%s/pull/%d", root, pull.Number)}, + {"Accepted change diff", root + "/compare/" + pull.PreMergeSHA + "..." + *pull.MergeSHA}, + } + evidence = append(evidence, refs...) + for _, issue := range linked { + evidence = append(evidence, fileDietEvidenceRef{ + fmt.Sprintf("Origin issue #%d", issue.Number), + fmt.Sprintf("%s/issues/%d", root, issue.Number), + }) + } + evidence = append(evidence, fileDietEvidenceRef{"Before revision", root + "/tree/" + pull.PreMergeSHA}, + fileDietEvidenceRef{"Merge revision", root + "/tree/" + *pull.MergeSHA}) + attribution := "corroborated" + for _, issue := range linked { + if receipt[issue.Number] { + attribution = "receipt-verified" + } + } + detail := strings.Join(targets, ". ") + if len(additions) != 0 { + detail += ". Added " + strings.Join(additions, ", ") + } + detail += ". Net change includes replacement oversized files. Behavior preservation and full acceptance remain unverified. Shared workflow-enabled contribution." + result.Events = append(result.Events, fileDietEvent{ + ID: pull.ID, At: *pull.MergedAt, Value: oversized(before) - oversized(after), + Attribution: attribution, Label: fmt.Sprintf("%s#%d", dailyFileDietRepository, pull.Number), + Title: pull.Title, Detail: detail, Reference: *pull.MergeSHA, + Evidence: evidence, Lineage: lineage, + }) + } + complete := len(proof.Discovery) != 0 + for _, discovery := range proof.Discovery { + complete = complete && discovery.Complete + } + for _, issue := range proof.Issues { + complete = complete && issue.Timeline.Complete && issue.Graph.HistoryComplete + } + for _, pull := range proof.Pulls { + if pull.MergedAt != nil { + complete = complete && pull.Graph.ClosingLinksComplete && pull.Graph.HistoryComplete + } + } + result.Coverage = "partial" + if complete { + result.Coverage = "complete" + } + for _, issue := range eligible { + state := "unknown" + if issue.Graph.HistoryComplete { + state = "open" + events := make([]fileDietGraphEvent, 0) + for _, event := range issue.Graph.Events { + if (event.Type == "closed" || event.Type == "reopened") && + fileDietTime(event.At).Before(fileDietTime(proof.Until)) { + events = append(events, event) + } + } + sort.SliceStable(events, func(i, j int) bool { + return fileDietTime(events[i].At).Before(fileDietTime(events[j].At)) + }) + if len(events) > 0 && events[len(events)-1].Type == "closed" { + state = "closed" + } + } + refs := []fileDietEvidenceRef{} + if issue.Run.Validation != nil && issue.Run.Validation.URL != "" { + refs = append(refs, fileDietEvidenceRef{"Referenced producer run", issue.Run.Validation.URL}) + } + origin := "corroborated" + if receipt[issue.Number] { + origin = "receipt-verified" + } + result.Outputs = append(result.Outputs, fileDietOutput{ + ID: issue.ID, Label: fmt.Sprintf("%s#%d", dailyFileDietRepository, issue.Number), + Title: issue.Title, URL: fmt.Sprintf("%s/issues/%d", root, issue.Number), + CreatedAt: issue.CreatedAt, State: state, Origin: origin, + ImplementationIDs: implementations[issue.ID], Evidence: refs, + }) + } + for _, snapshot := range proof.Snapshots { + files, err := inventory(snapshot.Commit) + if err != nil { + return result, err + } + refs := []fileDietEvidenceRef{} + var count int + for _, file := range files { + if file.Lines >= 800 { + count++ + ref, err := fileDietFileReference(file, snapshot.Commit, "") + if err != nil { + return result, err + } + refs = append(refs, ref) + } + } + result.Observations = append(result.Observations, fileDietObservation{ + At: snapshot.At, Value: count, Ref: snapshot.Commit, + URL: root + "/tree/" + snapshot.Commit, Items: refs, + }) + } + notes := []string{ + "Coverage is the surviving discovered File Diet issue cohort and its explicit merge links, not deleted outputs or an exclusive causal effect.", + "Intent is consolidated from the adoption-time Markdown and imports. Its original issue-creation threshold was 1000 lines; the preserved retrospective instrument measures the separately named 800+ structural facet, not unchanged full acceptance.", + } + for _, issue := range proof.Issues { + if issue.Eligible != nil && !*issue.Eligible || issue.Receipt != nil && issue.Receipt.Status == "not-applied-to-output" { + notes = append(notes, "Tracked auxiliary outputs and artifacts naming a different created output remain in the lifecycle proof but are not credited as eligible File Diet deliveries.") + break + } + } + if !complete { + notes = append(notes, "Discovery or relationship history is incomplete; observed events do not establish complete-period totals.") + } + if undatedLinks > 0 { + notes = append(notes, fmt.Sprintf("%d retained closing links lack a dated connection or closed-by event; these corroborate lineage rather than prove when that relationship was recorded.", undatedLinks)) + } + notes = append(notes, + "Creation receipts strengthen attribution within the observed contribution; they do not cap its independently measured native change.", + "Full behavior preservation, required coverage and all-new-files-under-500 acceptance remain unverified; this measures structural progress.", + fmt.Sprintf("The repository oversized-file count changes from %d to %d. Other actors' growth is not attributed to this workflow.", + result.Observations[0].Value, result.Observations[len(result.Observations)-1].Value), + "This contribution pulse is not a continuous repository-state or response-latency series; sparse repository snapshots cannot locate changes between their cutoffs.") + if proof.Acquisition != nil { + notes = append(notes, "GitHub relationships were freshly collected; immutable inventories and verified receipts are reused by digest where available. Expired receipts remain unavailable unless previously preserved.") + } else { + notes = append(notes, "Replay validates preserved digests and identities; it does not reacquire Git objects, historical links or expired receipt archives.") + } + result.Limitations = notes + return result, nil +} + +func fileDietExport(ledger fileDietLedger) (model.Row, error) { + proof, metric, snapshot := ledger.Proof, ledger.Metric, ledger.Snapshot + if ledger.Coverage != "complete" || len(ledger.Observations) < 2 || + !fileDietTime(proof.Until).After(fileDietTime(proof.Adoption.At)) || + fileDietTime(proof.Until).After(fileDietTime(proof.ObservedAt)) { + return nil, errors.New("incomplete File Diet acquisition; retaining accepted proof") + } + type contractInput struct { + Intent fileDietIntent `json:"intent"` + Metric fileDietMetric `json:"metric"` + Basis string `json:"basis"` + Snapshot fileDietMetric `json:"snapshot"` + CadenceDays int `json:"cadenceDays"` + MaturationDays int `json:"maturationDays"` + } + contract := fileDietDigest(contractInput{ledger.Intent, metric, "workflow-contribution", snapshot, 1, 0}) + facts := []any{} + add := func(kind, id string, fields model.Row) { + fact := map[string]any{"fact-kind": kind, "fact-id": id} + for key, value := range fields { + if value != nil { + fact[key] = value + } + } + facts = append(facts, fact) + } + add("contract", "contract", model.Row{ + "goal": ledger.Intent.Goal, "acceptance": ledger.Intent.Acceptance, + "metric-id": metric.ID, "metric-name": metric.Name, + "metric-unit": metric.Unit, "metric-direction": metric.Direction, + "metric-version": metric.Version, "metric-formula": metric.Formula, + "result-label": "file reductions", "measurement-basis": "workflow-contribution", + "snapshot-name": snapshot.Name, "snapshot-unit": snapshot.Unit, + "snapshot-direction": snapshot.Direction, "snapshot-id": snapshot.ID, + "snapshot-version": snapshot.Version, "snapshot-formula": snapshot.Formula, + "outputs-available": true, "repository-state-available": true, "checks-available": true, + "cadence-days": 1, "maturation-days": 0, "output-unit": "issues", + "implementation-unit": "merged PRs", + "output-scope": "Retained eligible File Diet proposal issues discovered through tracking markers and named producer references. Deleted or undiscovered outputs are not reconstructed.", + "output-coverage": ledger.Coverage, "workflow-name": "Daily File Diet", + }) + refs := func(kind, parent string, items []fileDietEvidenceRef) { + seen := map[string]bool{} + for _, item := range items { + id := fileDietDigest([]any{item.Label, item.URL, nil}) + if seen[id] { + continue + } + seen[id] = true + fields := model.Row{"label": item.Label, "url": item.URL} + if parent != "" { + fields["parent-id"] = parent + } + add(kind, id, fields) + } + } + refs("intent-source", "", ledger.Intent.Sources) + seen := map[string]bool{} + for _, requirement := range ledger.Intent.Requirements { + if !seen[requirement] { + add("requirement", fileDietDigest(requirement), model.Row{"label": requirement}) + seen[requirement] = true + } + } + for _, output := range ledger.Outputs { + add("output", output.ID, model.Row{"label": output.Label, "title": output.Title, + "url": output.URL, "at": output.CreatedAt, "state": output.State, "origin": output.Origin}) + for _, id := range output.ImplementationIDs { + add("output-implementation", id, model.Row{"parent-id": output.ID, "related-id": id}) + } + refs("output-evidence", output.ID, output.Evidence) + } + outputURL := map[string]string{} + for _, output := range ledger.Outputs { + outputURL[output.ID] = output.URL + } + for _, event := range ledger.Events { + var link string + for _, ref := range event.Evidence { + if strings.Contains(strings.ToLower(ref.Label), "implement") || + strings.Contains(strings.ToLower(ref.Label), "pull") || + strings.Contains(ref.Label, "PR") { + link = ref.URL + break + } + } + fields := model.Row{"at": event.At, "accepted-at": event.At, "value": event.Value, + "label": event.Label, "detail": event.Detail, "ref": event.Reference, + "origin": event.Attribution} + if event.Title != "" { + fields["title"] = event.Title + } + if link != "" { + fields["url"] = link + } + add("implementation", event.ID, fields) + refs("implementation-evidence", event.ID, event.Evidence) + for _, edge := range event.Lineage { + add("implementation-lineage", fileDietDigest(edge), model.Row{ + "parent-id": event.ID, "related-id": edge.OutputID, + "label": edge.Relation, "at": edge.At, "url": outputURL[edge.OutputID], + }) + } + } + for _, observation := range ledger.Observations { + id := fileDietDigest([]string{observation.At, observation.Ref}) + add("snapshot", id, model.Row{"at": observation.At, "value": observation.Value, + "ref": observation.Ref, "url": observation.URL, "items-available": true}) + refs("snapshot-item", id, observation.Items) + } + checks := []struct{ name, status, detail string }{ + {"Structural change measured", "not-verified", "No linked implementation was measured; output creation alone does not establish a structural result."}, + {"Behavior preserved", "not-verified", "This evidence does not independently establish unchanged behavior for every implementation."}, + {"All quality requirements met", "not-verified", "Required coverage and the size of every new file are not fully verified by this structural measure."}, + } + if len(ledger.Events) > 0 { + checks[0].status = "verified" + checks[0].detail = "File counts were checked before and after linked merges, including replacement files and zero or negative changes." + } + for _, check := range checks { + add("quality-check", fileDietDigest(check.name), model.Row{ + "label": check.name, "status": check.status, "detail": check.detail}) + } + add("coverage", fileDietDigest([]string{proof.Adoption.At, proof.Until}), model.Row{ + "at": proof.Adoption.At, "end-at": proof.Until, "coverage-status": ledger.Coverage}) + seen = map[string]bool{} + for _, limitation := range ledger.Limitations { + if !seen[limitation] { + add("limitation", fileDietDigest(limitation), model.Row{ + "label": "Evidence limitation", "detail": limitation}) + seen[limitation] = true + } + } + add("limitation", "repository-state", model.Row{ + "label": "Independent repository state", + "detail": "These snapshots include everyone's changes. They are sparse observations, not continuous monitoring; changes between cutoffs and their causes are unknown.", + }) + packet := model.Row{ + "schemaVersion": 1, "id": "operational-value-evidence:" + + url.QueryEscape(strings.ToLower(proof.Repository)+":"+dailyFileDietWorkflow+":"+metric.ID+":"+metric.Version), + "repository": proof.Repository, "workflow": dailyFileDietWorkflow, + "sourcePath": fileDietSourcePath, "observedAt": proof.ObservedAt, + "windowStart": proof.Adoption.At, "windowEnd": proof.Until, + "contractDigest": contract, "facts": facts, + } + // All selected contributions must be tied to a retained eligible output, + // including negative and zero net changes. + accepted := map[string]bool{} + for _, output := range ledger.Outputs { + for _, id := range output.ImplementationIDs { + accepted[id] = true + } + } + for _, event := range ledger.Events { + if !accepted[event.ID] || !fileDietTime(event.At).Before(fileDietTime(proof.Until)) || + fileDietTime(event.At).Before(fileDietTime(proof.Adoption.At)) { + return nil, errors.New("file Diet contribution lacks eligible output trail") + } + } + if len(facts) > 20000 || len(fileDietEncodeJSON(packet)) > 8<<20 { + return nil, errors.New("file Diet evidence packet exceeds publication byte budget") + } + return packet, nil +} diff --git a/server/internal/collect/projector_remote_value_git.go b/server/internal/collect/projector_remote_value_git.go new file mode 100644 index 0000000000..690561364b --- /dev/null +++ b/server/internal/collect/projector_remote_value_git.go @@ -0,0 +1,330 @@ +package collect + +import ( + "bytes" + "context" + "crypto/sha1" // #nosec G505 -- Git repository object IDs use SHA-1 by design. + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +const ( + fileDietHistoricalRevision = "0782148c0af21f4f1d10b7cb5e81b571ed58c463" + fileDietHistoricalPath = "docs/operational-value/proof/file-diet-lifetime" + fileDietGitOutputLimit = 128 << 20 + fileDietGitWorkspaceLimit = 512 << 20 +) + +// Git reads only public immutable objects from the two reviewed coordinates. +// No installation credential or ambient Git configuration crosses this boundary. +type nativeFileDietGit struct { + binary string + dir string +} + +func newNativeFileDietGit(ctx context.Context, workspace, repository string) (*nativeFileDietGit, error) { + if repository != dailyFileDietRepository && repository != "githubnext/gh-cao" { + return nil, errors.New("unreviewed File Diet git repository") + } + binary, err := exec.LookPath("git") + if err != nil { + return nil, errors.New("reviewed File Diet immutable Git transport is unavailable") + } + binary, err = filepath.EvalSymlinks(binary) + if err != nil { + return nil, err + } + binary, err = filepath.Abs(binary) + if err != nil { + return nil, err + } + git := &nativeFileDietGit{binary: binary, dir: filepath.Join(workspace, "git")} + if _, err := git.run(ctx, nil, "init", "--bare", "--quiet", git.dir); err != nil { + return nil, err + } + if _, err := git.run(ctx, nil, "remote", "add", "origin", "https://github.com/"+repository+".git"); err != nil { + return nil, err + } + for _, args := range [][]string{ + {"config", "remote.origin.promisor", "true"}, + {"config", "remote.origin.partialclonefilter", "blob:none"}, + {"config", "core.hooksPath", "/dev/null"}, + } { + if _, err := git.run(ctx, nil, args...); err != nil { + return nil, err + } + } + return git, nil +} + +func (g *nativeFileDietGit) run(ctx context.Context, stdin []byte, args ...string) ([]byte, error) { + runCtx, cancel := context.WithTimeout(ctx, 240*time.Second) + defer cancel() + command := exec.CommandContext(runCtx, g.binary, append([]string{"--git-dir", g.dir}, args...)...) // #nosec G204 -- pinned executable, bounded reviewed immutable object operations. + command.Env = []string{ + "PATH=" + filepath.Dir(g.binary) + ":/usr/bin:/bin", + "HOME=" + filepath.Dir(g.dir), + "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_ASKPASS=/bin/false", "SSH_ASKPASS=/bin/false", "GIT_SSH_COMMAND=false", + "GIT_OPTIONAL_LOCKS=0", "GIT_LFS_SKIP_SMUDGE=1", + "GIT_HTTP_MAX_REQUESTS=1", "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=protocol.file.allow", "GIT_CONFIG_VALUE_0=never", + } + command.Stdin = bytes.NewReader(stdin) + var stdout bytes.Buffer + command.Stdout = &boundedGitOutput{writer: &stdout, remaining: fileDietGitOutputLimit} + command.Stderr = &boundedGitOutput{writer: io.Discard, remaining: 16 << 10} + remoteValueProcessGroup(command) + command.WaitDelay = 5 * time.Second + err := command.Run() + stopRemoteValueProcessGroup(command) + if err != nil { + if runCtx.Err() != nil { + return nil, errors.New("reviewed File Diet Git acquisition timed out") + } + return nil, errors.New("reviewed File Diet Git acquisition failed") + } + return stdout.Bytes(), nil +} + +type boundedGitOutput struct { + writer io.Writer + remaining int +} + +func (w *boundedGitOutput) Write(value []byte) (int, error) { + if len(value) > w.remaining { + return 0, errors.New("reviewed File Diet Git output budget exceeded") + } + w.remaining -= len(value) + return w.writer.Write(value) +} + +func validFileDietSHA(value string) bool { + if len(value) != 40 { + return false + } + _, err := hex.DecodeString(value) + return err == nil && value == strings.ToLower(value) +} + +func (g *nativeFileDietGit) fetch(ctx context.Context, revisions []string, blobs bool) error { + values := slices.Clone(revisions) + slices.Sort(values) + values = slices.Compact(values) + for _, value := range values { + if !validFileDietSHA(value) { + return errors.New("unreviewed immutable Git object identity") + } + } + limit := 100 + if blobs { + limit = 50 + } + for start := 0; start < len(values); start += limit { + end := min(start+limit, len(values)) + args := []string{"fetch", "--quiet", "--no-tags", "--no-write-fetch-head"} + if !blobs { + args = append(args, "--depth=1", "--filter=blob:none") + } + args = append(args, "origin") + if blobs { + args = append(args, "--stdin") + if _, err := g.run(ctx, []byte(strings.Join(values[start:end], "\n")+"\n"), args...); err != nil { + return err + } + } else { + if _, err := g.run(ctx, nil, append(args, values[start:end]...)...); err != nil { + return err + } + } + if err := fileDietGitWorkspaceWithinBudget(g.dir); err != nil { + return err + } + } + return nil +} + +func fileDietGitWorkspaceWithinBudget(root string) error { + var total int64 + files := 0 + return filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("reviewed File Diet Git workspace contains a symlink") + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + files++ + if total > fileDietGitWorkspaceLimit || files > 100000 { + return errors.New("reviewed File Diet Git workspace exceeds its byte or file budget") + } + return nil + }) +} + +type fileDietTreeEntry struct { + mode string + kind string + sha string + path string +} + +func parseFileDietTree(content []byte, prefix string, max int) ([]fileDietTreeEntry, error) { + rows := bytes.Split(bytes.TrimSuffix(content, []byte{0}), []byte{0}) + if len(rows) > max { + return nil, errors.New("immutable Git tree exceeds File Diet budget") + } + result := make([]fileDietTreeEntry, 0, len(rows)) + for _, row := range rows { + if len(row) == 0 { + continue + } + header, path, ok := bytes.Cut(row, []byte{'\t'}) + fields := strings.Fields(string(header)) + if !ok || len(fields) != 3 || !validFileDietSHA(fields[2]) || + strings.ContainsRune(string(path), 0) || !strings.HasPrefix(string(path), prefix) { + return nil, errors.New("invalid immutable File Diet Git tree") + } + result = append(result, fileDietTreeEntry{mode: fields[0], kind: fields[1], sha: fields[2], path: string(path)}) + } + return result, nil +} + +func (g *nativeFileDietGit) tree(ctx context.Context, commit, path string, limit int) ([]fileDietTreeEntry, error) { + if !validFileDietSHA(commit) || path != "pkg" && path != fileDietHistoricalPath { + return nil, errors.New("unreviewed File Diet Git tree scope") + } + output, err := g.run(ctx, nil, "ls-tree", "-r", "-z", commit, "--", path) + if err != nil { + return nil, err + } + return parseFileDietTree(output, path+"/", limit) +} + +func (g *nativeFileDietGit) firstParent(ctx context.Context, commit string) (string, error) { + if !validFileDietSHA(commit) { + return "", errors.New("invalid immutable merge revision") + } + output, err := g.run(ctx, nil, "cat-file", "-p", commit) + if err != nil { + return "", err + } + for _, line := range bytes.Split(output, []byte{'\n'}) { + if parent, ok := bytes.CutPrefix(line, []byte("parent ")); ok && validFileDietSHA(string(parent)) { + return string(parent), nil + } + if len(line) == 0 { + break + } + } + return "", errors.New("accepted revision lacks an actual first parent") +} + +func (g *nativeFileDietGit) blob(ctx context.Context, sha string, maxBytes int) ([]byte, error) { + if !validFileDietSHA(sha) { + return nil, errors.New("invalid immutable blob identity") + } + output, err := g.run(ctx, []byte(sha+"\n"), "cat-file", "--batch") + if err != nil { + return nil, err + } + header, rest, ok := bytes.Cut(output, []byte{'\n'}) + fields := strings.Fields(string(header)) + if !ok || len(fields) != 3 || fields[0] != sha || fields[1] != "blob" { + return nil, errors.New("immutable Git blob identity mismatch") + } + size, err := strconv.Atoi(fields[2]) + if err != nil || size < 0 || size > maxBytes || len(rest) != size+1 || rest[size] != '\n' { + return nil, errors.New("immutable File Diet blob exceeds its byte budget") + } + content := rest[:size] + digest := sha1.New() // #nosec G401 -- Git object identity uses SHA-1, not a security signature. + _, _ = fmt.Fprintf(digest, "blob %d\x00", size) + _, _ = digest.Write(content) + if fmt.Sprintf("%x", digest.Sum(nil)) != sha { + return nil, errors.New("native Git blob digest mismatch") + } + return content, nil +} + +func (g *nativeFileDietGit) measureBlobs(ctx context.Context, shas []string) (map[string][2]int, error) { + if len(shas) == 0 || len(shas) > 50 { + return nil, errors.New("reviewed File Diet blob batch exceeds its request budget") + } + for _, sha := range shas { + if !validFileDietSHA(sha) { + return nil, errors.New("invalid immutable blob identity") + } + } + output, err := g.run(ctx, []byte(strings.Join(shas, "\n")+"\n"), "cat-file", "--batch") + if err != nil { + return nil, err + } + return parseFileDietBlobMeasurements(output, shas) +} + +func parseFileDietBlobMeasurements(output []byte, shas []string) (map[string][2]int, error) { + measurements := make(map[string][2]int, len(shas)) + cursor := 0 + for _, sha := range shas { + if !validFileDietSHA(sha) { + return nil, errors.New("invalid immutable blob identity") + } + end := bytes.IndexByte(output[cursor:], '\n') + if end < 0 { + return nil, errors.New("malformed immutable Git blob header") + } + fields := strings.Fields(string(output[cursor : cursor+end])) + if len(fields) != 3 || fields[0] != sha || fields[1] != "blob" { + return nil, errors.New("immutable Git blob identity mismatch") + } + size, err := strconv.Atoi(fields[2]) + cursor += end + 1 + if err != nil || size < 0 || size >= fileDietGitOutputLimit || + size+1 > len(output)-cursor || output[cursor+size] != '\n' { + return nil, errors.New("immutable File Diet blob exceeds its batch byte budget") + } + content := output[cursor : cursor+size] + digest := sha1.New() // #nosec G401 -- Git object identity uses SHA-1, not a security signature. + _, _ = fmt.Fprintf(digest, "blob %d\x00", size) + _, _ = digest.Write(content) + if fmt.Sprintf("%x", digest.Sum(nil)) != sha { + return nil, errors.New("native Git blob digest mismatch") + } + measurements[sha] = [2]int{bytes.Count(content, []byte{'\n'}), size} + cursor += size + 1 + } + if cursor != len(output) { + return nil, errors.New("unexpected trailing immutable Git blob content") + } + return measurements, nil +} + +func fileDietGitWorkspace(root string) (string, error) { + if root == "" { + return "", errors.New("reviewed File Diet requires a private catalog workspace") + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return "", errors.New("reviewed File Diet catalog workspace is unsafe") + } + return os.MkdirTemp(root, ".cao-file-diet-") +} diff --git a/server/internal/collect/projector_remote_value_history.go b/server/internal/collect/projector_remote_value_history.go new file mode 100644 index 0000000000..495f02b8f2 --- /dev/null +++ b/server/internal/collect/projector_remote_value_history.go @@ -0,0 +1,917 @@ +package collect + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" +) + +const fileDietSourcePath = ".github/workflows/daily-file-diet.md" + +var errFileDietReferenceNotFound = errors.New("file Diet tracking reference not found") + +var ( + fileDietCommentPattern = regexp.MustCompile(`(?s)`) + fileDietPathPattern = regexp.MustCompile(`pkg/[\w./-]+\.go\b`) + fileDietFooterPattern = regexp.MustCompile(`AI generated by \[Daily File Diet\]\(https://github\.com/([\w.-]+/[\w.-]+)/actions/runs/(\d+)\)`) + fileDietRunURLPattern = regexp.MustCompile(`^https://github\.com/([\w.-]+/[\w.-]+)/actions/runs/(\d+)(?:/attempts/(\d+))?$`) + fileDietLegacyRun = regexp.MustCompile(`^https://github\.com/([\w.-]+/[\w.-]+)/actions/runs/(\d+)$`) + fileDietSourceName = regexp.MustCompile(`(?i)^\[file-diet\]`) + fileDietRefactor = regexp.MustCompile(`(?i)\brefactor\b`) +) + +type fileDietRead func(context.Context, string, string, bool) ([]byte, error) + +func fileDietReadJSON[T any](ctx context.Context, read fileDietRead, endpoint, query string) (T, error) { + var value T + content, err := read(ctx, endpoint, query, false) + if err != nil { + return value, err + } + if err := json.Unmarshal(content, &value); err != nil { + return value, errors.New("incomplete reviewed File Diet GitHub response") + } + return value, nil +} + +func fileDietInstant(text string) bool { + return !fileDietTime(text).IsZero() +} + +func fileDietReferenceFor(body string) (*fileDietReference, error) { + var workflowID, trackerID string + var candidates []fileDietReference + for _, match := range fileDietCommentPattern.FindAllStringSubmatch(body, -1) { + comment := strings.TrimSpace(match[1]) + for _, marker := range []struct { + prefix string + target *string + }{{"gh-aw-workflow-id:", &workflowID}, {"gh-aw-tracker-id:", &trackerID}} { + if strings.HasPrefix(comment, marker.prefix) { + value := strings.TrimSpace(strings.TrimPrefix(comment, marker.prefix)) + if value == "" || strings.ContainsAny(value, " \t\r\n") || + *marker.target != "" && *marker.target != value { + return nil, errors.New("conflicting File Diet tracking identities") + } + *marker.target = value + } + } + if strings.HasPrefix(comment, "gh-aw-execution:") { + var marker struct { + Version int `json:"version"` + Repository string `json:"repository"` + RunID int64 `json:"run_id"` + RunAttempt int `json:"run_attempt"` + AWWorkflowID string `json:"aw_workflow_id"` + RunURL string `json:"run_url"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(strings.TrimPrefix(comment, "gh-aw-execution:"))), &marker); err != nil || + marker.Version != 1 || marker.RunURL == "" { + return nil, errors.New("invalid File Diet execution marker") + } + if marker.AWWorkflowID != dailyFileDietWorkflow { + continue + } + urlParts := fileDietRunURLPattern.FindStringSubmatch(marker.RunURL) + if len(urlParts) != 4 || !strings.EqualFold(marker.Repository, urlParts[1]) || + strconv.FormatInt(marker.RunID, 10) != urlParts[2] || marker.RunAttempt < 1 || + urlParts[3] != "" && urlParts[3] != strconv.Itoa(marker.RunAttempt) { + return nil, errors.New("conflicting File Diet execution identity") + } + candidates = append(candidates, fileDietReference{ + Repository: urlParts[1], RunID: &urlParts[2], Attempt: &marker.RunAttempt, Marker: "gh-aw-execution-v1", + }) + } + if strings.HasPrefix(comment, "gh-aw-agentic-workflow:") && + strings.HasPrefix(strings.TrimSpace(strings.TrimPrefix(comment, "gh-aw-agentic-workflow:")), "Daily File Diet") { + runField := strings.SplitN(comment, "run:", 2) + if len(runField) != 2 { + return nil, errors.New("legacy File Diet marker lacks a run reference") + } + matches := fileDietLegacyRun.FindStringSubmatch(strings.TrimSpace(runField[1])) + if len(matches) != 3 { + return nil, errors.New("legacy File Diet marker lacks a run reference") + } + for _, field := range []struct { + name string + delimiter string + }{{"workflow_id: ", "workflow_id: "}, {"id: ", ", id: "}} { + if marker := strings.SplitN(comment, field.delimiter, 2); len(marker) == 2 { + selected := strings.FieldsFunc(marker[1], func(r rune) bool { return r == ',' || r == ' ' || r == '\n' }) + if len(selected) == 0 || field.name == "workflow_id: " && selected[0] != dailyFileDietWorkflow || + field.name == "id: " && selected[0] != matches[2] { + return nil, errors.New("conflicting legacy File Diet identity") + } + } + } + candidates = append(candidates, fileDietReference{Repository: matches[1], RunID: &matches[2], Marker: "gh-aw-agentic-workflow"}) + } + } + for _, match := range fileDietFooterPattern.FindAllStringSubmatch(body, -1) { + candidates = append(candidates, fileDietReference{Repository: match[1], RunID: &match[2], Marker: "named-run-footer"}) + } + var selected *fileDietReference + runID := "" + attempt := 0 + for index := range candidates { + candidate := &candidates[index] + if !strings.EqualFold(candidate.Repository, dailyFileDietRepository) && + !strings.EqualFold(candidate.Repository, "githubnext/gh-aw") { + return nil, errors.New("file Diet producer outside reviewed repository") + } + if runID != "" && runID != *candidate.RunID || + attempt != 0 && candidate.Attempt != nil && attempt != *candidate.Attempt { + return nil, errors.New("conflicting File Diet producer runs or attempts") + } + runID = *candidate.RunID + if candidate.Attempt != nil { + attempt = *candidate.Attempt + } + if selected == nil || candidate.Marker == "gh-aw-execution-v1" { + selected = candidate + } + } + if workflowID != "" && workflowID != dailyFileDietWorkflow { + return nil, errFileDietReferenceNotFound + } + if selected == nil && workflowID != dailyFileDietWorkflow && trackerID != dailyFileDietWorkflow { + return nil, errFileDietReferenceNotFound + } + if selected == nil { + selected = &fileDietReference{Repository: dailyFileDietRepository, Marker: "tracking-only"} + } + selected.WorkflowID = dailyFileDietWorkflow + if trackerID != "" { + selected.TrackerID = &trackerID + } + return selected, nil +} + +type fileDietGraphEntity struct { + Type string `json:"__typename"` + ID string `json:"id"` + Number int `json:"number"` + URL string `json:"url"` + Repository struct { + Name string `json:"nameWithOwner"` + } `json:"repository"` + State string `json:"state"` + StateReason *string `json:"stateReason"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + ClosedAt *string `json:"closedAt"` + MergedAt *string `json:"mergedAt"` + HeadRefOID string `json:"headRefOid"` + MergeCommit *struct { + OID string `json:"oid"` + } `json:"mergeCommit"` + ClosingIssuesReferences *struct { + Nodes []fileDietGraphEntity `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + } `json:"pageInfo"` + } `json:"closingIssuesReferences"` + TimelineItems *struct { + Nodes []struct { + Type string `json:"__typename"` + CreatedAt string `json:"createdAt"` + Closer *fileDietGraphEntity `json:"closer"` + Source *fileDietGraphEntity `json:"source"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + } `json:"pageInfo"` + } `json:"timelineItems"` +} + +func fileDietGraphTarget(entity fileDietGraphEntity) (fileDietTarget, error) { + target := fileDietTarget{entity.ID, entity.Number, entity.Repository.Name, entity.URL} + if target.ID == "" || target.Number <= 0 || target.Repository == "" || + target.URL != "https://github.com/"+target.Repository+"/issues/"+strconv.Itoa(target.Number) && + target.URL != "https://github.com/"+target.Repository+"/pull/"+strconv.Itoa(target.Number) { + return target, errors.New("invalid File Diet graph entity identity") + } + return target, nil +} + +func fileDietNormalizeGraph(entity fileDietGraphEntity, observed string) (fileDietGraph, error) { + var graph fileDietGraph + ref, err := fileDietGraphTarget(entity) + if err != nil { + return graph, err + } + if !strings.EqualFold(ref.Repository, dailyFileDietRepository) || + (entity.Type != "Issue" && entity.Type != "PullRequest") || + !fileDietInstant(observed) || !fileDietInstant(entity.CreatedAt) || !fileDietInstant(entity.UpdatedAt) || + entity.TimelineItems == nil || entity.TimelineItems.Nodes == nil { + return graph, errors.New("incomplete File Diet entity graph") + } + for _, optional := range []*string{entity.ClosedAt, entity.MergedAt} { + if optional != nil && !fileDietInstant(*optional) { + return graph, errors.New("invalid File Diet graph timestamp") + } + } + graph = fileDietGraph{ + fileDietTarget: ref, Kind: entity.Type, State: entity.State, StateReason: entity.StateReason, + CreatedAt: entity.CreatedAt, UpdatedAt: entity.UpdatedAt, ClosedAt: entity.ClosedAt, + MergedAt: entity.MergedAt, ObservedAt: observed, + HistoryComplete: !entity.TimelineItems.PageInfo.HasNextPage, + ClosingLinksComplete: true, Edges: []fileDietEdge{}, Events: []fileDietGraphEvent{}, + } + if entity.MergeCommit != nil { + graph.MergeCommit = &entity.MergeCommit.OID + } + for _, event := range entity.TimelineItems.Nodes { + if !fileDietInstant(event.CreatedAt) { + return fileDietGraph{}, errors.New("file Diet graph event lacks timestamp") + } + kind := map[string]string{ + "ClosedEvent": "closed", "ReopenedEvent": "reopened", + "MergedEvent": "merged", "CrossReferencedEvent": "referenced", + }[event.Type] + if kind == "" { + return fileDietGraph{}, errors.New("unexpected File Diet graph event") + } + projected := fileDietGraphEvent{Type: kind, At: event.CreatedAt} + if kind == "closed" && event.Closer != nil && event.Closer.Type == "PullRequest" { + target, err := fileDietGraphTarget(*event.Closer) + if err != nil { + return fileDietGraph{}, err + } + graph.Edges = append(graph.Edges, fileDietEdge{Relation: "closed-by", At: event.CreatedAt, Target: target}) + projected.CloserID = target.ID + } + if kind == "referenced" { + if event.Source == nil { + return fileDietGraph{}, errors.New("missing File Diet graph reference source") + } + target, err := fileDietGraphTarget(*event.Source) + if err != nil { + return fileDietGraph{}, err + } + graph.Edges = append(graph.Edges, fileDietEdge{Relation: "referenced-by", At: event.CreatedAt, Target: target}) + } + graph.Events = append(graph.Events, projected) + } + if entity.Type == "PullRequest" { + if entity.ClosingIssuesReferences == nil || entity.ClosingIssuesReferences.Nodes == nil { + return fileDietGraph{}, errors.New("missing File Diet closing relationship coverage") + } + for _, issue := range entity.ClosingIssuesReferences.Nodes { + target, err := fileDietGraphTarget(issue) + if err != nil { + return fileDietGraph{}, err + } + graph.Edges = append(graph.Edges, fileDietEdge{Relation: "declares-closing", Target: target}) + } + graph.ClosingLinksComplete = !entity.ClosingIssuesReferences.PageInfo.HasNextPage + } + return graph, nil +} + +func fileDietGraphs(ctx context.Context, read fileDietRead, numbers []int) (map[int]fileDietGraph, error) { + graphs := make(map[int]fileDietGraph, len(numbers)) + for offset := 0; offset < len(numbers); offset += 10 { + batch := numbers[offset:min(offset+10, len(numbers))] + var response struct { + Data struct { + Repository map[string]fileDietGraphEntity `json:"repository"` + } `json:"data"` + Errors []json.RawMessage `json:"errors"` + } + var err error + response, err = fileDietReadJSON[struct { + Data struct { + Repository map[string]fileDietGraphEntity `json:"repository"` + } `json:"data"` + Errors []json.RawMessage `json:"errors"` + }](ctx, read, "graphql", remoteValueExpectedGraphQuery(dailyFileDietRepository, batch)) + if err != nil || len(response.Errors) > 0 || response.Data.Repository == nil { + return nil, errors.New("incomplete File Diet graph acquisition") + } + for _, number := range batch { + entity, ok := response.Data.Repository["n"+strconv.Itoa(number)] + if !ok { + return nil, errors.New("missing File Diet graph entity") + } + graph, err := fileDietNormalizeGraph(entity, time.Now().UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z")) + if err != nil || graph.Number != number { + return nil, errors.New("file Diet graph identity changed") + } + graphs[number] = graph + } + } + return graphs, nil +} + +func fileDietReadTimeline(ctx context.Context, read fileDietRead, number int, until string) (fileDietTimeline, error) { + result := fileDietTimeline{Events: []fileDietTimelineEvent{}} + types := map[string]bool{"assigned": true, "unassigned": true, "connected": true, + "disconnected": true, "cross-referenced": true, "closed": true, "reopened": true, + "copilot_work_started": true, "copilot_work_finished": true, "merged": true} + for page := 1; page <= 5; page++ { + endpoint := fmt.Sprintf("repos/%s/issues/%d/timeline?per_page=100&page=%d", dailyFileDietRepository, number, page) + var rows []struct { + Event string `json:"event"` + CreatedAt string `json:"created_at"` + Subject *struct { + Number int `json:"number"` + Title string `json:"title"` + Type string `json:"type"` + PullRequest json.RawMessage `json:"pull_request"` + RepositoryURL string `json:"repository_url"` + Repository *struct { + FullName string `json:"full_name"` + } `json:"repository"` + } `json:"subject"` + Source *struct { + Issue *struct { + Number int `json:"number"` + Title string `json:"title"` + Type string `json:"type"` + PullRequest json.RawMessage `json:"pull_request"` + RepositoryURL string `json:"repository_url"` + Repository *struct { + FullName string `json:"full_name"` + } `json:"repository"` + } `json:"issue"` + } `json:"source"` + } + var err error + rows, err = fileDietReadJSON[[]struct { + Event string `json:"event"` + CreatedAt string `json:"created_at"` + Subject *struct { + Number int `json:"number"` + Title string `json:"title"` + Type string `json:"type"` + PullRequest json.RawMessage `json:"pull_request"` + RepositoryURL string `json:"repository_url"` + Repository *struct { + FullName string `json:"full_name"` + } `json:"repository"` + } `json:"subject"` + Source *struct { + Issue *struct { + Number int `json:"number"` + Title string `json:"title"` + Type string `json:"type"` + PullRequest json.RawMessage `json:"pull_request"` + RepositoryURL string `json:"repository_url"` + Repository *struct { + FullName string `json:"full_name"` + } `json:"repository"` + } `json:"issue"` + } `json:"source"` + }](ctx, read, endpoint, "") + if err != nil { + return result, err + } + for _, row := range rows { + if !types[row.Event] { + continue + } + if !fileDietInstant(row.CreatedAt) { + return result, errors.New("file Diet timeline event lacks a timestamp") + } + if !fileDietTime(row.CreatedAt).Before(fileDietTime(until)) { + continue + } + event := fileDietTimelineEvent{Type: row.Event, At: row.CreatedAt} + // Decode the two alternative subject shapes through the same bounded + // projection, keeping foreign repositories out of related PR scope. + var subject json.RawMessage + if row.Subject != nil { + subject, _ = json.Marshal(row.Subject) + } else if row.Source != nil && row.Source.Issue != nil { + subject, _ = json.Marshal(row.Source.Issue) + } + if len(subject) > 0 { + var candidate struct { + Number int `json:"number"` + Title string `json:"title"` + Type string `json:"type"` + PullRequest json.RawMessage `json:"pull_request"` + RepositoryURL string `json:"repository_url"` + Repository *struct { + FullName string `json:"full_name"` + } `json:"repository"` + } + _ = json.Unmarshal(subject, &candidate) + repository := strings.TrimPrefix(candidate.RepositoryURL, "https://api.github.com/repos/") + if candidate.Repository != nil && candidate.Repository.FullName != "" { + repository = candidate.Repository.FullName + } + if strings.EqualFold(repository, dailyFileDietRepository) { + if candidate.Number <= 0 { + return result, errors.New("invalid File Diet timeline target") + } + kind := "issue" + if candidate.Type == "pull_request" || len(candidate.PullRequest) > 0 && + !bytes.Equal(candidate.PullRequest, []byte("null")) { + kind = "pull-request" + } + event.Target = &fileDietTimelineTarget{candidate.Number, candidate.Title, kind} + } + } + result.Events = append(result.Events, event) + } + if len(rows) < 100 { + result.Complete = true + break + } + } + return result, nil +} + +type fileDietCommit struct { + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date string `json:"date"` + } `json:"committer"` + } `json:"commit"` +} + +func fileDietAcquireHistory(ctx context.Context, read fileDietRead, until string, previous *fileDietProof) (fileDietProof, error) { + var proof fileDietProof + cutoff := fileDietTime(until) + if cutoff.IsZero() || cutoff.After(time.Now()) || cutoff.Format(time.RFC3339) != until { + return proof, errors.New("file Diet requires a valid non-future UTC cutoff") + } + if previous != nil && (!fileDietInstant(previous.Until) || fileDietTime(previous.Until).After(cutoff)) { + return proof, errors.New("previous File Diet proof follows requested cutoff") + } + var identity struct { + ID int64 `json:"id"` + FullName string `json:"full_name"` + } + for _, name := range []string{dailyFileDietRepository, "githubnext/gh-aw"} { + got, err := fileDietReadJSON[struct { + ID int64 `json:"id"` + FullName string `json:"full_name"` + }](ctx, read, "repos/"+name, "") + if err != nil || got.ID <= 0 || got.FullName != dailyFileDietRepository || + identity.ID != 0 && identity.ID != got.ID { + return proof, errors.New("file Diet repository alias does not identify declared target") + } + identity = got + } + proof = fileDietProof{SchemaVersion: 1, Repository: dailyFileDietRepository, + RepositoryID: identity.ID, Aliases: []string{dailyFileDietRepository, "githubnext/gh-aw"}, + SourcePath: fileDietSourcePath, Until: until, + ObservedAt: time.Now().UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z"), + Discovery: []fileDietDiscovery{}, Issues: []fileDietIssue{}, Pulls: []fileDietPull{}, + Snapshots: []fileDietSnapshot{}, History: []fileDietHistoryRevision{}} + source := "repos/" + dailyFileDietRepository + "/commits?path=" + fileDietSourcePath + if previous != nil { + source += "&since=" + previous.Until + } + source += "&until=" + until + "&per_page=100" + commits, err := fileDietReadJSON[[]fileDietCommit](ctx, read, source, "") + if err != nil || len(commits) >= 100 || previous == nil && len(commits) == 0 { + return proof, errors.New("file Diet workflow source history is absent or truncated") + } + if previous != nil { + proof.Adoption = previous.Adoption + revisions := map[string]fileDietHistoryRevision{} + for _, row := range previous.History { + revisions[row.Commit] = row + } + for _, row := range commits { + revision := fileDietHistoryRevision{row.SHA, row.Commit.Committer.Date} + if retained, ok := revisions[revision.Commit]; ok && retained != revision { + return proof, errors.New("preserved File Diet source revision changed") + } + revisions[revision.Commit] = revision + } + for _, row := range revisions { + proof.History = append(proof.History, row) + } + sort.Slice(proof.History, func(i, j int) bool { + return fileDietTime(proof.History[i].At).After(fileDietTime(proof.History[j].At)) + }) + } else { + oldest := commits[len(commits)-1] + var introduction struct { + Files []struct{ Filename, Status string } `json:"files"` + Parents []struct { + SHA string `json:"sha"` + } `json:"parents"` + } + introduction, err = fileDietReadJSON[struct { + Files []struct{ Filename, Status string } `json:"files"` + Parents []struct { + SHA string `json:"sha"` + } `json:"parents"` + }](ctx, read, "repos/"+dailyFileDietRepository+"/commits/"+oldest.SHA, "") + if err != nil || len(introduction.Parents) != 1 { + return proof, errors.New("file Diet workflow adoption identity is incomplete") + } + added := map[string]bool{} + for _, file := range introduction.Files { + if file.Status == "added" { + added[file.Filename] = true + } + } + if !added[fileDietSourcePath] || !added[strings.TrimSuffix(fileDietSourcePath, ".md")+".lock.yml"] { + return proof, errors.New("file Diet workflow adoption identity is incomplete") + } + proof.Adoption = fileDietAdoption{Commit: oldest.SHA, + At: oldest.Commit.Committer.Date, BaselineCommit: introduction.Parents[0].SHA} + for _, row := range commits { + proof.History = append(proof.History, fileDietHistoryRevision{row.SHA, row.Commit.Committer.Date}) + } + } + if proof.Adoption.Commit != "1186030a620f4113f655c156bedf70cf2c164f79" || + proof.Adoption.At != "2025-11-15T13:36:21Z" { + return proof, errors.New("file Diet frozen adoption changed") + } + discoveryStart := proof.Adoption.At + field := "created" + if previous != nil { + discoveryStart, field = previous.Until, "updated" + } + terms := []string{`"[file-diet]" in:title`, `"gh-aw-workflow-id: daily-file-diet" in:body`, + `"gh-aw-tracker-id: daily-file-diet" in:body`, `"gh-aw-agentic-workflow: Daily File Diet" in:body`, + `"Daily File Diet" "AI generated by" in:body`, `"gh-aw-execution:" "daily-file-diet" in:body`} + type searchItem struct { + Number int `json:"number"` + NodeID string `json:"node_id"` + Title string `json:"title"` + Body string `json:"body"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + PullRequest json.RawMessage `json:"pull_request"` + } + found := map[int]searchItem{} + for _, term := range terms { + query := "repo:" + dailyFileDietRepository + " " + term + " " + field + ":" + + discoveryStart[:10] + ".." + until[:10] + total, count := -1, 0 + for page := 1; page <= 10; page++ { + endpoint := fmt.Sprintf("search/issues?q=%s&sort=%s&order=asc&per_page=100&page=%d", + url.QueryEscape(query), field, page) + var batch struct { + TotalCount int `json:"total_count"` + IncompleteResults bool `json:"incomplete_results"` + Items []searchItem `json:"items"` + } + batch, err = fileDietReadJSON[struct { + TotalCount int `json:"total_count"` + IncompleteResults bool `json:"incomplete_results"` + Items []searchItem `json:"items"` + }](ctx, read, endpoint, "") + if err != nil || batch.IncompleteResults || batch.TotalCount >= 1000 || batch.Items == nil || + total >= 0 && total != batch.TotalCount { + return proof, errors.New("file Diet issue search coverage is incomplete") + } + total = batch.TotalCount + count += len(batch.Items) + for _, item := range batch.Items { + at := item.CreatedAt + if field == "updated" { + at = item.UpdatedAt + } + if !fileDietTime(at).Before(fileDietTime(discoveryStart)) && fileDietTime(at).Before(cutoff) { + if retained, ok := found[item.Number]; ok && retained.NodeID != item.NodeID { + return proof, errors.New("conflicting File Diet discovered output") + } + found[item.Number] = item + } + } + if len(batch.Items) < 100 { + break + } + } + if count != total { + return proof, errors.New("file Diet issue search coverage was truncated") + } + proof.Discovery = append(proof.Discovery, fileDietDiscovery{query, total, true}) + } + previousIssues := map[int]fileDietIssue{} + if previous != nil { + for _, item := range previous.Issues { + previousIssues[item.Number] = item + } + } + newlySelected := map[int]searchItem{} + issueNumbers := map[int]bool{} + for number := range previousIssues { + issueNumbers[number] = true + } + for _, item := range found { + if len(item.PullRequest) != 0 && !bytes.Equal(item.PullRequest, []byte("null")) { + continue + } + ref, err := fileDietReferenceFor(item.Body) + if errors.Is(err, errFileDietReferenceNotFound) { + continue + } + if err != nil { + return proof, err + } + if ref != nil { + newlySelected[item.Number], issueNumbers[item.Number] = item, true + } + } + if len(issueNumbers) == 0 || len(issueNumbers) > 200 { + return proof, errors.New("file Diet tracked output collection budget exceeded or no outputs found") + } + numbers := make([]int, 0, len(issueNumbers)) + for number := range issueNumbers { + numbers = append(numbers, number) + } + slices.Sort(numbers) + graphs, err := fileDietGraphs(ctx, read, numbers) + if err != nil { + return proof, err + } + for _, number := range numbers { + timeline, err := fileDietReadTimeline(ctx, read, number, until) + if err != nil { + return proof, err + } + if retained, exists := previousIssues[number]; exists { + retained.Graph, retained.Timeline = graphs[number], timeline + proof.Issues = append(proof.Issues, retained) + continue + } + item := newlySelected[number] + ref, err := fileDietReferenceFor(item.Body) + if err != nil || ref == nil { + return proof, errors.New("file Diet output tracking changed") + } + run := fileDietRun{Unavailable: json.RawMessage("null")} + var artifacts json.RawMessage = []byte("null") + if ref.RunID == nil { + run.Unavailable = []byte(`{"reason":"Tracking identity has no producer run reference"}`) + } else { + endpoint := "repos/" + dailyFileDietRepository + "/actions/runs/" + *ref.RunID + var producer struct { + ID int64 `json:"id"` + Repository struct { + FullName string `json:"full_name"` + } `json:"repository"` + Path string `json:"path"` + HeadSHA string `json:"head_sha"` + RunAttempt int `json:"run_attempt"` + CreatedAt string `json:"created_at"` + HTMLURL string `json:"html_url"` + Unavailable json.RawMessage `json:"unavailable"` + } + producer, err = fileDietReadJSON[struct { + ID int64 `json:"id"` + Repository struct { + FullName string `json:"full_name"` + } `json:"repository"` + Path string `json:"path"` + HeadSHA string `json:"head_sha"` + RunAttempt int `json:"run_attempt"` + CreatedAt string `json:"created_at"` + HTMLURL string `json:"html_url"` + Unavailable json.RawMessage `json:"unavailable"` + }](ctx, read, endpoint, "") + if err != nil { + return proof, err + } + if len(producer.Unavailable) != 0 { + run.Unavailable = producer.Unavailable + } else { + if strconv.FormatInt(producer.ID, 10) != *ref.RunID || + !strings.EqualFold(producer.Repository.FullName, dailyFileDietRepository) && + !strings.EqualFold(producer.Repository.FullName, "githubnext/gh-aw") || + strings.Split(producer.Path, "@")[0] != ".github/workflows/daily-file-diet.lock.yml" || + !validFileDietSHA(producer.HeadSHA) || + ref.Attempt != nil && producer.RunAttempt != *ref.Attempt { + return proof, errors.New("file Diet producer run does not match workflow, repository, revision or attempt") + } + run.Validation = &fileDietValidation{"verified-run-reference", producer.HeadSHA, + producer.RunAttempt, fileDietSourcePath, producer.CreatedAt, producer.HTMLURL, + "Tracking and run references are not applied safe-output receipts."} + var inventory struct { + TotalCount int `json:"total_count"` + Artifacts []json.RawMessage `json:"artifacts"` + } + inventory, err = fileDietReadJSON[struct { + TotalCount int `json:"total_count"` + Artifacts []json.RawMessage `json:"artifacts"` + }](ctx, read, endpoint+"/artifacts?per_page=100", "") + if err != nil || inventory.Artifacts == nil || inventory.TotalCount != len(inventory.Artifacts) { + return proof, errors.New("truncated File Diet producer artifact inventory") + } + artifacts, _ = json.Marshal(inventory.Artifacts) + } + } + target := fileDietPathPattern.FindString(item.Title) + if target == "" { + target = fileDietPathPattern.FindString(item.Body) + } + var path *string + if target != "" { + path = &target + } + proof.Issues = append(proof.Issues, fileDietIssue{ + Number: number, ID: item.NodeID, Title: item.Title, CreatedAt: item.CreatedAt, + Eligible: func() *bool { + ok := fileDietSourceName.MatchString(item.Title) || + fileDietRefactor.MatchString(item.Title) && fileDietPathPattern.MatchString(item.Body) + return &ok + }(), + BodySHA256: fileDietRawBytesDigest([]byte(item.Body)), TargetPath: path, + Reference: *ref, Run: run, RetainedArtifacts: artifacts, Graph: graphs[number], + Timeline: timeline, + }) + } + related := map[int]bool{} + if previous != nil { + for _, pull := range previous.Pulls { + related[pull.Number] = true + } + } + for _, item := range found { + if len(item.PullRequest) == 0 || bytes.Equal(item.PullRequest, []byte("null")) { + continue + } + ref, err := fileDietReferenceFor(item.Body) + if errors.Is(err, errFileDietReferenceNotFound) { + continue + } + if err != nil { + return proof, err + } + if fileDietSourceName.MatchString(item.Title) || ref != nil { + related[item.Number] = true + } + } + for _, issue := range proof.Issues { + for _, event := range issue.Timeline.Events { + if event.Target != nil && event.Target.Kind == "pull-request" { + related[event.Target.Number] = true + } + } + for _, edge := range issue.Graph.Edges { + if edge.Target.Repository == dailyFileDietRepository && + edge.Target.URL == fmt.Sprintf("https://github.com/%s/pull/%d", + dailyFileDietRepository, edge.Target.Number) { + related[edge.Target.Number] = true + } + } + } + if len(related) > 100 { + return proof, errors.New("related File Diet implementation budget exceeded") + } + pullNumbers := make([]int, 0, len(related)) + for number := range related { + pullNumbers = append(pullNumbers, number) + } + slices.Sort(pullNumbers) + previousPulls := map[int]fileDietPull{} + if previous != nil { + for _, pull := range previous.Pulls { + previousPulls[pull.Number] = pull + } + } + for _, number := range pullNumbers { + if retained, ok := previousPulls[number]; ok && retained.MergedAt != nil && + fileDietTime(*retained.MergedAt).Before(fileDietTime(previous.Until)) { + proof.Pulls = append(proof.Pulls, retained) + continue + } + endpoint := fmt.Sprintf("repos/%s/pulls/%d", dailyFileDietRepository, number) + var pull struct { + Number int `json:"number"` + NodeID string `json:"node_id"` + Title string `json:"title"` + CreatedAt string `json:"created_at"` + MergedAt *string `json:"merged_at"` + Head struct { + SHA string `json:"sha"` + } `json:"head"` + Base struct { + SHA string `json:"sha"` + } `json:"base"` + MergeCommitSHA string `json:"merge_commit_sha"` + ChangedFiles int `json:"changed_files"` + } + pull, err = fileDietReadJSON[struct { + Number int `json:"number"` + NodeID string `json:"node_id"` + Title string `json:"title"` + CreatedAt string `json:"created_at"` + MergedAt *string `json:"merged_at"` + Head struct { + SHA string `json:"sha"` + } `json:"head"` + Base struct { + SHA string `json:"sha"` + } `json:"base"` + MergeCommitSHA string `json:"merge_commit_sha"` + ChangedFiles int `json:"changed_files"` + }](ctx, read, endpoint, "") + if err != nil || pull.Number != number || pull.NodeID == "" || pull.ChangedFiles > 1000 { + return proof, errors.New("invalid File Diet implementation metadata") + } + graph, err := fileDietGraphs(ctx, read, []int{number}) + if err != nil { + return proof, err + } + files := []fileDietPullFile{} + for page := 1; page <= 10; page++ { + var pageFiles []struct { + Filename string `json:"filename"` + Status string `json:"status"` + PreviousFilename *string `json:"previous_filename"` + } + pageFiles, err = fileDietReadJSON[[]struct { + Filename string `json:"filename"` + Status string `json:"status"` + PreviousFilename *string `json:"previous_filename"` + }](ctx, read, fmt.Sprintf("%s/files?per_page=100&page=%d", endpoint, page), "") + if err != nil || pageFiles == nil { + return proof, errors.New("invalid File Diet implementation file inventory") + } + for _, file := range pageFiles { + files = append(files, fileDietPullFile{file.Filename, file.Status, file.PreviousFilename}) + } + if len(pageFiles) < 100 { + break + } + } + if len(files) != pull.ChangedFiles { + return proof, errors.New("truncated File Diet implementation file inventory") + } + var mergeSHA *string + if pull.MergedAt != nil { + mergeSHA = &pull.MergeCommitSHA + } + proof.Pulls = append(proof.Pulls, fileDietPull{ + Number: number, ID: pull.NodeID, Title: pull.Title, CreatedAt: pull.CreatedAt, + MergedAt: pull.MergedAt, HeadSHA: pull.Head.SHA, BaseSHA: pull.Base.SHA, + MergeSHA: mergeSHA, Graph: graph[number], Files: files, + }) + } + start := fileDietTime(proof.Adoption.At) + if previous != nil { + start = fileDietTime(previous.Until) + } + if previous == nil { + proof.Snapshots = append(proof.Snapshots, fileDietSnapshot{proof.Adoption.At, proof.Adoption.Commit}) + start = time.Date(start.Year(), start.Month()+1, 1, 0, 0, 0, 0, time.UTC) + } else { + start = time.Date(start.Year(), start.Month(), start.Day()+1, 0, 0, 0, 0, time.UTC) + } + cutoffs := []time.Time{} + for start.Before(cutoff) && len(cutoffs) < 4096 { + cutoffs = append(cutoffs, start) + if previous == nil { + start = start.AddDate(0, 1, 0) + } else { + start = start.AddDate(0, 0, 1) + } + } + if len(cutoffs) == 4096 { + return proof, errors.New("file Diet snapshot cadence exceeds bounded collection") + } + cutoffs = append(cutoffs, cutoff) + for _, at := range cutoffs { + instant := at.UTC().Format(time.RFC3339) + before := at.Add(-time.Second).UTC().Format("2006-01-02T15:04:05.000Z") + commits, err := fileDietReadJSON[[]fileDietCommit](ctx, read, + "repos/"+dailyFileDietRepository+"/commits?until="+before+"&per_page=1", "") + if err != nil || len(commits) != 1 || + !fileDietTime(commits[0].Commit.Committer.Date).Before(at) { + return proof, errors.New("missing immutable File Diet snapshot cutoff") + } + proof.Snapshots = append(proof.Snapshots, fileDietSnapshot{instant, commits[0].SHA}) + } + if previous != nil { + snapshots := map[string]fileDietSnapshot{} + for _, row := range previous.Snapshots { + snapshots[row.At] = row + } + for _, row := range proof.Snapshots { + if retained, ok := snapshots[row.At]; ok && retained != row { + return proof, errors.New("preserved File Diet snapshot revision changed") + } + snapshots[row.At] = row + } + proof.Snapshots = proof.Snapshots[:0] + for _, row := range snapshots { + proof.Snapshots = append(proof.Snapshots, row) + } + sort.Slice(proof.Snapshots, func(i, j int) bool { + return proof.Snapshots[i].At < proof.Snapshots[j].At + }) + } + return proof, nil +} diff --git a/server/internal/collect/projector_remote_value_inventory.go b/server/internal/collect/projector_remote_value_inventory.go new file mode 100644 index 0000000000..b14593920c --- /dev/null +++ b/server/internal/collect/projector_remote_value_inventory.go @@ -0,0 +1,283 @@ +package collect + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/githubnext/gh-cao/server/internal/postgresx" +) + +type fileDietMeasurements struct { + inventories map[string]fileDietInventory + newInventories int + newBlobs int + uniqueBlobs int +} + +const fileDietAcquisitionInstrumentSHA = "d7f314ce0e23e2e3a1e5a8f5f059f974a13bf98f737a251ced2b93fee2e89a23" + +func fileDietAcquireInventories( + ctx context.Context, workspace string, proof *fileDietProof, previous *fileDietArchive, +) (fileDietMeasurements, error) { + result := fileDietMeasurements{inventories: map[string]fileDietInventory{}} + blobs := map[string][2]int{} + if previous != nil { + for commit, inventory := range previous.inventories { + result.inventories[commit] = inventory + for _, file := range inventory.Files { + if prior, exists := blobs[file.Blob]; exists && prior != [2]int{file.Lines, file.Bytes} { + return result, errors.New("conflicting preserved File Diet blob measurements") + } + blobs[file.Blob] = [2]int{file.Lines, file.Bytes} + } + } + } + git, err := newNativeFileDietGit(ctx, workspace, dailyFileDietRepository) + if err != nil { + return result, err + } + revisions := map[string]bool{proof.Adoption.BaselineCommit: true, proof.Adoption.Commit: true} + for _, row := range proof.Snapshots { + revisions[row.Commit] = true + } + for _, row := range proof.History { + revisions[row.Commit] = true + } + for _, issue := range proof.Issues { + if issue.Run.Validation != nil { + revisions[issue.Run.Validation.HeadSHA] = true + } + } + for _, pull := range proof.Pulls { + if pull.MergeSHA != nil { + revisions[pull.HeadSHA], revisions[*pull.MergeSHA] = true, true + } + } + missing := make([]string, 0) + for revision := range revisions { + if !validFileDietSHA(revision) { + return result, errors.New("invalid File Diet immutable revision") + } + if _, exists := result.inventories[revision]; !exists { + missing = append(missing, revision) + } + } + if err := git.fetch(ctx, missing, false); err != nil { + return result, err + } + priorPulls := map[string]fileDietPull{} + if previous != nil { + for _, pull := range previous.proof.Pulls { + priorPulls[pull.ID] = pull + } + } + for index := range proof.Pulls { + pull := &proof.Pulls[index] + if pull.MergeSHA == nil || !fileDietTime(*pull.MergedAt).Before(fileDietTime(proof.Until)) { + continue + } + if retained, ok := priorPulls[pull.ID]; ok && retained.MergeSHA != nil && + *retained.MergeSHA == *pull.MergeSHA && retained.PreMergeSHA != "" { + pull.PreMergeSHA = retained.PreMergeSHA + } else { + // A retained merge inventory does not imply the commit object is + // present in this invocation's disposable Git store. + if _, exists := result.inventories[*pull.MergeSHA]; exists { + if err := git.fetch(ctx, []string{*pull.MergeSHA}, false); err != nil { + return result, err + } + } + pull.PreMergeSHA, err = git.firstParent(ctx, *pull.MergeSHA) + if err != nil { + return result, err + } + } + revisions[pull.PreMergeSHA] = true + } + missing = missing[:0] + for revision := range revisions { + if !validFileDietSHA(revision) { + return result, errors.New("invalid File Diet merge parent revision") + } + if _, exists := result.inventories[revision]; !exists { + missing = append(missing, revision) + } + } + if err := git.fetch(ctx, missing, false); err != nil { + return result, err + } + newFiles := map[string][]fileDietFile{} + newBlobs := map[string]bool{} + for _, revision := range missing { + tree, err := git.tree(ctx, revision, "pkg", 50000) + if err != nil { + return result, err + } + files := []fileDietFile{} + paths := map[string]bool{} + for _, entry := range tree { + if entry.kind != "blob" || entry.mode != "100644" && entry.mode != "100755" || + !fileDietEligible(entry.path) { + continue + } + if paths[entry.path] { + return result, errors.New("duplicate native file in File Diet tree") + } + paths[entry.path] = true + files = append(files, fileDietFile{Path: entry.path, Blob: entry.sha}) + if _, exists := blobs[entry.sha]; !exists { + newBlobs[entry.sha] = true + } + } + newFiles[revision] = files + } + if len(blobs)+len(newBlobs) > 50000 { + return result, errors.New("file Diet native blob measurement budget exceeded") + } + newSHA := make([]string, 0, len(newBlobs)) + for blob := range newBlobs { + newSHA = append(newSHA, blob) + } + slices.Sort(newSHA) + for offset := 0; offset < len(newSHA); offset += 50 { + batch := newSHA[offset:min(offset+50, len(newSHA))] + if err := git.fetch(ctx, batch, true); err != nil { + return result, err + } + measured, err := git.measureBlobs(ctx, batch) + if err != nil { + return result, err + } + for sha, count := range measured { + blobs[sha] = count + } + } + result.newBlobs, result.newInventories = len(newBlobs), len(newFiles) + for revision, files := range newFiles { + for index := range files { + count := blobs[files[index].Blob] + files[index].Lines, files[index].Bytes = count[0], count[1] + } + result.inventories[revision] = fileDietInventory{Commit: revision, + Files: files, InventoryDigest: fileDietJSONDigest(files)} + } + allBlobs := map[string]bool{} + for _, inventory := range result.inventories { + for _, file := range inventory.Files { + allBlobs[file.Blob] = true + } + } + result.uniqueBlobs = len(allBlobs) + return result, nil +} + +func fileDietBuildProofFiles( + proof *fileDietProof, measured fileDietMeasurements, previous *fileDietArchive, +) ([]postgresx.ValueProofFile, error) { + // Prefix retained native proof identically to the reviewed Actions collector. + files := map[string][]byte{} + keys := make([]string, 0, len(measured.inventories)) + for commit := range measured.inventories { + keys = append(keys, commit) + } + slices.Sort(keys) + proof.InventoryPartitions = []fileDietPartition{} + for _, commit := range keys { + inventory := measured.inventories[commit] + name := "inventory-" + commit + ".json.gz" + digest := fileDietJSONDigest(inventory) + if previous != nil { + if archive, ok := previous.files[name]; ok { + files[name] = archive + for _, part := range previous.proof.InventoryPartitions { + if part.File == name { + digest = part.SHA256 + break + } + } + } + } + if files[name] == nil { + var err error + files[name], err = fileDietArchiveProof(inventory) + if err != nil { + return nil, err + } + } + proof.InventoryPartitions = append(proof.InventoryPartitions, fileDietPartition{ + File: name, Commit: commit, SHA256: digest, + }) + } + proof.UniqueBlobs = measured.uniqueBlobs + kind := "fresh-github-and-immutable-git" + if previous != nil { + kind = "incremental-github-and-immutable-git" + } + proof.Acquisition = &fileDietAcquisition{Kind: kind, Endpoint: proof.Until, + InstrumentSHA256: fileDietAcquisitionInstrumentSHA, + NewInventories: measured.newInventories, NewBlobs: measured.newBlobs} + proof.ObservedAt = time.Now().UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z") + if previous != nil { + proof.PreviousProof = &fileDietPreviousProof{Digest: fileDietRawDigest(previous.rawProof), + ObservedAt: previous.proof.ObservedAt, Until: previous.proof.Until} + } + proofArchive, err := fileDietArchiveProof(proof) + if err != nil { + return nil, err + } + files["proof.json.gz"] = proofArchive + projection := make([]postgresx.ValueProofFile, 0, len(files)) + var size int64 + for name, content := range files { + size += int64(len(content)) + if len(content) > postgresx.ValueProofFileMax || size > postgresx.ValueProofMaxBytes || + len(files) > postgresx.ValueProofMaxFiles { + return nil, errors.New("file Diet retained proof exceeds PostgreSQL file budget") + } + projection = append(projection, postgresx.ValueProofFile{Path: "report/proof/" + name, Content: content}) + } + slices.SortFunc(projection, func(left, right postgresx.ValueProofFile) int { + return strings.Compare(left.Path, right.Path) + }) + if _, err := fileDietLoadProof(files); err != nil { + return nil, fmt.Errorf("new File Diet proof failed independent replay: %w", err) + } + return projection, nil +} + +// A frozen historical seed is proof of already observed facts, not an +// acquisition endpoint with mutable repository authority. +func fileDietLoadStartingProof(ctx context.Context, root string, restored []postgresx.ValueProofFile) (fileDietArchive, error) { + if len(restored) > 0 { + return fileDietRestore(restored) + } + workspace, err := fileDietGitWorkspace(root) + if err != nil { + return fileDietArchive{}, err + } + defer func() { _ = os.RemoveAll(workspace) }() + seed, err := fileDietSeed(ctx, workspace) + if err != nil { + return fileDietArchive{}, err + } + if fileDietTime(seed.proof.Until).After(time.Now()) { + return fileDietArchive{}, errors.New("historical File Diet seed is in the future") + } + return seed, nil +} + +func fileDietInventoryByRevision(measured fileDietMeasurements, revision string) ([]fileDietFile, error) { + if !validFileDietSHA(revision) { + return nil, errors.New("invalid immutable File Diet inventory revision") + } + inventory, ok := measured.inventories[revision] + if !ok { + return nil, errors.New("missing immutable File Diet inventory") + } + return inventory.Files, nil +} diff --git a/server/internal/collect/projector_remote_value_native.go b/server/internal/collect/projector_remote_value_native.go new file mode 100644 index 0000000000..8bb3a520ee --- /dev/null +++ b/server/internal/collect/projector_remote_value_native.go @@ -0,0 +1,165 @@ +package collect + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + "strings" +) + +const ( + dailyFileDietRepository = "github/gh-aw" + dailyFileDietWorkflow = "daily-file-diet" + dailyFileDietCollector = ".github/value-collectors/daily-file-diet.mjs" +) + +var ( + fileDietRepositoryEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+$`) + fileDietSourceEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+/commits(?:\?path=|/[a-f0-9]+$)`) + fileDietTimelineEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+/issues/[0-9]+/timeline\?`) + fileDietRunEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+/actions/runs/[0-9]+$`) + fileDietArtifactEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+/actions/(?:runs/[0-9]+/artifacts|artifacts/[0-9]+(?:/zip)?)\b`) + fileDietPullEndpoint = regexp.MustCompile(`^repos/[\w.-]+/[\w.-]+/pulls/[0-9]+(?:/files\?)?`) +) + +// These are the frozen requirements from the reviewed Actions collector. The +// identity, pool, source, and primitive must agree before a native read occurs. +var dailyFileDietRequirements = []remoteValueRequirement{ + {"github", "core", "github.repository-metadata.v1", "repository", "invocation"}, + {"github", "core", "github.workflow-source-history.v1", "daily-file-diet", "invocation"}, + {"github", "search", "github.issue-search.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "core", "github.issue-timeline.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "graphql", "github.issue-pull-relationships.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "core", "github.actions-run-provenance.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "core", "github.actions-artifact-receipts.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "core", "github.pull-metadata-files.v1", "daily-file-diet-outputs", "invocation"}, + {"github", "core", "github.repository-commit-history.v1", "daily-file-diet-snapshots", "invocation"}, + {"git", "git", "git.commit-inventory.v1", "daily-file-diet-go-files", "immutable"}, + {"git", "git", "git.blob-measurement.v1", "daily-file-diet-go-files", "immutable"}, +} + +func reviewedDailyFileDiet(entry remoteValueEntry) error { + if entry.Repository != dailyFileDietRepository || entry.Workflow != dailyFileDietWorkflow || + entry.Collector != dailyFileDietCollector || entry.APIRequests < 1 || entry.APIRequests > 1000 || + entry.Acquisition == nil || len(entry.Acquisition.Requirements) != len(dailyFileDietRequirements) { + return errors.New("unsupported reviewed remote Value collector registration") + } + required := make(map[string]bool, len(dailyFileDietRequirements)) + for _, requirement := range dailyFileDietRequirements { + required[remoteValueRequirementKey(requirement)] = true + } + for _, requirement := range entry.Acquisition.Requirements { + key := remoteValueRequirementKey(requirement) + if !required[key] { + return errors.New("unsupported reviewed remote Value acquisition manifest") + } + delete(required, key) + } + if len(required) > 0 { + return errors.New("incomplete reviewed remote Value acquisition manifest") + } + return nil +} + +func fileDietRequirement(endpoint, query string) (remoteValueRequirement, error) { + lookup := func(primitive string) (remoteValueRequirement, error) { + for _, requirement := range dailyFileDietRequirements { + if requirement.Primitive == primitive { + return requirement, nil + } + } + return remoteValueRequirement{}, errors.New("unreviewed File Diet acquisition primitive") + } + switch { + case query != "" || endpoint == "graphql": + if endpoint != "graphql" || query == "" { + break + } + return lookup("github.issue-pull-relationships.v1") + case strings.HasPrefix(endpoint, "search/issues?"): + return lookup("github.issue-search.v1") + case fileDietRepositoryEndpoint.MatchString(endpoint): + return lookup("github.repository-metadata.v1") + case fileDietSourceEndpoint.MatchString(endpoint): + return lookup("github.workflow-source-history.v1") + case strings.HasPrefix(endpoint, "repos/") && strings.Contains(endpoint, "/commits?until="): + return lookup("github.repository-commit-history.v1") + case fileDietTimelineEndpoint.MatchString(endpoint): + return lookup("github.issue-timeline.v1") + case fileDietRunEndpoint.MatchString(endpoint): + return lookup("github.actions-run-provenance.v1") + case fileDietArtifactEndpoint.MatchString(endpoint): + return lookup("github.actions-artifact-receipts.v1") + case fileDietPullEndpoint.MatchString(endpoint): + return lookup("github.pull-metadata-files.v1") + } + return remoteValueRequirement{}, errors.New("file Diet GitHub request has no reviewed acquisition requirement") +} + +type nativeFileDietReader struct { + api *remoteValueAPI + cache map[string][]byte +} + +func (r *nativeFileDietReader) read(ctx context.Context, endpoint, query string, binary bool) ([]byte, error) { + if r.api.authorize != nil { + if err := r.api.authorize(ctx); err != nil { + return nil, err + } + } + requirement, err := fileDietRequirement(endpoint, query) + if err != nil { + return nil, err + } + key := remoteValueRequirementID(requirement) + "\x00" + endpoint + "\x00" + query + if binary { + key += "\x00binary" + } + if value, ok := r.cache[key]; ok { + return bytes.Clone(value), nil + } + var value []byte + if query != "" { + value, err = r.api.graphQL(ctx, query, remoteValueRequirementID(requirement)) + } else { + var status int + value, _, status, err = r.api.request(ctx, endpoint, remoteValueRequirementID(requirement)) + if err == nil && status != http.StatusOK { + if status == http.StatusNotFound && fileDietRunEndpoint.MatchString(endpoint) { + value = []byte(fmt.Sprintf(`{"unavailable":{"status":404,"endpoint":%q,"reason":"Producer run is no longer available"}}`, endpoint)) + } else { + return nil, fmt.Errorf("file Diet GitHub evidence unavailable (HTTP %d)", status) + } + } + } + if err != nil { + return nil, err + } + if !binary { + var payload any + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.UseNumber() + if err := decoder.Decode(&payload); err != nil || payload == nil { + return nil, errors.New("file Diet GitHub response is not complete JSON") + } + if query != "" { + var graph struct { + Data json.RawMessage `json:"data"` + Errors []json.RawMessage `json:"errors"` + } + if err := json.Unmarshal(value, &graph); err != nil || len(graph.Data) == 0 || + bytes.Equal(graph.Data, []byte("null")) || len(graph.Errors) != 0 { + return nil, errors.New("file Diet GraphQL response is incomplete") + } + } + } + if r.cache == nil { + r.cache = map[string][]byte{} + } + r.cache[key] = bytes.Clone(value) + return value, nil +} diff --git a/server/internal/collect/projector_remote_value_native_test.go b/server/internal/collect/projector_remote_value_native_test.go new file mode 100644 index 0000000000..f634febc7d --- /dev/null +++ b/server/internal/collect/projector_remote_value_native_test.go @@ -0,0 +1,567 @@ +package collect + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha1" // #nosec G505 -- test fixture uses Git's SHA-1 object format. + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/postgresx" + "github.com/githubnext/gh-cao/server/internal/testutil" +) + +func testFileDietSHA(character string) string { return strings.Repeat(character, 40) } + +func TestNativeFileDietBlobBatchIsBoundedAndVerified(t *testing.T) { + content := []byte("first\nsecond\n") + sum := sha1.Sum(append([]byte(fmt.Sprintf("blob %d\x00", len(content))), content...)) // #nosec G401 -- validates Git's SHA-1 blob object identifier. + sha := fmt.Sprintf("%x", sum) + packet := append([]byte(fmt.Sprintf("%s blob %d\n", sha, len(content))), content...) + packet = append(packet, '\n') + measured, err := parseFileDietBlobMeasurements(packet, []string{sha}) + if err != nil || measured[sha] != [2]int{2, len(content)} { + t.Fatalf("valid bounded Git blob batch rejected: %v %v", measured, err) + } + for _, invalid := range [][]byte{ + append(bytes.Clone(packet), '\n'), + bytes.Replace(bytes.Clone(packet), []byte("second"), []byte("tamper"), 1), + bytes.Replace(bytes.Clone(packet), []byte(" blob "), []byte(" tree "), 1), + packet[:len(packet)-1], + } { + if _, err := parseFileDietBlobMeasurements(invalid, []string{sha}); err == nil { + t.Fatal("invalid immutable Git blob batch was accepted") + } + } + if _, err := parseFileDietBlobMeasurements(packet, []string{testFileDietSHA("f")}); err == nil { + t.Fatal("unreviewed immutable Git blob identity was accepted") + } +} + +func TestNativeFileDietCadenceRequiresCompleteCoverage(t *testing.T) { + due := fileDietTime("2025-11-17T13:36:21Z") + packet := model.Row{"windowEnd": due.Format(time.RFC3339), "facts": []any{ + map[string]any{"fact-kind": "contract", "cadence-days": 1}, + map[string]any{"fact-kind": "coverage", "coverage-status": "complete", + "at": due.Add(-24 * time.Hour).Format(time.RFC3339), "end-at": due.Format(time.RFC3339)}, + }} + notDue, err := fileDietCadenceSatisfied(packet, due.Add(time.Hour)) + if err != nil || !notDue { + t.Fatalf("complete retained coverage should suppress duplicate acquisition: %v %v", notDue, err) + } + notDue, err = fileDietCadenceSatisfied(packet, due.Add(24*time.Hour+time.Hour)) + if err != nil || notDue { + t.Fatalf("next UTC coverage window was skipped: %v %v", notDue, err) + } + packet["facts"].([]any)[1].(map[string]any)["coverage-status"] = "partial" + notDue, err = fileDietCadenceSatisfied(packet, due.Add(time.Hour)) + if err != nil || notDue { + t.Fatalf("partial coverage suppressed retry: %v %v", notDue, err) + } +} + +func testFileDietReplay() (fileDietProof, fileDietMeasurements) { + adopted := "1186030a620f4113f655c156bedf70cf2c164f79" + parent, merged := testFileDietSHA("2"), testFileDietSHA("3") + at := "2026-01-05T00:00:00Z" + created := "2026-01-02T00:00:00Z" + target := "pkg/a.go" + issue := fileDietIssue{ + Number: 1, ID: "I_1", Title: "[file-diet] Refactor pkg/a.go", CreatedAt: created, + TargetPath: &target, Receipt: &fileDietReceipt{Status: "missing"}, + Graph: fileDietGraph{fileDietTarget: fileDietTarget{ID: "I_1", Number: 1, + Repository: dailyFileDietRepository, URL: "https://github.com/github/gh-aw/issues/1"}, + HistoryComplete: true, Events: []fileDietGraphEvent{}, Edges: []fileDietEdge{}}, + Timeline: fileDietTimeline{Complete: true, Events: []fileDietTimelineEvent{{ + Type: "connected", At: "2026-01-04T00:00:00Z", + Target: &fileDietTimelineTarget{Kind: "pull-request", Number: 2}, + }}}, + } + pull := fileDietPull{ID: "PR_2", Number: 2, Title: "Decompose pkg/a.go", + CreatedAt: "2026-01-03T00:00:00Z", MergedAt: &at, + PreMergeSHA: parent, MergeSHA: &merged, Files: []fileDietPullFile{{ + Path: "pkg/replacement.go", Status: "added", + }}, + Graph: fileDietGraph{fileDietTarget: fileDietTarget{ID: "PR_2", Number: 2, + Repository: dailyFileDietRepository, URL: "https://github.com/github/gh-aw/pull/2"}, + State: "MERGED", MergedAt: &at, MergeCommit: &merged, + ClosingLinksComplete: true, HistoryComplete: true, Edges: []fileDietEdge{}, + Events: []fileDietGraphEvent{}}, + } + mkFile := func(path, blob string, lines int) fileDietFile { + return fileDietFile{Path: path, Blob: testFileDietSHA(blob), Lines: lines, Bytes: lines * 2} + } + inventories := map[string]fileDietInventory{ + adopted: {Commit: adopted, Files: []fileDietFile{mkFile("pkg/a.go", "a", 1129)}}, + parent: {Commit: parent, Files: []fileDietFile{mkFile("pkg/a.go", "a", 1129), mkFile("pkg/unrelated.go", "b", 900)}}, + merged: {Commit: merged, Files: []fileDietFile{mkFile("pkg/a.go", "c", 248), mkFile("pkg/replacement.go", "d", 850)}}, + } + proof := fileDietProof{SchemaVersion: 1, Repository: dailyFileDietRepository, + Aliases: []string{dailyFileDietRepository, "githubnext/gh-aw"}, + SourcePath: fileDietSourcePath, Adoption: fileDietAdoption{Commit: adopted, + BaselineCommit: testFileDietSHA("0"), At: "2025-11-15T13:36:21Z"}, + Until: "2026-05-01T00:00:00Z", ObservedAt: "2026-05-01T01:00:00Z", + Discovery: []fileDietDiscovery{{Complete: true}}, Issues: []fileDietIssue{issue}, + Pulls: []fileDietPull{pull}, Snapshots: []fileDietSnapshot{ + {At: "2025-11-15T13:36:21Z", Commit: adopted}, + {At: "2026-05-01T00:00:00Z", Commit: merged}, + }} + return proof, fileDietMeasurements{inventories: inventories} +} + +func TestNativeFileDietReplayMatchesIndependentActionsOracle(t *testing.T) { + proof, measured := testFileDietReplay() + ledger, err := fileDietReplay(proof, measured) + if err != nil { + t.Fatal(err) + } + packet, err := fileDietExport(ledger) + if err != nil { + t.Fatal(err) + } + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + oracles := []string{ + filepath.Join(root, ".github", "skills", "add-operational-value", "examples", "daily-file-diet.mjs"), + filepath.Join(root, ".github", "skills", "add-operational-value", "scripts", "export-evidence.mjs"), + filepath.Join(root, "dashboard", "site", "src", "data", "model", "operational-value-evidence.js"), + } + input := struct { + Proof fileDietProof `json:"proof"` + Inventories map[string][]fileDietFile `json:"inventories"` + }{Proof: proof, Inventories: map[string][]fileDietFile{}} + for revision, inventory := range measured.inventories { + input.Inventories[revision] = inventory.Files + } + content, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + code := `import { pathToFileURL } from 'node:url'; +const {fileDietEvents} = await import(pathToFileURL(process.argv[2])); +const {exportOperationalValueEvidence} = await import(pathToFileURL(process.argv[3])); +const {validateOperationalValueEvidence} = await import(pathToFileURL(process.argv[4])); +let input = ''; +for await (const chunk of process.stdin) input += chunk; +const {proof, inventories} = JSON.parse(input); +process.stdout.write(JSON.stringify(validateOperationalValueEvidence(exportOperationalValueEvidence(fileDietEvents({ + proof, inventories: new Map(Object.entries(inventories)) +})))));` + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, "oracle", oracles[0], oracles[1], oracles[2]) // #nosec G204 -- fixed checked-in independent fixture implementation. + command.Stdin = bytes.NewReader(content) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("Actions oracle rejected the same proof: %v: %.1000s", err, output) + } + var expected model.Row + if err := json.Unmarshal(output, &expected); err != nil { + t.Fatal(err) + } + normalize := func(row model.Row) []byte { + encoded, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + return encoded + } + if !bytes.Equal(normalize(packet), normalize(expected)) { + gotFacts, wantFacts := packet["facts"].([]any), expected["facts"].([]any) + t.Fatalf("File Diet Go proof replay differs from Actions oracle: digest %v / %v, facts %d / %d; first differing fact %s", + packet["contractDigest"], expected["contractDigest"], len(gotFacts), len(wantFacts), + firstFileDietDifference(gotFacts, wantFacts)) + } +} + +func firstFileDietDifference(left, right []any) string { + for i := 0; i < min(len(left), len(right)); i++ { + a, _ := json.Marshal(left[i]) + b, _ := json.Marshal(right[i]) + if !bytes.Equal(a, b) { + return string(a) + " vs " + string(b) + } + } + return "no differing fact" +} + +func TestNativeFileDietRequiresFrozenManifestAndScope(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + instrument, err := os.ReadFile(filepath.Join(root, ".github", "skills", "add-operational-value", + "examples", "acquire-file-diet.mjs")) // #nosec G304 -- reads a fixed reviewed source under the repository root. + if err != nil || fileDietRawBytesDigest(instrument) != fileDietAcquisitionInstrumentSHA { + t.Fatalf("reviewed acquisition instrument changed without the native equivalent: %v", err) + } + collector := filepath.Join(root, dailyFileDietCollector) + code := `import { pathToFileURL } from 'node:url'; +const {definition} = await import(pathToFileURL(process.argv[2])); +process.stdout.write(JSON.stringify(definition.acquisition));` + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, "oracle", collector) // #nosec G204 -- fixed reviewed collector fixture. + content, err := command.CombinedOutput() + if err != nil { + t.Fatalf("read frozen Actions collector manifest: %v: %.300s", err, content) + } + var reviewed remoteValueAcquisition + if err := json.Unmarshal(content, &reviewed); err != nil { + t.Fatal(err) + } + if len(reviewed.Requirements) != len(dailyFileDietRequirements) { + t.Fatalf("reviewed Actions manifest changed without native implementation: %+v", reviewed) + } + required := map[string]bool{} + for _, requirement := range dailyFileDietRequirements { + required[remoteValueRequirementKey(requirement)] = true + } + for _, requirement := range reviewed.Requirements { + if !required[remoteValueRequirementKey(requirement)] { + t.Fatalf("Actions collector primitive lacks native implementation: %+v", requirement) + } + delete(required, remoteValueRequirementKey(requirement)) + } + if len(required) > 0 { + t.Fatalf("native frozen acquisition diverged from Actions: %+v", required) + } + entry := remoteValueEntry{Repository: dailyFileDietRepository, Workflow: dailyFileDietWorkflow, + Collector: dailyFileDietCollector, APIRequests: 700, + Acquisition: &remoteValueAcquisition{Requirements: append([]remoteValueRequirement(nil), dailyFileDietRequirements...)}} + if err := reviewedDailyFileDiet(entry); err != nil { + t.Fatal(err) + } + for _, invalid := range []remoteValueEntry{ + {Repository: "other/gh-aw", Workflow: entry.Workflow, Collector: entry.Collector, + APIRequests: entry.APIRequests, Acquisition: entry.Acquisition}, + {Repository: entry.Repository, Workflow: entry.Workflow, Collector: ".github/value-collectors/unreviewed.mjs", + APIRequests: entry.APIRequests, Acquisition: entry.Acquisition}, + {Repository: entry.Repository, Workflow: entry.Workflow, Collector: entry.Collector, + APIRequests: 0, Acquisition: entry.Acquisition}, + } { + if err := reviewedDailyFileDiet(invalid); err == nil { + t.Fatalf("unreviewed File Diet registration accepted: %+v", invalid) + } + } + broken := entry + broken.Acquisition = &remoteValueAcquisition{Requirements: append([]remoteValueRequirement(nil), dailyFileDietRequirements...)} + broken.Acquisition.Requirements[len(broken.Acquisition.Requirements)-1].Pool = "core" + if err := reviewedDailyFileDiet(broken); err == nil { + t.Fatal("git blob measurement was allowed to borrow GitHub core quota") + } + valid, rejected := supportedRemoteValueEntries([]remoteValueEntry{broken, entry}) + if rejected == nil || len(valid) != 1 || valid[0].Workflow != dailyFileDietWorkflow { + t.Fatalf("unsupported registration suppressed later reviewed collector: %+v %v", valid, rejected) + } + for _, request := range []string{"repos/github/gh-aw/issues/1/comments", "search/repositories?q=github"} { + if _, err := fileDietRequirement(request, ""); err == nil { + t.Fatalf("out-of-manifest File Diet endpoint admitted: %q", request) + } + } +} + +func TestNativeFileDietProofArchiveRejectsTamper(t *testing.T) { + proof, measured := testFileDietReplay() + proof.InventoryPartitions = []fileDietPartition{} + proof.UniqueBlobs = 4 + files := map[string][]byte{} + for _, revision := range []string{proof.Adoption.Commit, proof.Pulls[0].PreMergeSHA, *proof.Pulls[0].MergeSHA} { + inventory := measured.inventories[revision] + inventory.InventoryDigest = fileDietJSONDigest(inventory.Files) + name := "inventory-" + revision + ".json.gz" + proof.InventoryPartitions = append(proof.InventoryPartitions, fileDietPartition{ + File: name, Commit: revision, SHA256: fileDietJSONDigest(inventory), + }) + var err error + files[name], err = fileDietArchiveProof(inventory) + if err != nil { + t.Fatal(err) + } + } + var err error + files["proof.json.gz"], err = fileDietArchiveProof(proof) + if err != nil { + t.Fatal(err) + } + if _, err := fileDietLoadProof(files); err != nil { + t.Fatalf("valid bounded File Diet proof rejected: %v", err) + } + changed := map[string][]byte{} + for key, value := range files { + changed[key] = bytes.Clone(value) + } + var plain bytes.Buffer + writer := gzip.NewWriter(&plain) + _, _ = writer.Write([]byte(`{"commit":"wrong","files":[],"inventoryDigest":"broken"}`)) + _ = writer.Close() + changed[proof.InventoryPartitions[0].File] = plain.Bytes() + if _, err := fileDietLoadProof(changed); err == nil { + t.Fatal("tampered accepted File Diet inventory replayed") + } +} + +func TestNativeFileDietGitScopeAndNoAmbientCredentials(t *testing.T) { + root, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + workspace, err := fileDietGitWorkspace(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workspace) }) + if _, err := newNativeFileDietGit(context.Background(), workspace, "another/private-repository"); err == nil { + t.Fatal("unreviewed Git target admitted") + } + git, err := newNativeFileDietGit(t.Context(), workspace, dailyFileDietRepository) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(git.dir, "HEAD")); err != nil { + t.Fatalf("private immutable Git workspace not initialized: %v", err) + } + if err := git.fetch(t.Context(), []string{"not-an-immutable-commit"}, false); err == nil { + t.Fatal("unreviewed immutable Git object identity was fetched") + } + if _, err := parseFileDietTree([]byte("120000 blob "+testFileDietSHA("a")+"\tpkg/link.go\x00"), "docs/", 1); err == nil { + t.Fatal("out-of-scope immutable Git tree admitted") + } + if _, err := fileDietReferenceFor(""); !errors.Is(err, errFileDietReferenceNotFound) { + t.Fatalf("unrelated tracking marker was not treated as absent: %v", err) + } +} + +func TestNativeFileDietLegacyRunReferenceRequiresExactGitHubURL(t *testing.T) { + valid := "" + reference, err := fileDietReferenceFor(valid) + if err != nil || reference == nil || *reference.RunID != "42" { + t.Fatalf("valid legacy File Diet run reference rejected: reference=%+v err=%v", reference, err) + } + + for _, runURL := range []string{ + "https://attacker.example/https://github.com/github/gh-aw/actions/runs/42", + "https://github.com/github/gh-aw/actions/runs/42.attacker.example", + } { + body := "" + if _, err := fileDietReferenceFor(body); err == nil { + t.Errorf("accepted a non-exact legacy run URL: %q", runURL) + } + } +} + +func TestNativeFileDietHistoricalSeed(t *testing.T) { + if os.Getenv("CAO_TEST_REMOTE_VALUE_GIT") != "1" { + t.Skip("set CAO_TEST_REMOTE_VALUE_GIT=1 to fetch the frozen public historical proof") + } + root, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + workspace, err := fileDietGitWorkspace(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workspace) }) + seed, err := fileDietSeed(t.Context(), workspace) + if err != nil { + t.Fatal(err) + } + if seed.proof.Adoption.Commit != "1186030a620f4113f655c156bedf70cf2c164f79" || + len(seed.inventories) == 0 || seed.proof.UniqueBlobs == 0 { + t.Fatal("frozen historical Git seed lacked native proof") + } + ledger, err := fileDietReplay(seed.proof, fileDietMeasurements{inventories: seed.inventories}) + if err != nil { + t.Fatalf("frozen seed failed native replay: %v", err) + } + if _, err := fileDietExport(ledger); err != nil { + t.Fatalf("frozen seed failed native evidence export: %v", err) + } +} + +func TestNativeFileDietNewPrivateProofReplaysInActionsRuntime(t *testing.T) { + proof, measured := testFileDietReplay() + measured.uniqueBlobs = 4 + original := map[string][]byte{} + for revision, inventory := range measured.inventories { + inventory.InventoryDigest = fileDietJSONDigest(inventory.Files) + name := "inventory-" + revision + ".json.gz" + proof.InventoryPartitions = append(proof.InventoryPartitions, fileDietPartition{ + File: name, Commit: revision, SHA256: fileDietJSONDigest(inventory), + }) + content, err := fileDietArchiveProof(inventory) + if err != nil { + t.Fatal(err) + } + original[name] = content + } + proof.UniqueBlobs = 4 + var err error + original["proof.json.gz"], err = fileDietArchiveProof(proof) + if err != nil { + t.Fatal(err) + } + prior, err := fileDietLoadProof(original) + if err != nil { + t.Fatal(err) + } + next := prior.proof + next.Until = "2026-05-02T00:00:00Z" + next.Snapshots = append(next.Snapshots, fileDietSnapshot{At: next.Until, Commit: *next.Pulls[0].MergeSHA}) + proofFiles, err := fileDietBuildProofFiles(&next, measured, &prior) + if err != nil { + t.Fatalf("new native proof rejected on private re-read: %v", err) + } + if _, err := fileDietRestore(proofFiles); err != nil { + t.Fatalf("native proof did not survive PostgreSQL file shape: %v", err) + } + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + workspace, err := fileDietGitWorkspace(root) // private project-owned scratch, not system temp + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workspace) }) + for _, file := range proofFiles { + path := filepath.Join(workspace, strings.TrimPrefix(file.Path, "report/proof/")) + if err := os.WriteFile(path, file.Content, 0o600); err != nil { + t.Fatal(err) + } + } + source := filepath.Join(root, ".github", "skills", "add-operational-value", + "examples", "daily-file-diet.mjs") + code := `import { pathToFileURL } from 'node:url'; +const {readFileDietProof} = await import(pathToFileURL(process.argv[2])); +process.stdout.write(readFileDietProof(process.argv[3]).proof.until);` + command := exec.CommandContext(t.Context(), "node", "--input-type=module", "-e", code, "oracle", source, workspace) // #nosec G204 -- Actions oracle reads only test-owned proof bytes. + output, err := command.CombinedOutput() + if err != nil || string(output) != next.Until { + t.Fatalf("Actions reader rejected native private proof: %.400s %v", output, err) + } +} + +func TestNativeFileDietIncrementalAcquisitionIsBoundedAndFailClosed(t *testing.T) { + previous, _ := testFileDietReplay() + previous.History = []fileDietHistoryRevision{{Commit: previous.Adoption.Commit, At: previous.Adoption.At}} + until := "2026-05-02T00:00:00Z" + reads := []string{} + read := func(_ context.Context, endpoint, query string, binary bool) ([]byte, error) { + if binary { + return nil, fmt.Errorf("unreviewed File Diet binary acquisition %s", endpoint) + } + reads = append(reads, endpoint) + switch { + case endpoint == "repos/github/gh-aw" || endpoint == "repos/githubnext/gh-aw": + return []byte(`{"id":42,"full_name":"github/gh-aw"}`), nil + case strings.HasPrefix(endpoint, "repos/github/gh-aw/commits?path="): + return []byte(`[]`), nil + case strings.HasPrefix(endpoint, "search/issues?"): + return []byte(`{"incomplete_results":false,"total_count":0,"items":[]}`), nil + case endpoint == "graphql": + if query != remoteValueExpectedGraphQuery(dailyFileDietRepository, []int{1}) { + return nil, fmt.Errorf("unexpected File Diet graph query") + } + return []byte(`{"data":{"repository":{"n1":{ + "__typename":"Issue","id":"I_1","number":1,"url":"https://github.com/github/gh-aw/issues/1", + "repository":{"nameWithOwner":"github/gh-aw"},"state":"OPEN", + "createdAt":"2026-01-02T00:00:00Z","updatedAt":"2026-05-01T00:00:00Z", + "timelineItems":{"nodes":[],"pageInfo":{"hasNextPage":false}} + }}}}`), nil + case strings.Contains(endpoint, "/issues/1/timeline?"): + return []byte(`[]`), nil + case strings.HasPrefix(endpoint, "repos/github/gh-aw/commits?until="): + return []byte(`[{"sha":"` + testFileDietSHA("3") + + `","commit":{"committer":{"date":"2026-05-01T00:00:00Z"}}}]`), nil + } + return nil, fmt.Errorf("unreviewed File Diet read %s", endpoint) + } + acquired, err := fileDietAcquireHistory(t.Context(), read, until, &previous) + if err != nil || len(acquired.Issues) != 1 || len(acquired.Pulls) != 1 || + len(acquired.Snapshots) != 3 || acquired.Snapshots[2].At != until || + acquired.Adoption != previous.Adoption { + t.Fatalf("incremental File Diet proof = %+v, %v", acquired, err) + } + for _, endpoint := range reads { + if strings.HasPrefix(endpoint, "search/issues?") && + !strings.Contains(endpoint, "updated%3A2026-05-01..2026-05-02") { + t.Fatalf("File Diet discovery exceeded prior cutoff: %s", endpoint) + } + } + if len(reads) > 16 { + t.Fatalf("one-day File Diet refresh made unbounded requests: %d", len(reads)) + } + denySearch := func(ctx context.Context, endpoint, query string, binary bool) ([]byte, error) { + if strings.HasPrefix(endpoint, "search/issues?") { + return []byte(`{"incomplete_results":true,"total_count":0,"items":[]}`), nil + } + return read(ctx, endpoint, query, binary) + } + if _, err := fileDietAcquireHistory(t.Context(), denySearch, until, &previous); err == nil { + t.Fatal("incomplete File Diet issue discovery was accepted") + } + denyAlias := func(ctx context.Context, endpoint, query string, binary bool) ([]byte, error) { + if endpoint == "repos/githubnext/gh-aw" { + return []byte(`{"id":7,"full_name":"other/repository"}`), nil + } + return read(ctx, endpoint, query, binary) + } + if _, err := fileDietAcquireHistory(t.Context(), denyAlias, until, &previous); err == nil { + t.Fatal("cross-repository File Diet alias was accepted") + } +} + +func TestNativeFileDietEvidenceAcceptedByProductionPostgres(t *testing.T) { + data := testutil.Postgres(t, t.Context(), "CAO_TEST_POSTGRES_URL", "POSTGRES_URL") + proof, measured := testFileDietReplay() + ledger, err := fileDietReplay(proof, measured) + if err != nil { + t.Fatal(err) + } + packet, err := fileDietExport(ledger) + if err != nil { + t.Fatal(err) + } + writer, err := data.BeginIngestion(t.Context()) + if err != nil { + t.Fatal(err) + } + defer writer.Abort(t.Context()) + repositoryID := "repository:" + model.EncodeCoordinate(dailyFileDietRepository) + workflowID := "workflow:" + model.EncodeCoordinate(dailyFileDietRepository+":"+fileDietSourcePath) + for source, row := range map[string]model.Row{ + "$repositories": {"id": repositoryID, "owner": "github", "name": "gh-aw"}, + "$workflows": {"id": workflowID, "repositoryId": repositoryID, "path": fileDietSourcePath}, + } { + if err := writer.Append(t.Context(), source, row); err != nil { + t.Fatal(err) + } + } + if _, err := writer.Publish(t.Context(), "file-diet-production-parent"); err != nil { + t.Fatal(err) + } + accepted, err := data.PublishOperationalValueWithProof(t.Context(), packet, + dailyFileDietRepository, dailyFileDietWorkflow, []postgresx.ValueProofFile{{ + Path: "report/proof/proof.json.gz", Content: []byte("frozen-private-proof"), + }}, time.Now()) + if err != nil || !accepted.Updated { + t.Fatalf("native File Diet packet rejected by production publication: %+v %v", accepted, err) + } + retained, err := data.ReadOperationalValue(t.Context(), dailyFileDietRepository, dailyFileDietWorkflow) + if err != nil || retained["contractDigest"] != packet["contractDigest"] { + t.Fatalf("accepted File Diet evidence not retained: %v %v", retained, err) + } + files, err := data.ReadValueProof(t.Context(), dailyFileDietRepository, dailyFileDietWorkflow) + if err != nil || len(files) != 1 || string(files[0].Content) != "frozen-private-proof" { + t.Fatalf("accepted private File Diet proof not retained: %v %v", files, err) + } +} diff --git a/server/internal/collect/projector_remote_value_plan.go b/server/internal/collect/projector_remote_value_plan.go index 766f17c626..c94e3b2223 100644 --- a/server/internal/collect/projector_remote_value_plan.go +++ b/server/internal/collect/projector_remote_value_plan.go @@ -11,7 +11,6 @@ import ( "slices" "strings" "sync" - "time" "github.com/githubnext/gh-cao/server/internal/operationalvalue" ) @@ -126,10 +125,7 @@ func (p Projector) remoteValuePlan(ctx context.Context) ([]remoteValueEntry, [sh if p.PolicyPath == "" || p.CatalogRoot == "" { return nil, zero, errors.New("reviewed remote Value policy and catalog are required") } - config := operationalvalue.Config{ - Root: p.CatalogRoot, NodeBinary: p.node(), - Environment: collectionEnvironment(p.GitHubAPIURL), WorkerTimeout: 30 * time.Second, - } + config := operationalvalue.Config{Root: p.CatalogRoot} var content []byte var registrations []operationalvalue.WorkflowRegistration var err error @@ -205,8 +201,7 @@ func (p Projector) remoteValuePlan(ctx context.Context) ([]remoteValueEntry, [sh entries = append(entries, entry) } } - // A policy edit during the runtime subprocess must not cache the plan - // under a digest belonging to the older policy. + // A policy edit during resolution must not cache the older policy digest. var current []byte if native { var snapshot operationalvalue.WorkflowPolicySnapshot @@ -229,7 +224,7 @@ func (p Projector) remoteValuePlan(ctx context.Context) ([]remoteValueEntry, [sh // RemoteValueRepositories returns only reviewed, enabled, API-bound remote // registrations that remain enrolled. It never acquires an App token or calls -// GitHub, and the policy runtime subprocess runs only when policy bytes change. +// GitHub; policy resolution does not invoke a subprocess. func (p Projector) RemoteValueRepositories(ctx context.Context) ([]string, error) { targets, err := p.RemoteValueTargets(ctx) if err != nil { diff --git a/server/internal/collect/projector_remote_value_proof.go b/server/internal/collect/projector_remote_value_proof.go new file mode 100644 index 0000000000..e2aa499d0f --- /dev/null +++ b/server/internal/collect/projector_remote_value_proof.go @@ -0,0 +1,461 @@ +package collect + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strings" + "time" + + "github.com/githubnext/gh-cao/server/internal/postgresx" +) + +const fileDietProofLimit = 16 << 20 + +var fileDietInventoryName = regexp.MustCompile(`^inventory-[a-f0-9]{40}\.json\.gz$`) + +type fileDietAdoption struct { + Commit string `json:"commit"` + At string `json:"at"` + BaselineCommit string `json:"baselineCommit"` +} +type fileDietSnapshot struct { + At string `json:"at"` + Commit string `json:"commit"` +} +type fileDietHistoryRevision struct { + Commit string `json:"commit"` + At string `json:"at"` +} +type fileDietDiscovery struct { + Query string `json:"query"` + TotalCount int `json:"totalCount"` + Complete bool `json:"complete"` +} +type fileDietTarget struct { + ID string `json:"id"` + Number int `json:"number"` + Repository string `json:"repository"` + URL string `json:"url"` +} +type fileDietEdge struct { + Relation string `json:"relation"` + At string `json:"at,omitempty"` + Target fileDietTarget `json:"target"` +} +type fileDietGraphEvent struct { + Type string `json:"type"` + At string `json:"at"` + CloserID string `json:"closerId,omitempty"` +} +type fileDietGraph struct { + fileDietTarget + Kind string `json:"kind"` + State string `json:"state"` + StateReason *string `json:"stateReason"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + ClosedAt *string `json:"closedAt"` + MergedAt *string `json:"mergedAt"` + MergeCommit *string `json:"mergeCommit"` + ObservedAt string `json:"observedAt"` + HistoryComplete bool `json:"historyComplete"` + ClosingLinksComplete bool `json:"closingLinksComplete"` + Edges []fileDietEdge `json:"edges"` + Events []fileDietGraphEvent `json:"events"` +} +type fileDietTimelineTarget struct { + Number int `json:"number"` + Title string `json:"title"` + Kind string `json:"kind"` +} +type fileDietTimelineEvent struct { + Type string `json:"type"` + At string `json:"at"` + Target *fileDietTimelineTarget `json:"target,omitempty"` +} +type fileDietTimeline struct { + Complete bool `json:"complete"` + Events []fileDietTimelineEvent `json:"events"` +} +type fileDietReference struct { + Repository string `json:"repository"` + RunID *string `json:"runId"` + Attempt *int `json:"attempt"` + Marker string `json:"marker"` + WorkflowID string `json:"workflowId"` + TrackerID *string `json:"trackerId"` +} +type fileDietValidation struct { + Status string `json:"status"` + HeadSHA string `json:"headSha"` + Attempt int `json:"attempt"` + SourcePath string `json:"sourcePath"` + CreatedAt string `json:"createdAt"` + URL string `json:"url"` + Limitation string `json:"limitation"` +} +type fileDietRun struct { + Validation *fileDietValidation `json:"validation"` + Unavailable json.RawMessage `json:"unavailable"` +} +type fileDietReceipt struct { + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Content string `json:"content,omitempty"` + EntrySHA256 string `json:"entrySha256,omitempty"` + SourceEntrySHA256 string `json:"sourceEntrySha256,omitempty"` + SourceRowIndex *int `json:"sourceRowIndex,omitempty"` + RunID int64 `json:"runId,omitempty"` + HeadSHA string `json:"headSha,omitempty"` + ArtifactID int64 `json:"artifactId,omitempty"` + Digest string `json:"digest,omitempty"` + ArchiveSHA256 string `json:"archiveSha256,omitempty"` + ObservedAt string `json:"observedAt,omitempty"` + ReusedFromPreservedProof bool `json:"reusedFromPreservedProof,omitempty"` +} +type fileDietIssue struct { + Number int `json:"number"` + ID string `json:"id"` + Title string `json:"title"` + CreatedAt string `json:"createdAt"` + Eligible *bool `json:"eligible,omitempty"` + BodySHA256 string `json:"bodySha256"` + TargetPath *string `json:"targetPath"` + Reference fileDietReference `json:"reference"` + Run fileDietRun `json:"run"` + RetainedArtifacts json.RawMessage `json:"retainedArtifacts"` + Graph fileDietGraph `json:"graph"` + Timeline fileDietTimeline `json:"timeline"` + Receipt *fileDietReceipt `json:"receipt,omitempty"` +} +type fileDietPullFile struct { + Path string `json:"path"` + Status string `json:"status"` + PreviousPath *string `json:"previousPath"` +} +type fileDietPull struct { + Number int `json:"number"` + ID string `json:"id"` + Title string `json:"title"` + CreatedAt string `json:"createdAt"` + MergedAt *string `json:"mergedAt"` + HeadSHA string `json:"headSha"` + BaseSHA string `json:"baseSha"` + MergeSHA *string `json:"mergeSha"` + Graph fileDietGraph `json:"graph"` + Files []fileDietPullFile `json:"files"` + PreMergeSHA string `json:"preMergeSha,omitempty"` +} +type fileDietPreviousProof struct { + Digest string `json:"digest"` + ObservedAt string `json:"observedAt"` + Until string `json:"until"` +} +type fileDietPartition struct { + File string `json:"file"` + Commit string `json:"commit"` + SHA256 string `json:"sha256"` +} +type fileDietAcquisition struct { + Kind string `json:"kind"` + Endpoint string `json:"endpoint"` + InstrumentSHA256 string `json:"instrumentSha256"` + NewInventories int `json:"newInventories"` + NewBlobs int `json:"newBlobs"` +} +type fileDietProof struct { + SchemaVersion int `json:"schemaVersion"` + Repository string `json:"repository"` + RepositoryID int64 `json:"repositoryId"` + Aliases []string `json:"aliases"` + SourcePath string `json:"sourcePath"` + Adoption fileDietAdoption `json:"adoption"` + Until string `json:"until"` + ObservedAt string `json:"observedAt"` + Discovery []fileDietDiscovery `json:"discovery"` + Issues []fileDietIssue `json:"issues"` + Pulls []fileDietPull `json:"pulls"` + Snapshots []fileDietSnapshot `json:"snapshots"` + History []fileDietHistoryRevision `json:"history"` + PreviousProof *fileDietPreviousProof `json:"previousProof,omitempty"` + InventoryPartitions []fileDietPartition `json:"inventoryPartitions"` + UniqueBlobs int `json:"uniqueBlobs"` + Acquisition *fileDietAcquisition `json:"acquisition,omitempty"` +} +type fileDietFile struct { + Path string `json:"path"` + Blob string `json:"blob"` + Lines int `json:"lines"` + Bytes int `json:"bytes"` +} +type fileDietInventory struct { + Commit string `json:"commit"` + Files []fileDietFile `json:"files"` + InventoryDigest string `json:"inventoryDigest"` +} +type fileDietArchive struct { + proof fileDietProof + inventories map[string]fileDietInventory + rawProof []byte + files map[string][]byte +} + +func fileDietJSONDigest(value any) string { + return fileDietDigest(value) +} + +func fileDietRawDigest(value []byte) string { + var compact bytes.Buffer + if json.Compact(&compact, value) != nil { + return "" + } + return fmt.Sprintf("%x", sha256.Sum256(compact.Bytes())) +} + +func fileDietReadArchive(files map[string][]byte, filename string) ([]byte, error) { + decoded, err := fileDietReadGzip(files, filename) + if err != nil { + return nil, err + } + var partition struct { + Kind string `json:"kind"` + SchemaVersion int `json:"schemaVersion"` + Encoding string `json:"encoding"` + Parts []struct { + File string `json:"file"` + SHA256 string `json:"sha256"` + } `json:"parts"` + SHA256 string `json:"sha256"` + } + if err := json.Unmarshal(decoded, &partition); err != nil { + return nil, err + } + if partition.Kind != "partitioned-census-json" { + return decoded, nil + } + if partition.SchemaVersion != 1 || partition.Encoding != "base64-json-utf8" || + len(partition.Parts) == 0 || len(partition.Parts) > 128 { + return nil, errors.New("invalid partitioned File Diet archive") + } + var joined []byte + for _, part := range partition.Parts { + if !strings.HasSuffix(part.File, ".json.gz") || strings.Contains(part.File, "/") || + strings.Contains(part.File, "\\") || strings.Contains(part.File, "..") || + part.File == filename { + return nil, errors.New("unsafe partitioned File Diet archive path") + } + encoded, err := fileDietReadGzip(files, part.File) + if err != nil { + return nil, err + } + var text string + if err := json.Unmarshal(encoded, &text); err != nil { + return nil, err + } + payload, err := base64.StdEncoding.Strict().DecodeString(text) + if err != nil || fileDietRawBytesDigest(payload) != part.SHA256 || + len(joined)+len(payload) > fileDietProofLimit { + return nil, errors.New("partitioned File Diet archive digest or size mismatch") + } + joined = append(joined, payload...) + } + if fileDietRawBytesDigest(joined) != partition.SHA256 { + return nil, errors.New("partitioned File Diet archive digest mismatch") + } + return joined, nil +} + +func fileDietReadGzip(files map[string][]byte, filename string) ([]byte, error) { + raw, ok := files[filename] + if !ok || len(raw) > fileDietProofLimit { + return nil, errors.New("missing or oversized File Diet archive") + } + reader, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, err + } + defer func() { _ = reader.Close() }() + decoded, err := io.ReadAll(io.LimitReader(reader, fileDietProofLimit+1)) + if err != nil || len(decoded) > fileDietProofLimit { + return nil, errors.New("file Diet archive exceeds its decoded size budget") + } + return decoded, nil +} + +func fileDietRawBytesDigest(content []byte) string { + return fmt.Sprintf("%x", sha256.Sum256(content)) +} + +func fileDietLoadProof(files map[string][]byte) (fileDietArchive, error) { + var result fileDietArchive + proofRaw, err := fileDietReadArchive(files, "proof.json.gz") + if err != nil { + return result, err + } + if err := json.Unmarshal(proofRaw, &result.proof); err != nil { + return result, err + } + proof := result.proof + if proof.SchemaVersion != 1 || proof.Repository != dailyFileDietRepository || + proof.SourcePath != ".github/workflows/daily-file-diet.md" || + !slices.Equal(proof.Aliases, []string{dailyFileDietRepository, "githubnext/gh-aw"}) || + !validFileDietSHA(proof.Adoption.Commit) || !validFileDietSHA(proof.Adoption.BaselineCommit) || + proof.Adoption.Commit != "1186030a620f4113f655c156bedf70cf2c164f79" || + proof.Adoption.At != "2025-11-15T13:36:21Z" || + !fileDietTime(proof.Adoption.At).Before(fileDietTime(proof.Until)) || + fileDietTime(proof.ObservedAt).Before(fileDietTime(proof.Until)) { + return result, errors.New("file Diet proof identity or frozen adoption changed") + } + issues, issueNumbers, pulls, pullNumbers := map[string]bool{}, map[int]bool{}, map[string]bool{}, map[int]bool{} + for _, issue := range proof.Issues { + if issue.ID == "" || issue.Number <= 0 || issues[issue.ID] || issueNumbers[issue.Number] { + return result, errors.New("duplicate File Diet output identity") + } + issues[issue.ID], issueNumbers[issue.Number] = true, true + } + for _, pull := range proof.Pulls { + if pull.ID == "" || pull.Number <= 0 || pulls[pull.ID] || pullNumbers[pull.Number] { + return result, errors.New("duplicate File Diet implementation identity") + } + pulls[pull.ID], pullNumbers[pull.Number] = true, true + } + result.inventories = make(map[string]fileDietInventory, len(proof.InventoryPartitions)) + measurements := map[string][2]int{} + for _, partition := range proof.InventoryPartitions { + if !fileDietInventoryName.MatchString(partition.File) || + partition.File != "inventory-"+partition.Commit+".json.gz" || + !validFileDietSHA(partition.Commit) { + return result, errors.New("invalid File Diet inventory identity") + } + raw, err := fileDietReadArchive(files, partition.File) + if err != nil { + return result, err + } + var envelope struct { + Commit string `json:"commit"` + Files json.RawMessage `json:"files"` + InventoryDigest string `json:"inventoryDigest"` + } + if err := json.Unmarshal(raw, &envelope); err != nil || + fileDietRawDigest(raw) != partition.SHA256 || + fileDietRawDigest(envelope.Files) != envelope.InventoryDigest || + envelope.Commit != partition.Commit { + return result, errors.New("file Diet inventory partition digest mismatch") + } + var inventory fileDietInventory + if err := json.Unmarshal(raw, &inventory); err != nil { + return result, err + } + paths := map[string]bool{} + for _, file := range inventory.Files { + count := [2]int{file.Lines, file.Bytes} + if paths[file.Path] || !fileDietEligible(file.Path) || !validFileDietSHA(file.Blob) || + file.Lines < 0 || file.Bytes < file.Lines { + return result, errors.New("invalid File Diet native file inventory") + } + if previous, ok := measurements[file.Blob]; ok && previous != count { + return result, errors.New("conflicting File Diet blob measurements") + } + measurements[file.Blob], paths[file.Path] = count, true + } + if _, ok := result.inventories[partition.Commit]; ok { + return result, errors.New("duplicate File Diet inventory revision") + } + result.inventories[partition.Commit] = inventory + } + if len(measurements) != proof.UniqueBlobs { + return result, errors.New("file Diet unique blob count changed") + } + result.rawProof, result.files = proofRaw, files + return result, nil +} + +func fileDietEligible(path string) bool { + return strings.HasPrefix(path, "pkg/") && strings.HasSuffix(path, ".go") && + !strings.HasSuffix(path, "_test.go") +} + +func fileDietTime(text string) time.Time { + at, _ := time.Parse(time.RFC3339Nano, text) + return at +} + +func fileDietSeed(ctx context.Context, workspace string) (fileDietArchive, error) { + git, err := newNativeFileDietGit(ctx, workspace, "githubnext/gh-cao") + if err != nil { + return fileDietArchive{}, err + } + if err := git.fetch(ctx, []string{fileDietHistoricalRevision}, false); err != nil { + return fileDietArchive{}, err + } + tree, err := git.tree(ctx, fileDietHistoricalRevision, fileDietHistoricalPath, 1024) + if err != nil { + return fileDietArchive{}, err + } + files := map[string][]byte{} + var total int + for _, entry := range tree { + name := strings.TrimPrefix(entry.path, fileDietHistoricalPath+"/") + if entry.mode != "100644" || entry.kind != "blob" || + name != "proof.json.gz" && !fileDietInventoryName.MatchString(name) || + len(files) >= postgresx.ValueProofMaxFiles { + return fileDietArchive{}, errors.New("historical File Diet proof contains unreviewed content") + } + if err := git.fetch(ctx, []string{entry.sha}, true); err != nil { + return fileDietArchive{}, err + } + content, err := git.blob(ctx, entry.sha, fileDietProofLimit-total) + if err != nil { + return fileDietArchive{}, err + } + total += len(content) + if total > fileDietProofLimit { + return fileDietArchive{}, errors.New("historical File Diet proof exceeds its byte budget") + } + files[name] = content + } + return fileDietLoadProof(files) +} + +func fileDietRestore(files []postgresx.ValueProofFile) (fileDietArchive, error) { + candidates := map[string][]byte{} + for _, file := range files { + name := strings.TrimPrefix(file.Path, "report/proof/") + if name == file.Path { + name = strings.TrimPrefix(file.Path, "proof/") + } + if name != "proof.json.gz" && !fileDietInventoryName.MatchString(name) { + continue + } + if _, exists := candidates[name]; exists { + return fileDietArchive{}, errors.New("duplicate restored File Diet proof archive") + } + candidates[name] = file.Content + } + return fileDietLoadProof(candidates) +} + +func fileDietArchiveProof(value any) ([]byte, error) { + content := fileDietEncodeJSON(value) + if len(content) == 0 || len(content)+1 > fileDietProofLimit { + return nil, errors.New("file Diet proof exceeds decoded byte budget") + } + var output bytes.Buffer + writer := gzip.NewWriter(&output) + if _, err := writer.Write(append(content, '\n')); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return output.Bytes(), nil +} diff --git a/server/internal/collect/projector_remote_value_receipts.go b/server/internal/collect/projector_remote_value_receipts.go new file mode 100644 index 0000000000..ac6e1c2944 --- /dev/null +++ b/server/internal/collect/projector_remote_value_receipts.go @@ -0,0 +1,218 @@ +package collect + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +func fileDietPreserveReceipt(issue *fileDietIssue, previous fileDietIssue) (bool, error) { + if previous.Receipt == nil || previous.Receipt.Status != "available" { + return false, nil + } + old := previous.Receipt + if previous.ID != issue.ID || previous.Number != issue.Number || + previous.Reference.RunID == nil || issue.Reference.RunID == nil || + *previous.Reference.RunID != *issue.Reference.RunID || + fileDietRawBytesDigest([]byte(old.Content)) != old.EntrySHA256 || + old.Digest != "sha256:"+old.ArchiveSHA256 || + strconv.FormatInt(old.RunID, 10) != *issue.Reference.RunID { + return false, errors.New("preserved File Diet receipt identity or digest changed") + } + if issue.Run.Validation != nil && old.HeadSHA != issue.Run.Validation.HeadSHA { + return false, errors.New("preserved File Diet receipt producer revision changed") + } + if issue.Run.Validation == nil { + issue.Run.Validation = previous.Run.Validation + } + retained := *old + retained.ReusedFromPreservedProof = true + issue.Receipt = &retained + return true, nil +} + +func fileDietAppliedReceipt(content string, issue fileDietIssue) (fileDietReceipt, error) { + var matches []struct { + raw []byte + index int + } + for index, line := range strings.Split(strings.TrimSpace(content), "\n") { + var row struct { + Type string `json:"type"` + Number int `json:"number"` + Repo string `json:"repo"` + URL string `json:"url"` + Metadata *struct { + NodeID string `json:"node_id"` + } `json:"metadata"` + } + if err := json.Unmarshal([]byte(line), &row); err != nil { + return fileDietReceipt{}, errors.New("invalid File Diet safe-output receipt JSON") + } + if row.Type != "create_issue" || row.Number != issue.Number || + (row.Repo != dailyFileDietRepository && row.Repo != "githubnext/gh-aw") || + row.URL != fmt.Sprintf("https://github.com/%s/issues/%d", row.Repo, row.Number) { + continue + } + if row.Metadata != nil && row.Metadata.NodeID != issue.ID { + return fileDietReceipt{}, errors.New("applied File Diet output node identity mismatch") + } + var compact bytes.Buffer + if err := json.Compact(&compact, []byte(line)); err != nil { + return fileDietReceipt{}, err + } + matches = append(matches, struct { + raw []byte + index int + }{compact.Bytes(), index}) + } + if len(matches) != 1 { + reason := "Producer artifact does not record creation of this output" + if len(matches) != 0 { + reason = "Ambiguous applied creation receipts" + } + return fileDietReceipt{Status: "not-applied-to-output", Reason: reason, + SourceEntrySHA256: fileDietRawBytesDigest([]byte(content))}, nil + } + return fileDietReceipt{Status: "available", Content: string(matches[0].raw), + EntrySHA256: fileDietRawBytesDigest(matches[0].raw), + SourceEntrySHA256: fileDietRawBytesDigest([]byte(content)), + SourceRowIndex: &matches[0].index}, nil +} + +func fileDietZIPReceipt(content []byte) (string, error) { + if len(content) > 2<<20 { + return "", errors.New("file Diet receipt archive exceeds its byte budget") + } + reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + if err != nil { + return "", errors.New("invalid File Diet receipt ZIP") + } + var selected *zip.File + for _, file := range reader.File { + if file.Name == "safe-output-items.jsonl" { + if selected != nil || file.UncompressedSize64 > 64<<10 || !file.Mode().IsRegular() { + return "", errors.New("file Diet receipt ZIP contains ambiguous or oversized safe output") + } + selected = file + } + } + if selected == nil { + return "", errors.New("file Diet receipt ZIP lacks its applied safe output") + } + input, err := selected.Open() + if err != nil { + return "", err + } + defer func() { _ = input.Close() }() + decoded, err := io.ReadAll(io.LimitReader(input, 64<<10+1)) + if err != nil || len(decoded) > 64<<10 { + return "", errors.New("file Diet receipt exceeds its decoded byte budget") + } + return string(decoded), nil +} + +func fileDietCollectReceipts( + ctx context.Context, read fileDietRead, proof *fileDietProof, previous *fileDietArchive, +) error { + priorIssues := map[string]fileDietIssue{} + if previous != nil { + for _, issue := range previous.proof.Issues { + priorIssues[issue.ID] = issue + } + } + for index := range proof.Issues { + issue := &proof.Issues[index] + if prior, exists := priorIssues[issue.ID]; exists { + retained, err := fileDietPreserveReceipt(issue, prior) + if err != nil { + return err + } + if retained { + continue + } + } + var artifacts []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Expired bool `json:"expired"` + Digest string `json:"digest"` + } + if len(issue.RetainedArtifacts) > 0 && !bytes.Equal(issue.RetainedArtifacts, []byte("null")) { + if err := json.Unmarshal(issue.RetainedArtifacts, &artifacts); err != nil { + return errors.New("invalid File Diet retained artifact inventory") + } + } + reason := "No retained applied safe-output receipt" + if len(issue.Run.Unavailable) > 0 && !bytes.Equal(issue.Run.Unavailable, []byte("null")) { + var unavailable struct { + Reason string `json:"reason"` + } + if err := json.Unmarshal(issue.Run.Unavailable, &unavailable); err != nil { + return err + } + if unavailable.Reason != "" { + reason = unavailable.Reason + } + } + issue.Receipt = &fileDietReceipt{Status: "missing", Reason: reason} + for _, artifact := range artifacts { + if artifact.Name != "safe-outputs-items" || artifact.Expired { + continue + } + if issue.Reference.RunID == nil || issue.Run.Validation == nil || artifact.ID <= 0 { + return errors.New("file Diet receipt lacks verified producer identity") + } + endpoint := fmt.Sprintf("repos/%s/actions/artifacts/%d", dailyFileDietRepository, artifact.ID) + var metadata struct { + SizeInBytes int64 `json:"size_in_bytes"` + Digest string `json:"digest"` + WorkflowRun struct { + ID int64 `json:"id"` + HeadSHA string `json:"head_sha"` + } `json:"workflow_run"` + } + var err error + metadata, err = fileDietReadJSON[struct { + SizeInBytes int64 `json:"size_in_bytes"` + Digest string `json:"digest"` + WorkflowRun struct { + ID int64 `json:"id"` + HeadSHA string `json:"head_sha"` + } `json:"workflow_run"` + }](ctx, read, endpoint, "") + if err != nil || metadata.SizeInBytes < 0 || metadata.SizeInBytes >= 2<<20 || + metadata.Digest != artifact.Digest || + strconv.FormatInt(metadata.WorkflowRun.ID, 10) != *issue.Reference.RunID || + metadata.WorkflowRun.HeadSHA != issue.Run.Validation.HeadSHA { + return errors.New("file Diet receipt metadata identity mismatch") + } + content, err := read(ctx, endpoint+"/zip", "", true) + if err != nil || "sha256:"+fileDietRawBytesDigest(content) != metadata.Digest { + return errors.New("file Diet receipt archive digest mismatch") + } + text, err := fileDietZIPReceipt(content) + if err != nil { + return err + } + receipt, err := fileDietAppliedReceipt(text, *issue) + if err != nil { + return err + } + receipt.RunID, receipt.HeadSHA = metadata.WorkflowRun.ID, metadata.WorkflowRun.HeadSHA + receipt.ArtifactID, receipt.Digest = artifact.ID, metadata.Digest + receipt.ArchiveSHA256 = fileDietRawBytesDigest(content) + receipt.ObservedAt = time.Now().UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z") + issue.Receipt = &receipt + break + } + } + return nil +} diff --git a/server/internal/collect/projector_remote_value_test.go b/server/internal/collect/projector_remote_value_test.go index 5cdf9bd8ec..8e2eace2d3 100644 --- a/server/internal/collect/projector_remote_value_test.go +++ b/server/internal/collect/projector_remote_value_test.go @@ -10,7 +10,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "slices" "strconv" "strings" @@ -18,7 +17,6 @@ import ( "time" "github.com/githubnext/gh-cao/server/internal/githubquota" - "github.com/githubnext/gh-cao/server/internal/model" "github.com/githubnext/gh-cao/server/internal/operational" "github.com/githubnext/gh-cao/server/internal/postgresx" ) @@ -52,66 +50,6 @@ process.stdout.write(graphQuery("octo/api", [5, 23]));` } } -func TestRemoteValuePublicationRejectsChangedStartingPacket(t *testing.T) { - initial := model.Row{"id": "proof", "observedAt": "2026-01-01T00:00:00Z", "facts": []any{"one"}} - same := model.Row{"facts": []any{"one"}, "id": "proof", "observedAt": "2026-01-01T00:00:00Z"} - if !remoteValuePacketCurrent(initial, same) || !remoteValuePacketCurrent(nil, nil) { - t.Fatal("unchanged previous proof was rejected") - } - for _, current := range []model.Row{nil, - {"id": "new-proof", "observedAt": "2026-01-02T00:00:00Z", "facts": []any{"one"}}, - {"id": "proof", "observedAt": "2026-01-01T00:00:00Z", "facts": []any{"different"}}, - } { - if remoteValuePacketCurrent(initial, current) { - t.Fatalf("stale collector proof was accepted: %v", current) - } - } - if remoteValuePacketCurrent(nil, initial) { - t.Fatal("concurrently published initial proof was accepted") - } -} - -func TestRemoteValueFailureUsesOnlyBoundedWorkerDiagnostic(t *testing.T) { - for _, testCase := range []struct { - name, stderr, reason, message string - }{ - { - name: "timeout", - stderr: "private command output\n" + remoteValueFailurePrefix + - `{"reasonCode":"timeout","detail":"collector command timed out (SIGTERM)"}` + "\n", - reason: "timeout", message: "remote Value collector command timed out; previous evidence retained", - }, - { - name: "api budget", - stderr: remoteValueFailurePrefix + - `{"reasonCode":"api-budget","detail":"collector exhausted its reviewed API request grant"}` + "\n", - reason: "api-budget", - message: "remote Value collector exhausted its reviewed API request grant; previous evidence retained", - }, - { - name: "malformed marker", - stderr: remoteValueFailurePrefix + `{"reasonCode":"secret-output"}` + "\nprivate command output\n", - reason: "collector-failed", message: "remote Value collector failed; previous evidence retained", - }, - { - name: "raw stderr", - stderr: "private command output\n", - reason: "collector-failed", message: "remote Value collector failed; previous evidence retained", - }, - } { - t.Run(testCase.name, func(t *testing.T) { - reason := remoteValueFailureReason([]byte(testCase.stderr)) - if reason != testCase.reason { - t.Fatalf("reason = %q, want %q", reason, testCase.reason) - } - message := remoteValueFailureError(reason).Error() - if message != testCase.message || strings.Contains(message, "private command output") { - t.Fatalf("safe diagnostic = %q, want %q", message, testCase.message) - } - }) - } -} - func TestRemoteValueRepositoriesReviewedPolicyCacheAndEnrollment(t *testing.T) { b := volatileBackfill(t) root, err := filepath.Abs(filepath.Join("..", "..", "..")) @@ -140,8 +78,13 @@ func TestRemoteValueRepositoriesReviewedPolicyCacheAndEnrollment(t *testing.T) { t.Fatalf("reviewed registered repository = %v, %v", names, err) } } - if !projector.RemoteValuePlan.loaded || len(projector.RemoteValuePlan.entries) != 1 || - projector.RemoteValuePlan.entries[0].APIRequests != 700 { + foundFileDiet := false + for _, entry := range projector.RemoteValuePlan.entries { + if entry.Workflow == dailyFileDietWorkflow { + foundFileDiet = entry.APIRequests == 700 + } + } + if !projector.RemoteValuePlan.loaded || !foundFileDiet { t.Fatal("reviewed API-bound registration was not cached") } disabled := strings.Replace(string(policy), `"api-requests": 700,`, `"api-requests": 700, "enabled": false,`, 1) @@ -151,8 +94,14 @@ func TestRemoteValueRepositoriesReviewedPolicyCacheAndEnrollment(t *testing.T) { if err := os.WriteFile(path, []byte(disabled), 0o600); err != nil { // #nosec G703 -- fixed filename beneath t.TempDir. t.Fatal(err) } - if names, err := projector.RemoteValueRepositories(t.Context()); err != nil || len(names) != 0 { - t.Fatalf("stale policy cache admitted disabled registration: %v, %v", names, err) + if names, err := projector.RemoteValueRepositories(t.Context()); err != nil || + !slices.Equal(names, []string{"github/gh-aw"}) { + t.Fatalf("disabling File Diet suppressed the independent registration: %v, %v", names, err) + } + for _, entry := range projector.RemoteValuePlan.entries { + if entry.Workflow == dailyFileDietWorkflow { + t.Fatal("stale policy cache admitted disabled File Diet registration") + } } } @@ -552,32 +501,6 @@ func TestRemoteValueGraphQLReservesUpperBoundCommitsObservedCost(t *testing.T) { if sent, _ := second.usage(); sent != 0 { t.Fatalf("reviewed GraphQL cache hit consumed request grant: %d", sent) } - if runtime.GOOS == "linux" { - api.max = 3 - socket, closeAPI, err := api.listen(t.Context()) - if err != nil { - t.Fatal(err) - } - defer closeAPI() - script := filepath.Join(t.TempDir(), "gh") - if err := os.WriteFile(script, []byte(remoteValueGH), 0o600); err != nil { - t.Fatal(err) - } - command := exec.CommandContext(t.Context(), "node", script, "api", "graphql", "-f", "query="+query) // #nosec G204 -- fixed bridge script and generated test query. - command.Env = []string{"PATH=" + os.Getenv("PATH"), "GH_TOKEN=remote-value-proxy-only", - "CAO_REMOTE_VALUE_SOCKET=" + socket} - output, err := command.CombinedOutput() - if err != nil || !strings.Contains(string(output), `"cost":3`) || len(paths) != 3 { - t.Fatalf("GraphQL gh wrapper bypassed metered host request: %v %s paths=%v", err, output, paths) - } - command = exec.CommandContext(t.Context(), "node", script, "api", "graphql", "-f", "query="+query) // #nosec G204 -- fixed bridge script and generated test query. - command.Env = []string{"PATH=" + os.Getenv("PATH"), "GH_TOKEN=remote-value-proxy-only", - "CAO_REMOTE_VALUE_SOCKET=" + socket, "CAO_VALUE_ACQUISITION_REQUIREMENT=" + requirementID} - output, err = command.CombinedOutput() - if err != nil || !strings.Contains(string(output), `"cost":3`) || len(paths) != 3 { - t.Fatalf("reviewed GraphQL gh wrapper cache miss: %v %s paths=%v", err, output, paths) - } - } } func TestRemoteValueGraphQLUnknownOrUnverifiableCostFailsClosed(t *testing.T) { @@ -722,78 +645,6 @@ func TestRemoteValueArtifactRedirectRejectsOtherDestinations(t *testing.T) { } } -func TestRemoteValueGHWrapperRejectsGraphQLWithoutToken(t *testing.T) { - directory := t.TempDir() - script := filepath.Join(directory, "gh") - if err := os.WriteFile(script, []byte(remoteValueGH), 0o600); err != nil { - t.Fatal(err) - } - command := exec.CommandContext(t.Context(), "node", script, "api", "graphql", "-f", "query=query{}") // #nosec G204 -- fixed bridge script beneath t.TempDir. - command.Env = []string{"PATH=" + os.Getenv("PATH"), - "CAO_REMOTE_VALUE_SOCKET=" + filepath.Join(directory, "missing.sock")} - output, err := command.CombinedOutput() - if err == nil || !strings.Contains(string(output), "remote Value API unavailable") { - t.Fatalf("GraphQL request bypassed the unavailable quota proxy: %v %s", err, output) - } -} - -func TestRemoteValueGHWrapperUsesHostQuotaAndKeepsTokenPrivate(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("remote Value authenticated API confinement requires Linux") - } - b := volatileBackfill(t) - quota, err := githubquota.New(b.Enrollment.Metadata.(operational.GitHubQuotaStore), - githubquota.Options{SafetyReserve: 1}) - if err != nil { - t.Fatal(err) - } - reset := time.Now().Add(time.Hour) - if err := quota.Observe(t.Context(), githubquota.BucketID{App: "github-app-1", Installation: 7}, - githubquota.Observation{Limit: 5000, Remaining: 4000, ResetAt: reset}); err != nil { - t.Fatal(err) - } - calls := 0 - upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - calls++ - if request.Header.Get("Authorization") != "Bearer private-installation-token" { - t.Error("App credential was not used exclusively by the Go proxy") - } - writer.Header().Set("X-RateLimit-Resource", "core") - writer.Header().Set("X-RateLimit-Limit", "5000") - writer.Header().Set("X-RateLimit-Remaining", "3999") - writer.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) - _, _ = io.WriteString(writer, `{"id":42}`) - })) - defer upstream.Close() - base, _ := url.Parse(upstream.URL) - api := &remoteValueAPI{client: upstream.Client(), quota: quota, searchQuota: quota, - app: "github-app-1", installation: 7, repository: "octo/api", - base: base, token: "private-installation-token", max: 1} - directory := t.TempDir() - socket, closeAPI, err := api.listen(t.Context()) - if err != nil { - t.Fatal(err) - } - defer closeAPI() - script := filepath.Join(directory, "gh") - if err := os.WriteFile(script, []byte(remoteValueGH), 0o600); err != nil { - t.Fatal(err) - } - command := exec.CommandContext(t.Context(), "node", script, "api", "repos/octo/api") // #nosec G204 -- fixed bridge script beneath t.TempDir. - command.Env = []string{"PATH=" + os.Getenv("PATH"), "GH_TOKEN=remote-value-proxy-only", - "CAO_REMOTE_VALUE_SOCKET=" + socket} - content, err := command.CombinedOutput() - if err != nil || string(content) != `{"id":42}` || calls != 1 { - t.Fatalf("confined request: output=%s calls=%d err=%v", content, calls, err) - } - denied := exec.CommandContext(t.Context(), "node", script, "api", "repos/octo/api") // #nosec G204 -- fixed bridge script beneath t.TempDir. - denied.Env = command.Env - content, err = denied.CombinedOutput() - if err == nil || calls != 1 { - t.Fatalf("request cap bypassed: output=%s calls=%d err=%v", content, calls, err) - } -} - func TestRemoteValueDefersWithoutHostedServices(t *testing.T) { err := (Projector{}).CollectRemoteValues(t.Context(), "octo/api") if !errors.Is(err, ErrRemoteValueDeferred) { diff --git a/server/internal/doctor/checks_collect.go b/server/internal/doctor/checks_collect.go index 8358d38f4f..eac784b114 100644 --- a/server/internal/doctor/checks_collect.go +++ b/server/internal/doctor/checks_collect.go @@ -4,14 +4,12 @@ import ( "context" "fmt" "os" - "os/exec" - "path/filepath" "strconv" "strings" - "time" "github.com/githubnext/gh-cao/server/internal/collect" "github.com/githubnext/gh-cao/server/internal/githubapp" + "github.com/githubnext/gh-cao/server/internal/operational" "github.com/githubnext/gh-cao/server/internal/server" ) @@ -81,23 +79,20 @@ func (d Doctor) checkCollectionSettings(context.Context) Check { } config := server.CollectorConfig{ AppID: appID, - AdmitOnly: profile.admitOnly, - LakeDirectory: d.getenv("CAO_COLLECT_LAKE_DIRECTORY"), CatalogRoot: d.getenv("CAO_COLLECT_CATALOG_ROOT"), ControlRepository: d.getenv("CAO_COLLECT_CONTROL_REPOSITORY"), PolicyPath: d.getenv("CAO_POLICY_PATH"), - StaticInventoryPath: d.getenv( - "CAO_COLLECT_STATIC_INVENTORY", - ), Workers: intEnv(d.getenv("CAO_COLLECT_WORKERS")), - RecoverDeliveries: envTruthy(d.getenv("CAO_COLLECT_RECOVER_DELIVERIES")), } if err := config.ApplyNativeEnvironment(d.getenv); err != nil { return failed(id, areaCollect, title, err) } + if !config.NativeAcquisition { + return failed(id, areaCollect, title, config.Validate()) + } keySource, keyPresent := "absent", false var keyErr error - if !config.NativeAcquisition || !config.AdmitOnly { + if !config.AdmitOnly { keySource, keyPresent, keyErr = d.privateKeySource() } details := []Detail{ @@ -105,7 +100,6 @@ func (d Doctor) checkCollectionSettings(context.Context) Check { detail("admitOnly", fmt.Sprint(profile.admitOnly)), detail("privateKey", keySource), detail("webhookSecret", presence(d.getenv("CAO_GITHUB_WEBHOOK_SECRET"))), - detail("lakeDirectory", presence(d.getenv("CAO_COLLECT_LAKE_DIRECTORY"))), detail("catalogRoot", presence(d.getenv("CAO_COLLECT_CATALOG_ROOT"))), detail("controlRepository", presence(d.getenv("CAO_COLLECT_CONTROL_REPOSITORY"))), detail("workers", orDefault(d.getenv("CAO_COLLECT_WORKERS"), "0")), @@ -142,6 +136,39 @@ func (d Doctor) checkCollectionSettings(context.Context) Check { } } +// checkCollectionStorage verifies the durable canonical store needed by +// native admission, acquisition, and publication. +func (d Doctor) checkCollectionStorage(context.Context) Check { + const id, title = "collect.storage", "Collection storage" + if !d.profile().collecting { + return skipped(id, areaCollect, title, "collection is not configured; this server runs the Actions profile") + } + if d.Backend != "" && d.Backend != "postgres" { + return failed(id, areaCollect, title, + fmt.Errorf("native collection requires PostgreSQL operational storage, not %s", d.Backend)) + } + if err := operational.CheckStore(d.Store); err != nil { + return failed(id, areaCollect, title, + fmt.Errorf("native collection requires durable PostgreSQL operational storage: %w", err)) + } + capability := d.Store.Capabilities().Collection + if capability.Scope != operational.ScopeDeployment || capability.Persistence != operational.PersistenceRestart { + return failed(id, areaCollect, title, + fmt.Errorf("%w: native collection requires durable deployment-scoped PostgreSQL operational storage", operational.ErrUnsupported)) + } + if d.Postgres == nil || !d.Postgres.Open() { + return Check{ + ID: id, Area: areaCollect, Title: title, Status: StatusFail, + Summary: "native collection requires an open, durable deployment-scoped PostgreSQL store", + Remedy: "configure and open the deployment's PostgreSQL dashboard store before starting collection", + } + } + return Check{ + ID: id, Area: areaCollect, Title: title, Status: StatusPass, + Summary: "native collection uses the deployment's PostgreSQL store", + } +} + // collectionSettingsReason names why classifyCollectionSettings reached its // result, stable across summary wording changes so it is useful to log // without exposing the configured App identifier. @@ -457,7 +484,7 @@ func classifyBackfillState(state collect.BackfillState) backfillClassification { return backfillClassification{ status: StatusWarn, summary: "cold start has never run in this namespace", - remedy: "run `cao-dashboard backfill` so an empty database is repopulated from installations and the evidence lake", + remedy: "run `cao-dashboard backfill` to enumerate installations and enqueue native acquisition", reason: backfillReasonNeverRun, } } @@ -493,7 +520,6 @@ func (d Doctor) checkBackfill(ctx context.Context) Check { detail("installations", fmt.Sprint(state.Installations)), detail("repositories", fmt.Sprint(state.Repositories)), detail("queuedRepositories", fmt.Sprint(state.QueuedRepositories)), - detail("lakeReplayed", fmt.Sprint(state.LakeReplayed)), detail("startedAt", orDefault(state.StartedAt, "(never)")), detail("completedAt", orDefault(state.CompletedAt, "(not completed)")), } @@ -621,223 +647,33 @@ func classifyBudgetHeadroom(rateLimits []collect.Headroom, floor int) budgetHead } } -// lakePopulationReason names why checkLake reached its status, stable across -// summary wording changes so it is useful to log without exposing the -// reported byte and run-shard counts. -type lakePopulationReason string - -const ( - lakePopulationReasonEmpty lakePopulationReason = "empty" - lakePopulationReasonPopulated lakePopulationReason = "populated" -) - -// lakePopulationClassification is the status, summary, and remedy -// classifyLakePopulation derives from the evidence lake's populated flag and -// reported byte and run-shard counts. -type lakePopulationClassification struct { - status Status - summary string - remedy string - reason lakePopulationReason -} - -// classifyLakePopulation decides the collect.lake check's outcome from the -// lake's populated flag and reported size alone. It is a pure function so -// the empty-versus-populated distinction is testable without a real -// directory tree on disk. -func classifyLakePopulation(populated bool, totalBytes int64, runShards int) lakePopulationClassification { - if !populated { - return lakePopulationClassification{ - status: StatusWarn, - summary: "the evidence lake holds no replayable evidence", - remedy: "until it is populated, a cold start must re-collect from GitHub rather than replay locally", - reason: lakePopulationReasonEmpty, - } - } - return lakePopulationClassification{ - status: StatusPass, - summary: fmt.Sprintf("%s of replayable evidence across %d run shards", - humanBytes(totalBytes), runShards), - reason: lakePopulationReasonPopulated, - } -} - -// checkLake inspects the evidence lake, which is what makes a cold start a -// local replay instead of a full re-collection from GitHub. -func (d Doctor) checkLake(context.Context) Check { - const id, title = "collect.lake", "Evidence lake" - profile := d.profile() - if !profile.collecting { - return skipped(id, areaCollect, title, "collection is not configured") - } - if profile.admitOnly { - return skipped(id, areaCollect, title, "admission-only collection holds no evidence lake") - } - if envTruthy(d.getenv("CAO_COLLECT_NATIVE")) { - return skipped(id, areaCollect, title, "native collection stores durable evidence in PostgreSQL, not an Activity lake") - } - directory := d.getenv("CAO_COLLECT_LAKE_DIRECTORY") - if directory == "" { - return Check{ - ID: id, Area: areaCollect, Title: title, Status: StatusFail, - Summary: "no evidence lake directory is configured", - Remedy: "set CAO_COLLECT_LAKE_DIRECTORY to a durable absolute path", - } - } - lake := collect.Lake{Directory: directory} - if err := lake.Validate(); err != nil { - return Check{ - ID: id, Area: areaCollect, Title: title, Status: StatusFail, - Summary: "the evidence lake directory is not usable: " + err.Error(), - Details: []Detail{detail("directory", directory)}, - Remedy: "the lake must be an absolute path on durable storage", - } - } - info, err := os.Stat(directory) - if err != nil { - return Check{ - ID: id, Area: areaCollect, Title: title, Status: StatusFail, - Summary: "the evidence lake directory could not be read: " + err.Error(), - Details: []Detail{detail("directory", directory)}, - Remedy: "confirm the volume is mounted and writable by this process", - } - } - shards, shardBytes := directorySize(lake.ShardDirectory()) - runs, runBytes := directorySize(lake.RunsDirectory()) - records, recordBytes := directorySize(lake.RecordsDirectory()) - details := []Detail{ - detail("directory", directory), - detail("mode", info.Mode().Perm().String()), - detail("shardFiles", fmt.Sprint(shards)), - detail("runFiles", fmt.Sprint(runs)), - detail("recordFiles", fmt.Sprint(records)), - detail("size", humanBytes(shardBytes+runBytes+recordBytes)), - detail("manifest", filePresence(lake.ManifestPath())), - detail("inventory", filePresence(lake.InventoryPath())), - } - populated, err := lake.Populated() - if err != nil { - return failed(id, areaCollect, title, err) - } - classification := classifyLakePopulation(populated, shardBytes+runBytes+recordBytes, runs) - doctorLog.Printf("evidence lake classified status=%s reason=%s", classification.status, classification.reason) - return Check{ - ID: id, Area: areaCollect, Title: title, Status: classification.status, - Summary: classification.summary, Details: details, Remedy: classification.remedy, - } -} - -// checkProjectionLock reports whether an ingestion is in flight. The lock has -// a time to live, so a held lock is information rather than a fault. -func (d Doctor) checkProjectionLock(ctx context.Context) Check { - const id, title = "collect.projection", "Ingestion lock" - if d.profile().collecting && envTruthy(d.getenv("CAO_COLLECT_NATIVE")) { - return skipped(id, areaCollect, title, "native publication uses PostgreSQL attempt/source fences, not the legacy projection lock") - } - if skip, ok := d.storeUnavailable(id, areaCollect, title); ok { - return skip - } - held, err := d.Store.OperationalServices().Leases.LockHeld(ctx, "projection") - if err != nil { - return failed(id, areaCollect, title, err) - } - if held { - return Check{ - ID: id, Area: areaCollect, Title: title, Status: StatusPass, - Summary: "an ingestion currently holds the lock", - Details: []Detail{detail("held", "true")}, - } - } - return Check{ - ID: id, Area: areaCollect, Title: title, Status: StatusPass, - Summary: "no ingestion is in flight", - Details: []Detail{detail("held", "false")}, - } -} - -// toolingLookup are the PATH and filesystem probes missingTooling needs, -// injected so the missing-tool detection is testable without depending on -// which binaries or files happen to exist on the test runner. -type toolingLookup struct { - lookPath func(string) (string, error) - stat func(string) (os.FileInfo, error) -} - -// missingTooling reports which collection-tooling dependencies (Node, gh, and -// the activity CLI script) are unavailable, using the injected lookup. It is -// a pure function so every combination of missing tool is testable without -// touching the real PATH or filesystem. -func missingTooling(lookup toolingLookup, node, gh, script string) []string { - var missing []string - if _, err := lookup.lookPath(node); err != nil { - missing = append(missing, "node") - } - if _, err := lookup.lookPath(gh); err != nil { - missing = append(missing, "gh") - } - if script == "" { - missing = append(missing, "catalog root") - } else if _, err := lookup.stat(script); err != nil { - missing = append(missing, "activity/cao.mjs") - } - return missing -} - -// checkTooling confirms the external programs collection shells out to are -// actually present, which is the difference between a working collector and -// one that fails on its first task. +// checkTooling checks only the optional Value lane's selected dependencies. +// Native acquisition itself does not need the selected Value dependencies. func (d Doctor) checkTooling(context.Context) Check { - const id, title = "runtime.tooling", "Collection tooling" + const id, title = "runtime.tooling", "Value runtime dependencies" profile := d.profile() if !profile.collecting || profile.admitOnly { - return skipped(id, areaRuntime, title, "this process does not run the collection CLI") + return skipped(id, areaRuntime, title, "this process does not execute native Value") } var config server.CollectorConfig if err := config.ApplyNativeEnvironment(d.getenv); err != nil { return failed(id, areaRuntime, title, err) } - if config.NativeAcquisition { - if !config.NativeValues { - return Check{ID: id, Area: areaRuntime, Title: title, Status: StatusPass, - Summary: "native Go collection requires no Node, gh or Activity CLI"} - } - values := collect.NativeValueService{Projector: collect.Projector{ - CatalogRoot: d.getenv("CAO_COLLECT_CATALOG_ROOT"), PolicyPath: d.getenv("CAO_POLICY_PATH"), - NodeBinary: d.getenv("CAO_COLLECT_NODE_BINARY"), - }} - if err := values.ValidateRuntime(); err != nil { - return failed(id, areaRuntime, title, err) - } - return Check{ID: id, Area: areaRuntime, Title: title, Status: StatusPass, - Summary: "selected Value execution has Node and reviewed policy/catalog assets; native collection needs no Activity CLI"} - } - node := orDefault(d.getenv("CAO_COLLECT_NODE_BINARY"), "node") - gh := orDefault(d.getenv("CAO_COLLECT_GH_BINARY"), "gh") - catalog := d.getenv("CAO_COLLECT_CATALOG_ROOT") - script := "" - if catalog != "" { - script = filepath.Join(catalog, "activity", "cao.mjs") - } - details := []Detail{ - detail("node", binaryPresence(node)), - detail("gh", binaryPresence(gh)), - detail("activityCli", orDefault(filePresence(script), "(no catalog root configured)")), + if !config.NativeAcquisition { + return failed(id, areaRuntime, title, config.Validate()) } - missing := missingTooling(toolingLookup{lookPath: exec.LookPath, stat: os.Stat}, node, gh, script) - doctorLog.Printf("collection tooling checked missing_count=%d", len(missing)) - if len(missing) > 0 { - return Check{ - ID: id, Area: areaRuntime, Title: title, Status: StatusFail, - Summary: "collection tooling is missing: " + strings.Join(missing, ", "), - Details: details, - Remedy: "collection shells out to these; without them every task fails at the first step", - } + if !config.NativeValues { + return Check{ID: id, Area: areaRuntime, Title: title, Status: StatusPass, + Summary: "native Go collection requires no Node or Activity CLI"} } - return Check{ - ID: id, Area: areaRuntime, Title: title, Status: StatusPass, - Summary: "node, gh, and the activity CLI are all available", - Details: details, + values := collect.NativeValueService{Projector: collect.Projector{ + CatalogRoot: d.getenv("CAO_COLLECT_CATALOG_ROOT"), PolicyPath: d.getenv("CAO_POLICY_PATH"), + }} + if err := values.ValidateRuntime(); err != nil { + return failed(id, areaRuntime, title, err) } + return Check{ID: id, Area: areaRuntime, Title: title, Status: StatusPass, + Summary: "selected Go Value execution has reviewed policy/catalog assets"} } func presence(value string) string { @@ -861,78 +697,3 @@ func intEnv(value string) int { } return parsed } - -func filePresence(path string) string { - if strings.TrimSpace(path) == "" { - return "" - } - info, err := os.Stat(path) - if err != nil { - return "missing" - } - return fmt.Sprintf("present (%s, modified %s)", humanBytes(info.Size()), - info.ModTime().UTC().Format(time.RFC3339)) -} - -func binaryPresence(name string) string { - resolved, err := exec.LookPath(name) - if err != nil { - return name + " (not found)" - } - return resolved -} - -// directorySizeReadFailure classifies why directorySize's os.ReadDir call did -// not return entries, so an unreadable lake directory is diagnosable without -// logging the directory path itself. -type directorySizeReadFailure string - -const ( - directorySizeReadFailureNone directorySizeReadFailure = "none" - directorySizeReadFailureMissing directorySizeReadFailure = "missing" - directorySizeReadFailureOther directorySizeReadFailure = "other" -) - -// classifyDirectorySizeReadFailure distinguishes a directory that does not -// exist, which is the unpopulated-lake state checkLake already reports, from -// any other read failure, such as a permissions error, which is worth -// surfacing as a diagnostic. It is a pure function extracted from -// directorySize so this distinction is testable against constructed errors -// without touching the filesystem. -func classifyDirectorySizeReadFailure(err error) directorySizeReadFailure { - switch { - case err == nil: - return directorySizeReadFailureNone - case os.IsNotExist(err): - return directorySizeReadFailureMissing - default: - return directorySizeReadFailureOther - } -} - -// directorySize counts the files and bytes in one lake directory. A missing -// directory reports zero rather than an error, because an unpopulated lake is -// a state the caller reports, not a failure to observe it. Any other read -// failure, such as a permissions error, is logged so it is diagnosable -// without being promoted to a check failure. -func directorySize(directory string) (files int, bytes int64) { - entries, err := os.ReadDir(directory) - if failure := classifyDirectorySizeReadFailure(err); failure != directorySizeReadFailureNone { - if failure == directorySizeReadFailureOther { - doctorLog.Printf("lake directory read failed") - } - return 0, 0 - } - for _, entry := range entries { - if entry.IsDir() { - continue - } - info, err := entry.Info() - if err != nil { - continue - } - files++ - bytes += info.Size() - } - return files, bytes -} diff --git a/server/internal/doctor/checks_collect_native_test.go b/server/internal/doctor/checks_collect_native_test.go index 145d7a92a9..9ea94d5a82 100644 --- a/server/internal/doctor/checks_collect_native_test.go +++ b/server/internal/doctor/checks_collect_native_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/githubnext/gh-cao/server/internal/operational/memory" ) func nativeDoctorEnvironment() map[string]string { @@ -16,10 +18,6 @@ func nativeDoctorEnvironment() map[string]string { "CAO_COLLECT_CONTROL_REPOSITORY": "octo/control", "CAO_COLLECT_NATIVE_INVENTORY_DIGEST": strings.Repeat("a", 64), "CAO_GITHUB_WEBHOOK_SECRET": "configured-not-read", - "CAO_COLLECT_LAKE_DIRECTORY": "/must-not-read", - "CAO_COLLECT_CATALOG_ROOT": "/must-not-read", - "CAO_COLLECT_STATIC_INVENTORY": "/must-not-read", - "CAO_POLICY_PATH": "/must-not-read", } } @@ -36,10 +34,8 @@ func TestNativeDoctorDoesNotRequireLegacyFilesOrExecutables(t *testing.T) { t.Errorf("%s unexpectedly requires a legacy dependency: %+v", check.ID, check) } } - for _, check := range []Check{d.checkLake(t.Context()), d.checkProjectionLock(t.Context())} { - if check.Status != StatusSkip { - t.Errorf("%s consulted legacy ownership/storage: %+v", check.ID, check) - } + if check := d.checkCollectionStorage(t.Context()); check.Status != StatusFail || !strings.Contains(check.Summary, "PostgreSQL") { + t.Fatalf("native collection accepted a missing PostgreSQL store: %+v", check) } } @@ -57,6 +53,9 @@ func TestNativeDoctorSettingsUseRuntimeValidation(t *testing.T) { {"malformed Value flag", "CAO_COLLECT_NATIVE_VALUES", "yes", "must be a boolean"}, {"malformed admission", "CAO_COLLECT_ADMIT_ONLY", "yes", "must be a boolean"}, {"malformed recovery", "CAO_COLLECT_RECOVER_DELIVERIES", "yes", "must be a boolean"}, + {"retired lake", "CAO_COLLECT_LAKE_DIRECTORY", "/old-lake", "CAO_COLLECT_LAKE_DIRECTORY"}, + {"retired inventory", "CAO_COLLECT_STATIC_INVENTORY", "/old-inventory", "CAO_COLLECT_STATIC_INVENTORY"}, + {"retired gh", "CAO_COLLECT_GH_BINARY", "/usr/bin/gh", "CAO_COLLECT_GH_BINARY"}, } { t.Run(testcase.name, func(t *testing.T) { values := nativeDoctorEnvironment() @@ -70,6 +69,64 @@ func TestNativeDoctorSettingsUseRuntimeValidation(t *testing.T) { } } +func TestDoctorRejectsLegacyAppBeforeReadingKey(t *testing.T) { + values := nativeDoctorEnvironment() + delete(values, "CAO_COLLECT_NATIVE") + values["CAO_COLLECT_PRIVATE_KEY_FILE"] = "/must-not-read" + d := Doctor{Getenv: func(name string) string { return values[name] }} + check := d.checkCollectionSettings(t.Context()) + if check.Status != StatusFail || !strings.Contains(check.Summary, "filesystem collector is retired") { + t.Fatalf("legacy App configuration did not fail before reading key: %+v", check) + } + if strings.Contains(check.Summary, "/must-not-read") { + t.Fatalf("legacy App configuration inspected key file: %+v", check) + } +} + +func TestDoctorActionsProfileKeepsCollectionOptional(t *testing.T) { + d := Doctor{Getenv: func(string) string { return "" }} + for _, check := range []Check{ + d.checkCollectionProfile(t.Context()), + d.checkCollectionSettings(t.Context()), + d.checkCollectionStorage(t.Context()), + d.checkTooling(t.Context()), + } { + want := StatusSkip + if check.ID == "collect.profile" { + want = StatusPass + } + if check.Status != want { + t.Errorf("%s status = %s, want %s: %+v", check.ID, check.Status, want, check) + } + } +} + +func TestNativeDoctorReportUsesPostgresInsteadOfLegacyChecks(t *testing.T) { + store, err := memory.New(memory.Config{}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = store.Close() }() + values := nativeDoctorEnvironment() + d := Doctor{Backend: "memory", Store: store, Getenv: func(name string) string { return values[name] }} + report := d.Run(t.Context()) + foundStorage := false + for _, check := range report.Checks { + switch check.ID { + case "collect.lake", "collect.projection": + t.Errorf("retired check remains in report: %+v", check) + case "collect.storage": + foundStorage = true + if check.Status != StatusFail || !strings.Contains(check.Summary, "PostgreSQL") { + t.Errorf("native storage requirement was not diagnosed: %+v", check) + } + } + } + if !foundStorage { + t.Fatal("native storage check missing from report") + } +} + func TestNativeDoctorAdmissionDoesNotInspectWorkerCredentials(t *testing.T) { values := nativeDoctorEnvironment() values["CAO_COLLECT_ADMIT_ONLY"] = "true" @@ -103,51 +160,26 @@ func TestNativeDoctorValueToolingChecksOnlySelectedDependencies(t *testing.T) { values["CAO_COLLECT_NATIVE_VALUES"] = "true" values["CAO_COLLECT_CATALOG_ROOT"] = root values["CAO_POLICY_PATH"] = filepath.Join(root, "reviewed-policy.json") - values["CAO_COLLECT_NODE_BINARY"] = filepath.Join(root, "node") d := Doctor{Getenv: func(name string) string { return values[name] }} - if check := d.checkTooling(t.Context()); check.Status != StatusFail || !strings.Contains(check.Summary, "requires Node") { - t.Fatalf("selected Value ignored its Node dependency: %+v", check) + if check := d.checkTooling(t.Context()); check.Status != StatusFail { + t.Fatalf("selected Value ignored its missing reviewed policy: %+v", check) } - if err := os.WriteFile(values["CAO_COLLECT_NODE_BINARY"], []byte("#!/bin/sh\nexit 1\n"), 0o700); err != nil { // #nosec G306 -- executable test-owned dependency probe, never launched. + if err := os.WriteFile(values["CAO_POLICY_PATH"], []byte("{}"), 0o600); err != nil { t.Fatal(err) } - assets := []string{ - values["CAO_POLICY_PATH"], - filepath.Join(root, "activity", "local-values.mjs"), - filepath.Join(root, "activity", "workflow-values.mjs"), - filepath.Join(root, ".github", "workflows", "shared", "policy.mjs"), - } - for _, path := range assets { - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, nil, 0o600); err != nil { - t.Fatal(err) - } - } if check := d.checkTooling(t.Context()); check.Status != StatusPass { - t.Fatalf("selected Value incorrectly required gh or activity/cao.mjs: %+v", check) + t.Fatalf("selected Value rejected Go policy without Node: %+v", check) } - for _, path := range assets { - t.Run(filepath.Base(path), func(t *testing.T) { - if err := os.Remove(path); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if err := os.WriteFile(path, nil, 0o600); err != nil { // #nosec G703 -- fixed asset path under t.TempDir, not external input. - t.Error(err) - } - }) - if check := d.checkTooling(t.Context()); check.Status != StatusFail { - t.Fatalf("selected Value ignored a missing reviewed asset: %+v", check) - } - }) + if err := os.Remove(values["CAO_POLICY_PATH"]); err != nil { + t.Fatal(err) + } + if check := d.checkTooling(t.Context()); check.Status != StatusFail { + t.Fatalf("selected Value ignored removed reviewed policy: %+v", check) } unselected := maps.Clone(values) unselected["CAO_COLLECT_NATIVE_VALUES"] = "false" - unselected["CAO_COLLECT_NODE_BINARY"] = "/nonexistent" d.Getenv = func(name string) string { return unselected[name] } if check := d.checkTooling(t.Context()); check.Status != StatusPass { - t.Fatalf("unselected Value retained a Node requirement: %+v", check) + t.Fatalf("unselected Value inspected optional policy: %+v", check) } } diff --git a/server/internal/doctor/checks_collect_test.go b/server/internal/doctor/checks_collect_test.go index d66bc89114..d54c9309cd 100644 --- a/server/internal/doctor/checks_collect_test.go +++ b/server/internal/doctor/checks_collect_test.go @@ -1,94 +1,16 @@ package doctor import ( - "errors" "os" - "path/filepath" - "slices" "strings" "testing" "github.com/githubnext/gh-cao/server/internal/collect" ) -// fakeLookPath and fakeStat are the injected probes for missingTooling. A -// map keyed by the name/path a real exec.LookPath or os.Stat call would -// receive lets each test describe exactly which dependencies are present -// without touching the real PATH or filesystem. -func fakeLookPath(present map[string]bool) func(string) (string, error) { - return func(name string) (string, error) { - if present[name] { - return "/usr/bin/" + name, nil - } - return "", errors.New("not found") - } -} - -func fakeStat(present map[string]bool) func(string) (os.FileInfo, error) { +func fakeStat() func(string) (os.FileInfo, error) { return func(path string) (os.FileInfo, error) { - if present[path] { - return nil, nil - } - return nil, errors.New("not found") - } -} - -func TestMissingToolingReportsNothingWhenEverythingIsAvailable(t *testing.T) { - lookup := toolingLookup{ - lookPath: fakeLookPath(map[string]bool{"node": true, "gh": true}), - stat: fakeStat(map[string]bool{"/catalog/activity/cao.mjs": true}), - } - missing := missingTooling(lookup, "node", "gh", "/catalog/activity/cao.mjs") - if len(missing) != 0 { - t.Fatalf("expected no missing tooling, got %v", missing) - } -} - -func TestMissingToolingReportsEachAbsentDependency(t *testing.T) { - lookup := toolingLookup{ - lookPath: fakeLookPath(map[string]bool{}), - stat: fakeStat(map[string]bool{}), - } - missing := missingTooling(lookup, "node", "gh", "/catalog/activity/cao.mjs") - want := []string{"node", "gh", "activity/cao.mjs"} - if !slices.Equal(missing, want) { - t.Fatalf("missing = %v, want %v", missing, want) - } -} - -func TestMissingToolingReportsCatalogRootWhenScriptPathIsEmpty(t *testing.T) { - lookup := toolingLookup{ - lookPath: fakeLookPath(map[string]bool{"node": true, "gh": true}), - stat: fakeStat(map[string]bool{}), - } - missing := missingTooling(lookup, "node", "gh", "") - want := []string{"catalog root"} - if !slices.Equal(missing, want) { - t.Fatalf("missing = %v, want %v", missing, want) - } -} - -func TestMissingToolingReportsOnlyTheScriptWhenBinariesArePresent(t *testing.T) { - lookup := toolingLookup{ - lookPath: fakeLookPath(map[string]bool{"node": true, "gh": true}), - stat: fakeStat(map[string]bool{}), - } - missing := missingTooling(lookup, "node", "gh", "/catalog/activity/cao.mjs") - want := []string{"activity/cao.mjs"} - if !slices.Equal(missing, want) { - t.Fatalf("missing = %v, want %v", missing, want) - } -} - -func TestMissingToolingReportsOnlyMissingBinariesWhenScriptIsPresent(t *testing.T) { - lookup := toolingLookup{ - lookPath: fakeLookPath(map[string]bool{"gh": true}), - stat: fakeStat(map[string]bool{"/catalog/activity/cao.mjs": true}), - } - missing := missingTooling(lookup, "node", "gh", "/catalog/activity/cao.mjs") - want := []string{"node"} - if !slices.Equal(missing, want) { - t.Fatalf("missing = %v, want %v", missing, want) + return nil, os.ErrNotExist } } @@ -143,7 +65,7 @@ func TestClassifyQueueBacklogPassesWhenNoMaximumIsConfigured(t *testing.T) { } func TestResolvePrivateKeySourceReportsAbsentWhenNothingIsConfigured(t *testing.T) { - result := resolvePrivateKeySource(fakeStat(nil), "", "", "") + result := resolvePrivateKeySource(fakeStat(), "", "", "") if result.label != "absent" || result.present || result.err != nil { t.Fatalf("result = %+v, want label=absent present=false err=nil", result) } @@ -166,7 +88,7 @@ func TestResolvePrivateKeySourcePrefersFileOverInline(t *testing.T) { } func TestResolvePrivateKeySourceReportsUnreadableFile(t *testing.T) { - result := resolvePrivateKeySource(fakeStat(nil), "/etc/cao/missing.pem", "", "") + result := resolvePrivateKeySource(fakeStat(), "/etc/cao/missing.pem", "", "") if result.label != "file (unreadable)" || result.present || result.err == nil { t.Fatalf("result = %+v, want label=file (unreadable) present=false non-nil err", result) } @@ -189,7 +111,7 @@ func TestResolvePrivateKeySourceRejectsADirectory(t *testing.T) { } func TestResolvePrivateKeySourceReportsInlineWhenNoFileIsConfigured(t *testing.T) { - result := resolvePrivateKeySource(fakeStat(nil), "", "", "inline-value") + result := resolvePrivateKeySource(fakeStat(), "", "", "inline-value") if result.label != "inline environment variable (configured)" || !result.present || result.err != nil { t.Fatalf("result = %+v, want label=inline environment variable (configured) present=true err=nil", result) } @@ -199,7 +121,7 @@ func TestResolvePrivateKeySourceReportsInlineWhenNoFileIsConfigured(t *testing.T } func TestResolvePrivateKeySourceReportsBase64BeforeInline(t *testing.T) { - result := resolvePrivateKeySource(fakeStat(nil), "", "base64-value", "inline-value") + result := resolvePrivateKeySource(fakeStat(), "", "base64-value", "inline-value") if result.label != "base64 environment variable (configured)" || !result.present || result.err != nil { t.Fatalf("result = %+v, want label=base64 environment variable (configured) present=true err=nil", result) } @@ -323,47 +245,6 @@ func TestClassifyBackfillStatePassesWhenComplete(t *testing.T) { } } -func TestClassifyLakePopulationWarnsWhenEmpty(t *testing.T) { - classification := classifyLakePopulation(false, 0, 0) - if classification.status != StatusWarn { - t.Fatalf("status = %v, want %v", classification.status, StatusWarn) - } - if classification.reason != lakePopulationReasonEmpty { - t.Fatalf("reason = %v, want %v", classification.reason, lakePopulationReasonEmpty) - } - if classification.remedy == "" { - t.Fatal("expected a remedy for an unpopulated evidence lake") - } -} - -func TestClassifyLakePopulationPassesWhenPopulated(t *testing.T) { - classification := classifyLakePopulation(true, 4096, 3) - if classification.status != StatusPass { - t.Fatalf("status = %v, want %v", classification.status, StatusPass) - } - if classification.reason != lakePopulationReasonPopulated { - t.Fatalf("reason = %v, want %v", classification.reason, lakePopulationReasonPopulated) - } - if classification.remedy != "" { - t.Fatalf("expected no remedy for a populated evidence lake, got %q", classification.remedy) - } - if !strings.Contains(classification.summary, "3 run shards") { - t.Fatalf("summary = %q, want it to mention the run shard count", classification.summary) - } -} - -func TestClassifyLakePopulationIgnoresSizeWhenEmpty(t *testing.T) { - // A reported size alongside an unpopulated lake must not flip the - // classification to pass; Populated() is the sole authority here. - classification := classifyLakePopulation(false, 4096, 3) - if classification.status != StatusWarn { - t.Fatalf("status = %v, want %v", classification.status, StatusWarn) - } - if classification.reason != lakePopulationReasonEmpty { - t.Fatalf("reason = %v, want %v", classification.reason, lakePopulationReasonEmpty) - } -} - func TestClassifyBudgetHeadroomWarnsOnParkedBeforeLow(t *testing.T) { // A parked installation alongside a low one must not mask the parked // classification; parked is checked first, matching the prior inline @@ -436,62 +317,3 @@ func TestClassifyBudgetHeadroomPassesWhenHealthy(t *testing.T) { t.Fatalf("summary = %q, want it to mention the installation count", classification.summary) } } - -func TestClassifyDirectorySizeReadFailureReportsNoneForNilError(t *testing.T) { - if got := classifyDirectorySizeReadFailure(nil); got != directorySizeReadFailureNone { - t.Fatalf("classification = %v, want %v", got, directorySizeReadFailureNone) - } -} - -func TestClassifyDirectorySizeReadFailureReportsMissingForNotExist(t *testing.T) { - _, err := os.ReadDir(filepath.Join(t.TempDir(), "does-not-exist")) - if err == nil { - t.Fatal("expected a not-exist error from os.ReadDir") - } - if got := classifyDirectorySizeReadFailure(err); got != directorySizeReadFailureMissing { - t.Fatalf("classification = %v, want %v", got, directorySizeReadFailureMissing) - } -} - -func TestClassifyDirectorySizeReadFailureReportsOtherForUnrelatedError(t *testing.T) { - if got := classifyDirectorySizeReadFailure(errors.New("permission denied")); got != directorySizeReadFailureOther { - t.Fatalf("classification = %v, want %v", got, directorySizeReadFailureOther) - } -} - -func TestDirectorySizeReportsZeroForMissingDirectory(t *testing.T) { - files, bytes := directorySize(filepath.Join(t.TempDir(), "does-not-exist")) - if files != 0 || bytes != 0 { - t.Fatalf("files=%d bytes=%d, want 0, 0 for a missing directory", files, bytes) - } -} - -func TestDirectorySizeCountsFilesAndBytesSkippingSubdirectories(t *testing.T) { - directory := t.TempDir() - if err := os.WriteFile(filepath.Join(directory, "a.txt"), []byte("hello"), 0o600); err != nil { - t.Fatalf("write first file: %v", err) - } - if err := os.WriteFile(filepath.Join(directory, "b.txt"), []byte("worldwide"), 0o600); err != nil { - t.Fatalf("write second file: %v", err) - } - if err := os.Mkdir(filepath.Join(directory, "subdir"), 0o700); err != nil { - t.Fatalf("make subdirectory: %v", err) - } - if err := os.WriteFile(filepath.Join(directory, "subdir", "c.txt"), []byte("ignored"), 0o600); err != nil { - t.Fatalf("write nested file: %v", err) - } - files, bytes := directorySize(directory) - if files != 2 { - t.Fatalf("files = %d, want 2 (subdirectory entries must be skipped)", files) - } - if want := int64(len("hello") + len("worldwide")); bytes != want { - t.Fatalf("bytes = %d, want %d", bytes, want) - } -} - -func TestDirectorySizeReportsZeroForEmptyDirectory(t *testing.T) { - files, bytes := directorySize(t.TempDir()) - if files != 0 || bytes != 0 { - t.Fatalf("files=%d bytes=%d, want 0, 0 for an empty directory", files, bytes) - } -} diff --git a/server/internal/doctor/doctor.go b/server/internal/doctor/doctor.go index 75aff88e11..0fb1de93b2 100644 --- a/server/internal/doctor/doctor.go +++ b/server/internal/doctor/doctor.go @@ -129,13 +129,12 @@ func (d Doctor) Run(ctx context.Context) Report { d.checkSourceReads, d.checkCollectionProfile, d.checkCollectionSettings, + d.checkCollectionStorage, d.checkEnrollment, d.checkQueue, d.checkBackfill, d.checkBudget, - d.checkLake, d.checkTooling, - d.checkProjectionLock, ) for _, run := range checks { @@ -229,7 +228,7 @@ func redactRedisURL(rawURL string) string { func (d Doctor) storeUnavailable(id, area, title string) (Check, bool) { if operational.CheckStore(d.Store) == nil { caps := d.Store.Capabilities() - collection := area == areaCollect && id != "collect.projection" + collection := area == areaCollect if collection && caps.Collection.Scope == operational.ScopeUnsupported { return skipped(id, area, title, "collection is unsupported by the selected operational backend"), true } diff --git a/server/internal/operationalvalue/dependabot.go b/server/internal/operationalvalue/dependabot.go new file mode 100644 index 0000000000..0a843854a6 --- /dev/null +++ b/server/internal/operationalvalue/dependabot.go @@ -0,0 +1,219 @@ +package operationalvalue + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "time" + + "github.com/githubnext/gh-cao/server/internal/query" +) + +const ( + dependabotRepository = "github/gh-aw" + dependabotWorkflow = "dependabot-update-planner" + dependabotCollector = "dependabot/operational-value/dependabot-update-planner.mjs" + dependabotAdoption = "2026-09-17T08:48:00Z" + dependabotEvaluator = "be54f77f49e5d28b30a4d72d41708f3b65d505f02cb166c24a9ae9887099e961" +) + +var dependabotMetrics = []struct { + id, name, unit, role, digest string +}{ + {"open-security-alert-count", "Open security alerts", "alerts", "primary", + "d73a2676ebb04ebf34093873dc5eb72b8e54a461c8b5f068b7b3bd100fc2507a"}, + {"open-high-critical-alert-count", "Open critical/high security alerts", "alerts", "diagnostic", + "c0bcc97030ca99e4ed5bbe1b89b64ebde02bc7b4659682058e407cf155603ed2"}, + {"open-dependabot-pr-count", "Open Dependabot pull requests", "pull-requests", "diagnostic", + "774112c80afca79b272669e30a75dc5b1216191f9345e6e14970863bb617652a"}, +} + +var errDependabotEvidenceUnavailable = errors.New("dependabot evidence unavailable") + +func dependabotCutoff(observed time.Time) (time.Time, bool) { + adoption, _ := time.Parse(time.RFC3339, dependabotAdoption) + period := 7 * 24 * time.Hour + if observed.Before(adoption.Add(period)) { + return time.Time{}, false + } + elapsedNanos := observed.Sub(adoption).Nanoseconds() + periodNanos := period.Nanoseconds() + return adoption.Add(time.Duration(elapsedNanos / periodNanos * periodNanos)), true +} + +func reviewedDependabotEvidence(reviewed valuePolicy) bool { + for _, entry := range reviewed.Control.Campaigns["activity"].Values { + if entry.Repository == dependabotRepository && entry.Workflow == dependabotWorkflow && + entry.Collector == dependabotCollector && entry.APIRequests == 111 { + return true + } + } + return false +} + +type dependabotSnapshot struct { + values [3]int + capture time.Time + digest string +} + +// The query reads only the three reviewed packet identities through the +// canonical SQL query boundary. A missing or partial triplet is not zero. +func readDependabotSnapshot( + ctx context.Context, cutoff time.Time, execute QueryExecutor, +) (*dependabotSnapshot, error) { + identity := "operational-value-evidence:" + route := dependabotRepository + ":.github/workflows/" + dependabotWorkflow + ".md" + factID := "cutoff:" + cutoff.UTC().Format(time.RFC3339) + definitions := []query.Definition{ + {Name: "operational-value-evidence", From: "$operationalValueEvidence"}, + {Name: "dependabot-cutoff", From: "operational-value-evidence", + Filter: &query.Filter{Predicates: []query.Predicate{ + {Field: "workflow-route", Equals: route}, + {Field: "fact-id", In: []any{"contract", factID}}, + }}}, + } + sources, err := execute(ctx, definitions, []string{"dependabot-cutoff"}) + if err != nil { + return nil, fmt.Errorf("hosted Dependabot canonical evidence query: %w", err) + } + source, ok := sources["dependabot-cutoff"] + if !ok { + return nil, errors.New("hosted Dependabot canonical query did not return its source") + } + if source.Metadata["availability"] != "available" || len(source.Rows) == 0 { + return nil, errDependabotEvidenceUnavailable + } + var result dependabotSnapshot + var captured bool + seen := make(map[string]map[string]bool, len(dependabotMetrics)) + for _, row := range source.Rows { + id := textField(row, "report-id") + index := -1 + for i, metric := range dependabotMetrics { + if id == identity+url.QueryEscape(dependabotRepository+":"+dependabotWorkflow+":"+metric.id+":dependabot-lifecycle-v1") { + index = i + break + } + } + if index < 0 || index >= len(dependabotMetrics) { + return nil, errors.New("unreviewed Dependabot metric packet in canonical query") + } + if seen[id] == nil { + seen[id] = make(map[string]bool, 3) + } + capture, err := time.Parse(time.RFC3339Nano, textField(row, "observed-at")) + if err != nil || capture.Before(cutoff) || capture.After(time.Now()) || + textField(row, "window-end") == "" { + return nil, errors.New("invalid Dependabot evidence capture or cutoff") + } + end, err := time.Parse(time.RFC3339Nano, textField(row, "window-end")) + if err != nil || !end.Equal(cutoff) || captured && !result.capture.Equal(capture) { + return nil, errors.New("dependabot evidence packets disagree on their capture or cutoff") + } + captured, result.capture = true, capture + if textField(row, "contract-digest") != dependabotMetrics[index].digest { + return nil, errors.New("dependabot evidence changed its frozen contract digest") + } + switch textField(row, "fact-kind") { + case "contract": + if seen[id]["contract"] || textField(row, "fact-id") != "contract" || + textField(row, "metric-id") != dependabotMetrics[index].id || + textField(row, "metric-name") != dependabotMetrics[index].name || + textField(row, "metric-unit") != dependabotMetrics[index].unit || + textField(row, "snapshot-id") != dependabotMetrics[index].id || + textField(row, "metric-direction") != "decrease" || + textField(row, "metric-version") != "dependabot-lifecycle-v1" { + return nil, errors.New("dependabot evidence changed its frozen metric contract") + } + seen[id]["contract"] = true + case "snapshot": + at, err := time.Parse(time.RFC3339Nano, textField(row, "at")) + value, valid := safeValueCount(row["value"]) + if seen[id]["snapshot"] || textField(row, "fact-id") != factID || + err != nil || !at.Equal(cutoff) || !valid || + row["items-available"] != false { + return nil, errors.New("dependabot snapshot is not a complete native count") + } + result.values[index], seen[id]["snapshot"] = value, true + case "coverage": + start, startErr := time.Parse(time.RFC3339Nano, textField(row, "at")) + end, endErr := time.Parse(time.RFC3339Nano, textField(row, "end-at")) + if seen[id]["coverage"] || textField(row, "fact-id") != factID || + startErr != nil || endErr != nil || !start.Equal(cutoff.Add(-7*24*time.Hour)) || + !end.Equal(cutoff) || row["coverage-status"] != "complete" { + return nil, errors.New("dependabot evidence is missing complete cutoff coverage") + } + seen[id]["coverage"] = true + default: + return nil, errors.New("unexpected Dependabot fact in scoped cutoff query") + } + } + for _, metric := range dependabotMetrics { + id := identity + url.QueryEscape(dependabotRepository+":"+dependabotWorkflow+":"+metric.id+":dependabot-lifecycle-v1") + if !seen[id]["contract"] || !seen[id]["snapshot"] || !seen[id]["coverage"] { + return nil, errDependabotEvidenceUnavailable + } + } + if result.values[1] > result.values[0] { + return nil, errors.New("critical/high Dependabot alerts exceed all open alerts") + } + encoded, err := json.Marshal(struct { + Values [3]int + Cutoff string + Capture string + }{result.values, optimizerTime(cutoff), optimizerTime(result.capture)}) + if err != nil { + return nil, err + } + hash := sha256.Sum256(encoded) + result.digest = hex.EncodeToString(hash[:]) + return &result, nil +} + +func writeDependabotValue( + ctx context.Context, output *bytes.Buffer, observed time.Time, + remote bool, execute QueryExecutor, +) error { + cutoff, due := dependabotCutoff(observed) + if !due { + return nil + } + var snapshot *dependabotSnapshot + if remote { + var err error + snapshot, err = readDependabotSnapshot(ctx, cutoff, execute) + if err != nil && !errors.Is(err, errDependabotEvidenceUnavailable) { + return err + } + } + if snapshot == nil { + return writeUnavailableDependabotValue(output, observed) + } + for index, metric := range dependabotMetrics { + record := map[string]any{ + "timestamp": optimizerTime(snapshot.capture), "repository": dependabotRepository, + "campaign": "dependabot", "campaign_id": "campaign:dependabot", + "value_id": dependabotWorkflow + "." + metric.id, "value": snapshot.values[index], + "metric_role": metric.role, "metric_name": metric.name, "metric_unit": metric.unit, + "metric_direction": "decrease", "maturity_status": "matured", + "adoption_at": dependabotAdoption, "evaluation_mode": "baseline-comparable", + "workflow_slug": dependabotWorkflow, "workflow_name": "Dependabot / Update Planner", + "evaluator_digest": dependabotEvaluator, "cadence_days": 7, + "window_start": optimizerTime(cutoff.Add(-7 * 24 * time.Hour)), + "window_end": optimizerTime(cutoff), "measurement_kind": "snapshot", + "coverage_status": "complete", "evidence_digest": snapshot.digest, + } + if err := json.NewEncoder(output).Encode(map[string]any{ + "schema_version": 2, "kind": "operational_value", "operational_value": record, + }); err != nil { + return err + } + } + return nil +} diff --git a/server/internal/operationalvalue/dependabot_test.go b/server/internal/operationalvalue/dependabot_test.go new file mode 100644 index 0000000000..2011708fb1 --- /dev/null +++ b/server/internal/operationalvalue/dependabot_test.go @@ -0,0 +1,270 @@ +package operationalvalue + +import ( + "bytes" + "context" + "encoding/json" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/query" +) + +func dependabotCutoffRows(at time.Time, values [3]int) []model.Row { + rows := []model.Row{} + for index, metric := range dependabotMetrics { + id := "operational-value-evidence:" + url.QueryEscape( + dependabotRepository+":"+dependabotWorkflow+":"+metric.id+":dependabot-lifecycle-v1") + common := model.Row{ + "report-id": id, "window-end": optimizerTime(at), + "observed-at": optimizerTime(at.Add(time.Hour)), + "contract-digest": metric.digest, + } + + add := func(fact model.Row) { + row := model.Row{} + for key, value := range common { + row[key] = value + } + for key, value := range fact { + row[key] = value + } + rows = append(rows, row) + } + factID := "cutoff:" + at.Format(time.RFC3339) + add(model.Row{"fact-kind": "contract", "fact-id": "contract", + "metric-id": metric.id, "metric-name": metric.name, "metric-unit": metric.unit, + "metric-direction": "decrease", "metric-version": "dependabot-lifecycle-v1", + "snapshot-id": metric.id}) + add(model.Row{"fact-kind": "snapshot", "fact-id": factID, "at": optimizerTime(at), + "value": json.Number(strconv.Itoa(values[index])), "items-available": false}) + add(model.Row{"fact-kind": "coverage", "fact-id": factID, + "at": optimizerTime(at.Add(-7 * 24 * time.Hour)), + "end-at": optimizerTime(at), "coverage-status": "complete"}) + } + return rows +} + +func TestHostedDependabotRegistrationConsumesOnlyCanonicalCompleteCutoff(t *testing.T) { + root, err := filepath.Abs("../../..") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + policy := filepath.Join(dir, "cao.json") + document := `{"version":1,"gh-aw-version":"v1.0.1","control-plane":{ + "scope":{"allowed-repositories":["github/gh-aw"]}, + "activity":{"repositories":["github/gh-aw"]}, + "campaigns":{ + "dependabot":{"targets":{"github/gh-aw":{"mode":"review"}}, + "workers":{"update-planner":{"workflow":"dependabot-update-planner"}}}, + "activity":{"workflow-values":[{"repository":"github/gh-aw", + "workflow":"dependabot-update-planner", + "collector":"dependabot/operational-value/dependabot-update-planner.mjs", + "api-requests":111,"acquisition":{"requirements":[ + {"source":"github","pool":"core","primitive":"github.dependabot-alert-lifecycle.v1","scope":"github/gh-aw","cache":"invocation"}, + {"source":"github","pool":"search","primitive":"github.dependabot-pull-request-lifecycle.v1","scope":"github/gh-aw","cache":"invocation"}]}}]}}}}` + if err := os.WriteFile(policy, []byte(document), 0o600); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(dir, "snapshot") + if err := os.MkdirAll(snapshot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { + t.Fatal(err) + } + at, _ := time.Parse(time.RFC3339, dependabotAdoption) + at = at.Add(7 * 24 * time.Hour) + config := Config{Root: root, Repositories: []string{dependabotRepository}, + ObservedAt: at.Add(2 * time.Hour), NodeBinary: "/nonexistent-node"} + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + for _, present := range []bool{false, true} { + output := filepath.Join(dir, strconv.FormatBool(present)+".jsonl") + calls := 0 + err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, output, + func(_ context.Context, defs []query.Definition, requested []string) (map[string]model.Source, error) { + calls++ + if len(requested) != 1 { + t.Fatalf("unexpected hosted query requests: %+v", requested) + } + if requested[0] == "dependabot-cutoff" { + if !present { + return map[string]model.Source{"dependabot-cutoff": { + Metadata: model.Metadata{"availability": "available"}, + }}, nil + } + return map[string]model.Source{"dependabot-cutoff": { + Rows: dependabotCutoffRows(at, [3]int{3, 1, 2}), + Metadata: model.Metadata{"availability": "available"}, + }}, nil + } + return map[string]model.Source{requested[0]: { + Metadata: model.Metadata{"availability": "available", "canonical-revision": int64(11)}, + }}, nil + }) + if err != nil || calls != 2 { + t.Fatalf("native local evaluation error=%v calls=%d", err, calls) + } + rows, err := ReadMeasurements(output) + if err != nil || len(rows) != 6 { + t.Fatalf("three Dependabot and three Optimization records expected: %d %v", len(rows), err) + } + var values []json.Number + for _, row := range rows { + if row["campaign"] != "dependabot" { + continue + } + if !present { + if row["value"] != nil || row["coverage-status"] != "missing" { + t.Fatalf("no accepted packet must be unavailable: %+v", row) + } + continue + } + value, ok := row["value"].(json.Number) + if !ok || row["coverage-status"] != "complete" || row["evaluator-digest"] != dependabotEvaluator { + t.Fatalf("complete packet was not evaluated: %+v", row) + } + values = append(values, value) + } + if present && (len(values) != 3 || values[0] != "3" || values[1] != "1" || values[2] != "2") { + t.Fatalf("frozen metric values differ from complete cutoff: %+v", values) + } + } + disabled := strings.Replace(document, `"api-requests":111`, `"api-requests":111,"enabled":false`, 1) + if err := os.WriteFile(policy, []byte(disabled), 0o600); err != nil { + t.Fatal(err) + } + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "disabled.jsonl") + if err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, output, + func(_ context.Context, _ []query.Definition, requested []string) (map[string]model.Source, error) { + if requested[0] == "dependabot-cutoff" { + return map[string]model.Source{"dependabot-cutoff": { + Rows: dependabotCutoffRows(at, [3]int{3, 1, 2}), + Metadata: model.Metadata{"availability": "available"}, + }}, nil + } + return map[string]model.Source{requested[0]: { + Metadata: model.Metadata{"availability": "available", "canonical-revision": int64(11)}, + }}, nil + }); err != nil { + t.Fatal(err) + } + rows, err := ReadMeasurements(output) + if err != nil || len(rows) != 6 { + t.Fatalf("disabled acquisition lost retained evaluation: %d %v", len(rows), err) + } + for _, row := range rows { + if row["campaign"] == "dependabot" && (row["value"] == nil || row["coverage-status"] != "complete") { + t.Fatalf("disabled acquisition erased accepted cutoff: %+v", row) + } + } +} + +func TestDependabotNativeCutoffRequiresCompleteCanonicalTriplet(t *testing.T) { + at, _ := time.Parse(time.RFC3339, dependabotAdoption) + at = at.Add(7 * 24 * time.Hour) + for _, tt := range []struct { + name string + mutate func([]model.Row) []model.Row + missing bool + shouldError bool + }{ + {"complete", func(rows []model.Row) []model.Row { return rows }, false, false}, + {"empty complete", func(rows []model.Row) []model.Row { + for _, row := range rows { + if row["fact-kind"] == "snapshot" { + row["value"] = json.Number("0") + } + } + return rows + }, false, false}, + {"missing metric", func(rows []model.Row) []model.Row { return rows[:6] }, true, false}, + {"missing coverage", func(rows []model.Row) []model.Row { return rows[:8] }, true, false}, + {"incomplete coverage", func(rows []model.Row) []model.Row { + rows[2]["coverage-status"] = "partial" + return rows + }, false, true}, + {"malformed count", func(rows []model.Row) []model.Row { + rows[1]["value"] = json.Number("-1") + return rows + }, false, true}, + {"unreviewed contract", func(rows []model.Row) []model.Row { + rows[0]["contract-digest"] = dependabotMetrics[1].digest + return rows + }, false, true}, + {"high exceeds total", func(rows []model.Row) []model.Row { + rows[4]["value"] = json.Number("9") + return rows + }, false, true}, + {"wrong capture", func(rows []model.Row) []model.Row { + rows[4]["observed-at"] = optimizerTime(at.Add(2 * time.Hour)) + return rows + }, false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + rows := tt.mutate(dependabotCutoffRows(at, [3]int{2, 1, 1})) + calls := 0 + source := func(_ context.Context, defs []query.Definition, requested []string) (map[string]model.Source, error) { + calls++ + if len(defs) != 2 || defs[0].From != "$operationalValueEvidence" || + defs[1].From != "operational-value-evidence" || + defs[1].Filter.Predicates[0].Equals != + dependabotRepository+":.github/workflows/"+dependabotWorkflow+".md" || + len(requested) != 1 || requested[0] != "dependabot-cutoff" { + t.Fatalf("Dependabot did not use an exact canonical query: %+v", defs) + } + return map[string]model.Source{"dependabot-cutoff": { + Rows: rows, Metadata: model.Metadata{"availability": "available"}, + }}, nil + } + var output bytes.Buffer + err := writeDependabotValue(t.Context(), &output, at.Add(2*time.Hour), true, source) + if (err != nil) != tt.shouldError { + t.Fatalf("cutoff error=%v wantError=%v", err, tt.shouldError) + } + if tt.shouldError { + return + } + if calls != 1 { + t.Fatalf("query calls=%d, want one", calls) + } + measurements := bytes.Split(bytes.TrimSpace(output.Bytes()), []byte{'\n'}) + if len(measurements) != 3 { + t.Fatalf("metric count=%d, want three", len(measurements)) + } + for i, line := range measurements { + var envelope measurementEnvelope + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.UseNumber() + if err := decoder.Decode(&envelope); err != nil { + t.Fatal(err) + } + row, err := canonicalMeasurement(envelope) + if err != nil { + t.Fatal(err) + } + if tt.missing { + if row["value"] != nil || row["coverage-status"] != "missing" { + t.Fatalf("incomplete evidence became numeric: %+v", row) + } + } else if row["value"] == nil || row["coverage-status"] != "complete" || + row["valueId"] != dependabotWorkflow+"."+dependabotMetrics[i].id { + t.Fatalf("complete metric is not a native cutoff: %+v", row) + } + } + }) + } +} diff --git a/server/internal/operationalvalue/local.go b/server/internal/operationalvalue/local.go index d042cefb92..09df2c7259 100644 --- a/server/internal/operationalvalue/local.go +++ b/server/internal/operationalvalue/local.go @@ -1,31 +1,28 @@ package operationalvalue import ( - "bufio" + "bytes" + "compress/gzip" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" "io" - "log/slog" - "net" + "math" + "net/url" "os" - "os/exec" "path/filepath" "slices" + "sort" "strings" - "sync" "time" "github.com/githubnext/gh-cao/server/internal/model" "github.com/githubnext/gh-cao/server/internal/query" ) -const ( - hostedQueryRequestBytes = 1 << 20 - hostedQueryResponseBytes = 16 << 20 - hostedQueryLimit = 32 -) +const hostedQueryLimit = 32 type QueryExecutor func(context.Context, []query.Definition, []string) (map[string]model.Source, error) @@ -37,51 +34,388 @@ type LocalValuePlan struct { } func WriteCollectionHealth(ctx context.Context, config Config, policy, snapshot string) error { - arguments := make([]string, 0, 19+2*len(config.Repositories)) - arguments = append(arguments, - filepath.Join(config.Root, "activity", "workflow-values.mjs"), - "--root", config.Root, "--policy", policy, "--defer", "shared-budget-unavailable", - "--database", filepath.Join(snapshot, "unused-value.sqlite"), - "--output", filepath.Join(snapshot, "gh-aw-logs-shards", "workflow-values.jsonl"), - "--inventory", filepath.Join(snapshot, "inventory-sources.json"), - "--workspace", filepath.Join(snapshot, "workflow-value-workspaces"), - "--localRequest", filepath.Join(snapshot, "gh-aw-logs-shards", "workflow-value-local.json"), - "--snapshot", "hosted", - ) + document, reviewed, err := loadValuePolicy(ctx, policy) + if err != nil { + return err + } + shards := filepath.Join(snapshot, "gh-aw-logs-shards") + if err := os.MkdirAll(shards, 0o700); err != nil { + return err + } + var inventory struct { + Repositories model.Source `json:"repositories"` + Workflows model.Source `json:"workflows"` + } + content, err := readBoundedValueFile(filepath.Join(snapshot, "inventory-sources.json"), 16<<20) + if err != nil || json.Unmarshal(content, &inventory) != nil { + return errors.New("invalid canonical Value inventory") + } + type target struct { + Repository string + Workflow string + Collector string + Enabled *bool + Evidence json.RawMessage + } + targets := map[string]target{} + include := func(entry target) { + if valuePolicyInScope(&reviewed, config.Repositories, entry.Repository) { + targets[strings.ToLower(entry.Repository)+":"+entry.Workflow] = entry + } + } + previous := map[string]model.Row{} + healthFile := filepath.Join(shards, "workflow-values-collections.jsonl") + if content, err := readBoundedValueFile(healthFile, 16<<20); err == nil { + for _, line := range bytes.Split(content, []byte{'\n'}) { + if len(bytes.TrimSpace(line)) == 0 { + continue + } + var envelope struct { + SchemaVersion int `json:"schema_version"` + Kind string `json:"kind"` + Record model.Row `json:"workflow_value_collection"` + } + if json.Unmarshal(line, &envelope) != nil || envelope.SchemaVersion != 2 || envelope.Kind != "workflow_value_collection" || + !valueRepositoryPattern.MatchString(textField(envelope.Record, "repository")) || + !valueWorkflowPattern.MatchString(textField(envelope.Record, "workflow")) { + return errors.New("invalid retained Value health") + } + repository, workflow := textField(envelope.Record, "repository"), textField(envelope.Record, "workflow") + include(target{Repository: repository, Workflow: workflow}) + previous[strings.ToLower(repository)+":"+workflow] = envelope.Record + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if inventory.Repositories.Metadata["availability"] != "unavailable" && + inventory.Workflows.Metadata["availability"] != "unavailable" { + repositories := map[string]bool{} + for _, row := range inventory.Repositories.Rows { + owner, name := textField(row, "organization"), textField(row, "repository") + if owner == "" { + owner = textField(row, "owner") + name = textField(row, "name") + } + repositories[strings.ToLower(owner+"/"+name)] = true + } + for _, row := range inventory.Workflows.Rows { + path := textField(row, "workflow") + if path == "" { + path = textField(row, "path") + } + if strings.HasPrefix(path, ".github/workflows/") && strings.HasSuffix(path, ".md") { + workflow := strings.TrimSuffix(strings.TrimPrefix(path, ".github/workflows/"), ".md") + repository := textField(row, "organization") + "/" + textField(row, "repository") + if valueWorkflowPattern.MatchString(workflow) && repositories[strings.ToLower(repository)] { + include(target{Repository: repository, Workflow: workflow}) + } + } + } + } + activity := reviewed.Control.Campaigns["activity"] + for _, entry := range activity.Values { + include(target{Repository: entry.Repository, Workflow: entry.Workflow, + Collector: entry.Collector, Enabled: entry.Enabled, Evidence: entry.Evidence}) + } + if len(targets) > 10000 { + return errors.New("reviewed Value collection inventory exceeds its row budget") + } + now := time.Now().UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z") + digest := valuePolicyDigest(document) + disabled := activity.Enabled != nil && !*activity.Enabled + keys := make([]string, 0, len(targets)) + for key := range targets { + keys = append(keys, key) + } + sort.Strings(keys) + var health, packets bytes.Buffer + for _, key := range keys { + if err := ctx.Err(); err != nil { + return err + } + entry := targets[key] + state, reason, registration := "unconfigured", "unconfigured", "unconfigured" + if entry.Enabled != nil && !*entry.Enabled { + state, reason, registration = "disabled", "disabled", "disabled" + } else if entry.Collector != "" { + registration = "enabled" + if disabled { + state, reason = "disabled", "activity-disabled" + } else { + state, reason = "deferred", "shared-budget-unavailable" + } + } + record := model.Row{ + "schemaVersion": 1, + "id": "workflow-value-collection:" + url.QueryEscape(strings.ToLower(entry.Repository)+":"+entry.Workflow), + "repository": strings.ToLower(entry.Repository), "workflow": entry.Workflow, + "sourcePath": ".github/workflows/" + entry.Workflow + ".md", + "policyDigest": digest, "observedAt": now, + "registration": registration, "state": state, "reasonCode": reason, + "reason": valueHealthReason[reason], + } + if entry.Collector != "" { + record["collector"] = entry.Collector + } + if retained := previous[key]; retained != nil && retained["lastAttemptAt"] != nil { + record["lastAttemptAt"], record["lastAttemptOutcome"] = retained["lastAttemptAt"], retained["lastAttemptOutcome"] + at, err := time.Parse(time.RFC3339Nano, textField(retained, "lastAttemptAt")) + if err != nil { + return errors.New("invalid retained Value attempt time") + } + if registration == "enabled" && !disabled && retained["policyDigest"] == digest && + !at.Before(time.Now().UTC().Truncate(24*time.Hour)) { + previousState, previousReason := textField(retained, "state"), textField(retained, "reasonCode") + if previousState == "available" || previousState == "failed" { + record["state"], record["reasonCode"], record["reason"] = + previousState, previousReason, collectionReason(previousReason) + } + } + } + if err := json.NewEncoder(&health).Encode(map[string]any{ + "schema_version": 2, "kind": "workflow_value_collection", "workflow_value_collection": record, + }); err != nil { + return err + } + if len(entry.Evidence) != 0 && !disabled && registration == "enabled" { + packet, err := readReviewedValuePacket(config.Root, entry) + if err != nil { + return err + } + if err := json.NewEncoder(&packets).Encode(map[string]any{ + "schema_version": 2, "kind": "operational_value_evidence", "operational_value_evidence": packet, + }); err != nil { + return err + } + } + } + if health.Len() > 16<<20 || packets.Len() > 16<<20 { + return errors.New("hosted Value preparation exceeds its byte budget") + } + eligible := make([]string, 0, len(config.Repositories)) for _, repository := range config.Repositories { - arguments = append(arguments, "--repository", repository) + if valuePolicyInScope(&reviewed, config.Repositories, repository) { + eligible = append(eligible, repository) + } + } + request, err := json.Marshal([]any{map[string]any{"schemaVersion": 1, "snapshot": "hosted", + "policyDigest": digest, "eligible": !disabled, + "reason": map[bool]string{true: "activity-disabled", false: "shared-budget-unavailable"}[disabled], + "repositories": eligible}}) + if err != nil { + return err } - return runLocalRuntime(ctx, config, arguments) + for _, output := range []struct { + name string + content []byte + }{ + {healthFile, health.Bytes()}, + {filepath.Join(shards, "workflow-values.jsonl"), packets.Bytes()}, + {filepath.Join(shards, "workflow-value-local.json"), request}, + } { + if err := os.WriteFile(output.name, output.content, 0o600); err != nil { // #nosec G304 -- private hosted transport names. + return err + } + } + return nil } -func ComputeLocalSnapshot(ctx context.Context, config Config, policy, snapshot, retentionDays string) error { - if err := runLocalRuntime(ctx, config, []string{ - filepath.Join(config.Root, "activity", "cao.mjs"), "ingest-jsonl", - "--database", config.Database, "--runs-dir", filepath.Join(snapshot, "gh-aw-logs-runs"), - "--records-dir", filepath.Join(snapshot, "gh-aw-logs-records"), - "--retention-days", retentionDays, "--run-retention-days", retentionDays, - }); err != nil { - return err +var valueHealthReason = map[string]string{ + "unconfigured": "No collector is registered for this workflow in the reviewed policy.", + "disabled": "The reviewed registration explicitly disables future collection; retained evidence is unchanged.", + "activity-disabled": "Activity collection is disabled in the reviewed policy.", + "shared-budget-unavailable": "Automatic remote acquisition is deferred until shared API request budgets can be enforced. Activity retains priority.", + "captured": "Accepted evidence is available at its original capture and cutoff dates.", + "not-due": "The frozen cadence is already covered by retained evidence; no new evidence was acquired.", + "api-budget": "GitHub API capacity is exhausted or reserved for ordinary Activity collection.", + "timeout": "The collector exceeded its time budget; previous accepted evidence is retained.", + "collector-failed": "The collector failed; previous accepted evidence is retained.", + "invalid-evidence": "The collector returned invalid or incomplete evidence; previous accepted evidence is retained.", + "batch-budget": "The collection batch exhausted its bounded time or output budget.", +} + +func collectionReason(reason string) string { return valueHealthReason[reason] } + +// CollectionAttempt updates only the actual-attempt clock and its decision. +// Callers publish this with the accepted packet under the same enrollment +// epoch; a cost record alone is not evidence that a capture succeeded. +func CollectionAttempt(previous model.Row, at time.Time, state, reason string) (model.Row, error) { + if previous == nil || at.IsZero() || (state != "available" && state != "failed") { + return nil, errors.New("invalid Value collection attempt") + } + switch state + ":" + reason { + case "available:captured", "available:not-due", + "failed:api-budget", "failed:timeout", "failed:collector-failed", + "failed:invalid-evidence", "failed:batch-budget": + default: + return nil, errors.New("invalid Value collection attempt reason") } - return runLocalRuntime(ctx, config, []string{ - filepath.Join(config.Root, "activity", "local-values.mjs"), "compute", - "--root", config.Root, "--policy", policy, "--snapshot", snapshot, "--snapshotKey", "hosted", - "--database", config.Database, "--output", config.Output, - }) + if previous["registration"] != "enabled" || !valueRepositoryPattern.MatchString(textField(previous, "repository")) || + !valueWorkflowPattern.MatchString(textField(previous, "workflow")) || + !sha256Fingerprint.MatchString(textField(previous, "policyDigest")) { + return nil, errors.New("value collection attempt has no reviewed registration") + } + at = at.UTC().Truncate(time.Millisecond) + if previousAt := textField(previous, "lastAttemptAt"); previousAt != "" { + prior, err := time.Parse(time.RFC3339Nano, previousAt) + if err != nil || at.Before(prior) { + return nil, errors.New("value collection attempt precedes retained attempt") + } + } + record := make(model.Row, len(previous)) + for key, value := range previous { + record[key] = value + } + record["observedAt"] = optimizerTime(time.Now()) + record["state"], record["reasonCode"], record["reason"] = state, reason, collectionReason(reason) + record["lastAttemptAt"], record["lastAttemptOutcome"] = optimizerTime(at), state + return record, nil +} + +func textField(row model.Row, key string) string { + text, _ := row[key].(string) + return text +} + +func readReviewedValuePacket(root string, entry struct { + Repository string + Workflow string + Collector string + Enabled *bool + Evidence json.RawMessage +}) (model.Row, error) { + var evidence struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + } + if err := json.Unmarshal(entry.Evidence, &evidence); err != nil { + return nil, err + } + path := root + for _, component := range strings.Split(evidence.Path, "/") { + path = filepath.Join(path, component) + info, err := os.Lstat(path) + if err != nil || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("reviewed Value evidence is missing or symlinked") + } + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() > 8<<20 { + return nil, errors.New("reviewed Value evidence is not a bounded regular file") + } + raw, err := os.ReadFile(path) // #nosec G304 -- confined reviewed evidence file with SHA-256 pin. + if err != nil { + return nil, err + } + if strings.HasSuffix(path, ".gz") { + reader, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, err + } + defer func() { _ = reader.Close() }() + raw, err = io.ReadAll(io.LimitReader(reader, (8<<20)+1)) + if err != nil { + return nil, err + } + } + hash := sha256.Sum256(raw) + if len(raw) > 8<<20 || fmt.Sprintf("%x", hash) != evidence.SHA256 { + return nil, errors.New("reviewed Value evidence digest mismatch") + } + var envelope struct { + SchemaVersion int `json:"schema_version"` + Kind string `json:"kind"` + Packet model.Row `json:"operational_value_evidence"` + } + var envelopeFields map[string]json.RawMessage + if json.Unmarshal(raw, &envelopeFields) != nil || len(envelopeFields) != 3 || + json.Unmarshal(raw, &envelope) != nil || envelope.SchemaVersion != 2 || + envelope.Kind != "operational_value_evidence" || + !strings.EqualFold(textField(envelope.Packet, "repository"), entry.Repository) || + textField(envelope.Packet, "workflow") != entry.Workflow || + textField(envelope.Packet, "sourcePath") != ".github/workflows/"+entry.Workflow+".md" { + return nil, errors.New("reviewed Value evidence identity mismatch") + } + capture, err := time.Parse(time.RFC3339Nano, textField(envelope.Packet, "observedAt")) + if err != nil || capture.After(time.Now()) { + return nil, errors.New("reviewed Value evidence has invalid capture time") + } + return envelope.Packet, nil } func ReadLocalValuePlan(ctx context.Context, config Config, policy, snapshot string) (LocalValuePlan, error) { - content, err := runLocalRuntimeOutput(ctx, config, []string{ - filepath.Join(config.Root, "activity", "local-values.mjs"), "plan", - "--policy", policy, "--snapshot", snapshot, "--snapshotKey", "hosted", - }) + document, reviewed, err := loadValuePolicy(ctx, policy) if err != nil { return LocalValuePlan{}, err } - var plan LocalValuePlan - if err := json.Unmarshal(content, &plan); err != nil { - return LocalValuePlan{}, errors.New("local Value plan returned an invalid result") + plan := LocalValuePlan{Repositories: []string{}, DisabledCampaigns: []string{}, + DisabledWorkflows: []string{}, DisabledCollections: []string{}} + content, err := readBoundedValueFile(filepath.Join(snapshot, "gh-aw-logs-shards", "workflow-value-local.json"), 16<<20) + if errors.Is(err, os.ErrNotExist) { + return plan, nil + } + if err != nil { + return plan, errors.New("invalid local Value request") + } + var rawRequests []json.RawMessage + if json.Unmarshal(content, &rawRequests) != nil || len(rawRequests) > 1000 { + return plan, errors.New("invalid local Value requests") + } + var requests []struct { + SchemaVersion int `json:"schemaVersion"` + Snapshot string `json:"snapshot"` + PolicyDigest string `json:"policyDigest"` + Eligible bool `json:"eligible"` + Repositories []string `json:"repositories"` } + if json.Unmarshal(content, &requests) != nil { + return plan, errors.New("invalid local Value requests") + } + repositories := map[string]bool{} + for i, request := range requests { + var fields map[string]json.RawMessage + if json.Unmarshal(rawRequests[i], &fields) != nil || + string(fields["eligible"]) != "true" && string(fields["eligible"]) != "false" { + return plan, errors.New("invalid local Value eligibility request") + } + if request.SchemaVersion != 1 || request.Repositories == nil || len(request.Repositories) > 10000 { + return plan, errors.New("invalid local Value request") + } + if !request.Eligible || request.Snapshot != "hosted" || request.PolicyDigest != valuePolicyDigest(document) { + continue + } + for _, repository := range request.Repositories { + if !valueRepositoryPattern.MatchString(repository) { + return plan, errors.New("invalid local Value repository request") + } + if valuePolicyInScope(&reviewed, request.Repositories, repository) { + repositories[strings.ToLower(repository)] = true + } + } + } + for name, campaign := range reviewed.Control.Campaigns { + if campaign.Enabled != nil && !*campaign.Enabled { + plan.DisabledCampaigns = append(plan.DisabledCampaigns, name) + } + } + for name, workflow := range reviewed.Control.Workflows { + if workflow.Enabled != nil && !*workflow.Enabled { + plan.DisabledWorkflows = append(plan.DisabledWorkflows, name) + } + } + for _, entry := range reviewed.Control.Campaigns["activity"].Values { + if entry.Enabled != nil && !*entry.Enabled { + plan.DisabledCollections = append(plan.DisabledCollections, strings.ToLower(entry.Repository)+":"+entry.Workflow) + } + } + if activity := reviewed.Control.Campaigns["activity"]; activity.Enabled == nil || *activity.Enabled { + for repository := range repositories { + plan.Repositories = append(plan.Repositories, repository) + } + } + sort.Strings(plan.Repositories) + sort.Strings(plan.DisabledCampaigns) + sort.Strings(plan.DisabledWorkflows) return plan, nil } @@ -94,197 +428,398 @@ func ComputeHostedSnapshot( if execute == nil { return errors.New("hosted local Value requires the canonical PostgreSQL query boundary") } - bridge, err := startHostedQueryBridge(ctx, execute) + plan, err := ReadLocalValuePlan(ctx, config, policy, snapshot) if err != nil { return err } - defer bridge.Close() - config.Environment = append(slices.Clone(config.Environment), "CAO_DASHBOARD_QUERY_SOCKET="+bridge.path) - return runLocalRuntime(ctx, config, []string{ - filepath.Join(config.Root, "activity", "local-values.mjs"), "compute", - "--root", config.Root, "--policy", policy, "--snapshot", snapshot, "--snapshotKey", "hosted", - "--output", output, - }) -} - -func runLocalRuntime(ctx context.Context, config Config, arguments []string) error { - content, err := runLocalRuntimeOutput(ctx, config, arguments) + if len(plan.Repositories) == 0 { + return os.WriteFile(output, nil, 0o600) + } + if err := verifyHostedLocalAdapters(config.Root, plan.DisabledCampaigns); err != nil { + return err + } + _, reviewed, err := loadValuePolicy(ctx, policy) if err != nil { return err } - if len(content) != 0 { - var result struct { - Warnings []json.RawMessage `json:"warnings"` + optimizationEnabled := !slices.Contains(plan.DisabledCampaigns, "optimization") && + !slices.Contains(plan.DisabledWorkflows, "optimization-token-optimizer") + // This is the one reviewed local-only adapter. Its implementation and + // fingerprint are pinned against the independent JS oracle in tests. + const evaluatorDigest = "5932b3eb53899fd949e92c6219d562ea853808cfb54d152e0c279ba41e48eacb" + const adoptionAt = "2026-09-15T23:30:36Z" + adoption, _ := time.Parse(time.RFC3339, adoptionAt) + supported := map[string]bool{ + "github/gh-aw": true, "github/gh-aw-actions": true, "github/gh-aw-firewall": true, + "github/gh-aw-mcpg": true, "github/gh-aw-threat-detection": true, "githubnext/gh-cao": true, + } + observed := time.Now().UTC().Truncate(time.Millisecond) + if !config.ObservedAt.IsZero() { + observed = config.ObservedAt.UTC().Truncate(time.Millisecond) + } + type window struct { + repository string + start, end time.Time + } + windows := make([]window, 0, len(plan.Repositories)) + for _, repository := range plan.Repositories { + if !optimizationEnabled { + break + } + if !supported[repository] || + slices.Contains(plan.DisabledCollections, repository+":optimization-token-optimizer") { + continue + } + end := adoption.Add(time.Duration(math.Floor(observed.Sub(adoption).Hours()/24)) * 24 * time.Hour) + windows = append(windows, window{repository, end.Add(-7 * 24 * time.Hour), end}) + } + if len(windows) > hostedQueryLimit { + return errors.New("hosted local Value exceeded its canonical query budget") + } + metricFields := []struct { + id, name, unit string + role string + }{ + {"aic-per-successful-run", "AI Credit per successful run", "aic-per-run", "primary"}, + {"failure-rate-percent", "Failure rate", "percent", "diagnostic"}, + {"cancellation-rate-percent", "Cancellation rate", "percent", "diagnostic"}, + } + var result bytes.Buffer + for _, selected := range windows { + if err := ctx.Err(); err != nil { + return err + } + byConclusion := func(values ...string) *query.Filter { + items := make([]any, len(values)) + for i, value := range values { + items[i] = value + } + return &query.Filter{Predicates: []query.Predicate{{Field: "conclusion", In: items}}} } - if err := json.Unmarshal(content, &result); err != nil { - return errors.New("local Value runtime returned an invalid result") + success := byConclusion("success") + definition := query.Definition{ + Name: "optimization-repository-efficiency", From: "runs", + Joins: []query.Join{{Source: "workflows", Type: "inner", + On: []query.JoinKey{{Left: "workflowId", Right: "id"}}, + Fields: []query.SelectedField{{Field: "path", As: "declaredWorkflowPath"}}}}, + Filter: &query.Filter{Predicates: []query.Predicate{ + {Field: "startedAt", GTE: selected.start.UTC().Format(time.RFC3339Nano)}, + {Field: "startedAt", LT: selected.end.UTC().Format(time.RFC3339Nano)}, + {Field: "declaredWorkflowPath", Includes: ".md"}, + {Field: "owner", Equals: strings.Split(selected.repository, "/")[0]}, + {Field: "repository", Equals: strings.Split(selected.repository, "/")[1]}, + }}, + Aggregate: &query.Aggregate{ + By: []string{"owner", "repository", "declaredWorkflowPath"}, + Values: []query.AggregateValue{ + {Field: "githubRunId", As: "concludedRuns", Reducer: "count", Filter: byConclusion("success", "failure", "cancelled", "timed_out", "startup_failure", "action_required", "neutral", "skipped", "stale")}, + {Field: "githubRunId", As: "successfulRuns", Reducer: "count", Filter: success}, + {Field: "aicTotal", As: "successfulRunsWithAic", Reducer: "count", Filter: success}, + {Field: "aicTotal", As: "aicPerSuccessfulRun", Reducer: "mean", Filter: success}, + {Field: "githubRunId", As: "failedRuns", Reducer: "count", Filter: byConclusion("failure", "timed_out", "startup_failure")}, + {Field: "githubRunId", As: "cancelledRuns", Reducer: "count", Filter: byConclusion("cancelled")}, + }, + }, } - if len(result.Warnings) != 0 { - slog.WarnContext(ctx, "local Value adapters deferred or failed; only completed evidence retained", - "warning_count", len(result.Warnings)) + sources, err := execute(ctx, []query.Definition{definition}, []string{definition.Name}) + if err != nil { + return fmt.Errorf("hosted canonical Value query: %w", err) + } + source, ok := sources[definition.Name] + if !ok || source.Metadata["availability"] != "available" { + return errors.New("hosted canonical Value query is unavailable") + } + revision, ok := source.Metadata["canonical-revision"] + if !ok { + return errors.New("hosted canonical Value query did not identify its revision") + } + provenance := []struct { + Repository string `json:"repository"` + Kind string `json:"kind"` + Ref string `json:"ref"` + }{{selected.repository, "cao-canonical-query-sqlite-sha256", + valuePolicyDigest([]byte(fmt.Sprint(revision)))}} + eligibleWorkflows := []string{} + var successes, failures, cancelled, concluded int + var aicTotal float64 + for _, row := range source.Rows { + path := textField(row, "declaredWorkflowPath") + if strings.Contains(path, "/optimization-") || strings.HasSuffix(path, "/optimization.md") { + continue + } + total, totalOK := safeValueCount(row["concludedRuns"]) + successCount, successOK := safeValueCount(row["successfulRuns"]) + withAIC, aicOK := safeValueCount(row["successfulRunsWithAic"]) + failedCount, failedOK := safeValueCount(row["failedRuns"]) + cancelCount, cancelOK := safeValueCount(row["cancelledRuns"]) + mean, meanOK := safeValueNumber(row["aicPerSuccessfulRun"]) + if path == "" || !totalOK || !successOK || !aicOK || !failedOK || + !cancelOK || !meanOK || mean < 0 || total < 3 || successCount == 0 || withAIC != successCount { + continue + } + eligibleWorkflows = append(eligibleWorkflows, selected.repository+"/"+path) + successes += successCount + concluded += total + failures += failedCount + cancelled += cancelCount + aicTotal += mean * float64(successCount) + } + sort.Strings(eligibleWorkflows) + populationJSON, err := json.Marshal(eligibleWorkflows) + if err != nil { + return err + } + const opportunity = "A declared non-Optimization agentic workflow in the authorized target repository with at least three completed runs, at least one successful run, and complete successful-run AI Credit coverage in the observation window." + filters := []string{ + "Measure runs in the authorized target repository, never the control repository as a proxy.", + "Include declared agentic workflow sources and exclude the Optimization campaign itself.", + "Require at least three completed runs and at least one successful run in the observation window.", + "Require authoritative AI Credit for every successful run in the observation window.", + "Measure native AI Credit per successful run, failed-run percentage, and cancelled-run percentage without normalizing them to a synthetic score.", + } + const collection = "Execute one bounded canonical Activity query over target-repository runs in the observation window, grouped by declared workflow. Apply the identical native formulas before and after adoption because AI Credit and run conclusions predate the Optimization workflow. Aggregate only eligible workflow rows into native repository measures. No eligible workflow or incomplete AI Credit evidence is missing." + type evidenceWindow struct { + Start string `json:"start"` + End string `json:"end"` + ObservedAt string `json:"observedAt"` + DurationDays int `json:"durationDays"` + CadenceDays int `json:"cadenceDays"` + MaturationDays int `json:"maturationDays"` + } + type evidenceRecord struct { + Key string `json:"key"` + Repositories []string `json:"repositories"` + Opportunity string `json:"opportunity"` + Filters []string `json:"filters"` + Collection string `json:"collection"` + Window evidenceWindow `json:"window"` + MaturityStatus string `json:"maturityStatus"` + Dubious bool `json:"dubious"` + CoverageStatus string `json:"coverageStatus"` + MissingReason string `json:"missingReason"` + EligibleCount int `json:"eligibleCount"` + PopulationDigest string `json:"populationDigest"` + EligibleWorkflowCount int `json:"eligibleWorkflowCount"` + SuccessfulRunCount int `json:"successfulRunCount"` + SuccessfulRunAicTotal float64 `json:"successfulRunAicTotal"` + ConcludedRunCount int `json:"concludedRunCount"` + FailedRunCount int `json:"failedRunCount"` + CancelledRunCount int `json:"cancelledRunCount"` + FailedRunPercentagePointTotal int `json:"failedRunPercentagePointTotal"` + CancelledRunPercentagePointTotal int `json:"cancelledRunPercentagePointTotal"` + } + status := "matured" + if selected.end.Sub(selected.start) < 7*24*time.Hour { + status = "interim" + } + missingReason := "source-window-coverage-unverified" + if len(eligibleWorkflows) == 0 { + missingReason = "no-eligible-opportunities" + } + evidence := evidenceRecord{ + "target-repository-agentic-workflow-efficiency", []string{selected.repository}, + opportunity, filters, collection, + evidenceWindow{optimizerTime(selected.start), optimizerTime(selected.end), + optimizerTime(observed), 7, 1, 0}, + status, status == "interim", "unknown", missingReason, + len(eligibleWorkflows), valuePolicyDigest(populationJSON), len(eligibleWorkflows), + successes, optimizerRound(aicTotal), concluded, failures, cancelled, failures * 100, cancelled * 100, + } + evidenceJSON, err := json.Marshal(struct { + Evidence evidenceRecord `json:"evidence"` + Provenance any `json:"provenance"` + }{evidence, provenance}) + if err != nil { + return err + } + for _, metric := range metricFields { + var value any + var numerator, denominator float64 + switch metric.id { + case "aic-per-successful-run": + numerator, denominator = evidence.SuccessfulRunAicTotal, float64(successes) + case "failure-rate-percent": + numerator, denominator = float64(failures*100), float64(concluded) + case "cancellation-rate-percent": + numerator, denominator = float64(cancelled*100), float64(concluded) + } + if len(eligibleWorkflows) != 0 && denominator > 0 { + value = optimizerRound(numerator / denominator) + } + maturity := status + if value == nil { + maturity = "unavailable" + } + record := map[string]any{ + "timestamp": optimizerTime(observed), "repository": selected.repository, + "campaign": "optimization", "campaign_id": "campaign:optimization", + "value_id": "optimization-token-optimizer." + metric.id, "value": value, + "metric_role": metric.role, "metric_name": metric.name, "metric_unit": metric.unit, + "metric_direction": "decrease", "maturity_status": maturity, + "adoption_at": adoptionAt, "evaluation_mode": "baseline-comparable", + "workflow_slug": "optimization-token-optimizer", "workflow_name": "Optimization / Token Optimizer", + "evaluator_digest": evaluatorDigest, "cadence_days": 1, + "window_start": optimizerTime(selected.start), "window_end": optimizerTime(selected.end), + "measurement_kind": "interval", "coverage_status": "unknown", "missing_reason": missingReason, + "eligible_count": len(eligibleWorkflows), "population_digest": evidence.PopulationDigest, + "evidence_digest": valuePolicyDigest(evidenceJSON), + } + if value != nil { + record["rollup_numerator"], record["rollup_denominator"] = numerator, denominator + } + if err := json.NewEncoder(&result).Encode(map[string]any{ + "schema_version": 2, "kind": "operational_value", "operational_value": record, + }); err != nil { + return err + } } } - return nil -} - -func runLocalRuntimeOutput(ctx context.Context, config Config, arguments []string) ([]byte, error) { - timeout := config.WorkerTimeout - if timeout <= 0 { - timeout = 2 * time.Minute - } - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - node := config.NodeBinary - if node == "" { - node = "node" - } - command := exec.CommandContext(ctx, node, arguments...) // #nosec G204 -- fixed trusted local runtime and explicit paths. - command.Dir = config.Root - command.Env = slices.DeleteFunc(slices.Clone(config.Environment), func(value string) bool { - return strings.HasPrefix(value, "GH_TOKEN=") || strings.HasPrefix(value, "GITHUB_TOKEN=") || - strings.HasPrefix(value, "CAO_OPERATIONAL_VALUE_GH_TOKEN=") - }) - command.Env = append(command.Env, "CAO_VALUE_SHARED_PROCESS_GROUP=1") - configureProcessTree(command) - command.Cancel = func() error { return terminateProcessTree(command.Process) } - command.WaitDelay = 5 * time.Second - stdout, stderr := newLimitedBuffer(1<<20, cancel), newLimitedBuffer(16384, cancel) - command.Stdout, command.Stderr = stdout, stderr - err := command.Run() - _ = terminateProcessTree(command.Process) - if err != nil || stdout.Exceeded() || stderr.Exceeded() { - return nil, errors.New("local Value runtime failed or exceeded its budget; accepted evidence retained") - } - return stdout.Bytes(), nil -} - -type hostedQueryBridge struct { - path string - listener net.Listener - cancel context.CancelFunc - done chan struct{} + if reviewedDependabotLocalValue(reviewed, plan) { + for _, repository := range plan.Repositories { + if repository == "github/gh-aw" { + if err := writeDependabotValue(ctx, &result, observed, + reviewedDependabotEvidence(reviewed), execute); err != nil { + return err + } + } + } + } + if result.Len() > 16<<20 { + return errors.New("hosted local Value output exceeds its byte budget") + } + return os.WriteFile(output, result.Bytes(), 0o600) // #nosec G304 -- private hosted output path. } -type hostedQueryRequest struct { - SchemaVersion int `json:"schemaVersion"` - Queries []query.Definition `json:"queries"` - Requested []string `json:"requested"` +func reviewedDependabotLocalValue(reviewed valuePolicy, plan LocalValuePlan) bool { + if slices.Contains(plan.DisabledCampaigns, "dependabot") || + slices.Contains(plan.DisabledWorkflows, "dependabot-update-planner") { + return false + } + campaign, ok := reviewed.Control.Campaigns["dependabot"] + if !ok { + return false + } + worker, ok := campaign.Workers["update-planner"] + if !ok || worker.Workflow != "dependabot-update-planner" || + worker.Enabled != nil && !*worker.Enabled { + return false + } + target, ok := campaign.Targets["github/gh-aw"] + return ok && (target.Mode == "review" || target.Mode == "live") } -type hostedQueryResponse struct { - SchemaVersion int `json:"schemaVersion"` - Sources map[string]model.Source `json:"sources,omitempty"` - Error string `json:"error,omitempty"` +func writeUnavailableDependabotValue(output *bytes.Buffer, observed time.Time) error { + const adoptionAt = "2026-09-17T08:48:00Z" + adoption, _ := time.Parse(time.RFC3339, adoptionAt) + cadence := 7 * 24 * time.Hour + if observed.Before(adoption.Add(cadence)) { + return nil + } + end := adoption.Add(time.Duration(math.Floor(observed.Sub(adoption).Hours()/cadence.Hours())) * cadence) + for _, metric := range dependabotMetrics { + record := map[string]any{ + "timestamp": optimizerTime(observed), "repository": "github/gh-aw", + "campaign": "dependabot", "campaign_id": "campaign:dependabot", + "value_id": "dependabot-update-planner." + metric.id, "value": nil, + "metric_role": metric.role, "metric_name": metric.name, "metric_unit": metric.unit, + "metric_direction": "decrease", "maturity_status": "unavailable", + "adoption_at": adoptionAt, "evaluation_mode": "baseline-comparable", + "workflow_slug": "dependabot-update-planner", "workflow_name": "Dependabot / Update Planner", + "evaluator_digest": dependabotEvaluator, + "cadence_days": 7, "window_start": optimizerTime(end.Add(-cadence)), + "window_end": optimizerTime(end), "measurement_kind": "snapshot", + "coverage_status": "missing", "missing_reason": "remote-acquisition-unavailable", + } + if err := json.NewEncoder(output).Encode(map[string]any{ + "schema_version": 2, "kind": "operational_value", "operational_value": record, + }); err != nil { + return err + } + } + return nil } -func startHostedQueryBridge(ctx context.Context, execute QueryExecutor) (*hostedQueryBridge, error) { - directory, err := os.MkdirTemp("", "cao-value-query-") +func verifyHostedLocalAdapters(root string, disabled []string) error { + directories, err := os.ReadDir(root) if err != nil { - return nil, err - } - path := filepath.Join(directory, "query.sock") - listener, err := (&net.ListenConfig{}).Listen(ctx, "unix", path) - if err != nil { - _ = os.RemoveAll(directory) - return nil, err + return err } - if err := os.Chmod(path, 0o600); err != nil { - _ = listener.Close() - _ = os.RemoveAll(directory) - return nil, err + if len(directories) > 256 { + return errors.New("hosted local Value catalog exceeds its campaign budget") } - bridgeCtx, cancel := context.WithCancel(ctx) - bridge := &hostedQueryBridge{ - path: path, listener: listener, cancel: cancel, done: make(chan struct{}), + for _, directory := range directories { + if !directory.IsDir() || directory.Name() == "activity" || + slices.Contains(disabled, directory.Name()) { + continue + } + wrapper := filepath.Join(root, directory.Name(), "operational-value.mjs") + wrapperBytes, wrapperErr := readBoundedValueFile(wrapper, 1<<20) + if errors.Is(wrapperErr, os.ErrNotExist) { + continue + } + if wrapperErr != nil { + return wrapperErr + } + if fmt.Sprintf("%x", sha256.Sum256(wrapperBytes)) != "771ab1363f035e771684fc334c38dfeeee1855d691758055d4637373d3617416" { + return errors.New("hosted local Value wrapper differs from its reviewed Go implementation") + } + if directory.Name() != "optimization" && directory.Name() != "dependabot" { + return errors.New("hosted local Value campaign is not supported by the Go evaluator") + } + path := filepath.Join(root, directory.Name(), "operational-value") + modules, err := os.ReadDir(path) + if err != nil { + return err + } + for _, module := range modules { + if !strings.HasSuffix(module.Name(), ".mjs") { + continue + } + expected := "4c69f080d1e7c1d07ee2201585bacefbdbbba652d75163f7bcaed19d294e1546" + name := "optimization-token-optimizer.mjs" + if directory.Name() == "dependabot" { + expected, name = "bb95c37d4e5a7204f5587f1b7732603f88da9fdf7e50933492bc6af90c97d0b9", + "dependabot-update-planner.mjs" + } + if module.Name() != name || !module.Type().IsRegular() { + return errors.New("hosted local Value adapter is not supported by the Go evaluator") + } + content, err := readBoundedValueFile(filepath.Join(path, module.Name()), 1<<20) + if err != nil || + fmt.Sprintf("%x", sha256.Sum256(content)) != expected { + return errors.New("hosted local Value adapter differs from its reviewed Go implementation") + } + } } - go bridge.serve(bridgeCtx, execute) - return bridge, nil + return nil } -func (b *hostedQueryBridge) Close() { - b.cancel() - _ = b.listener.Close() - <-b.done - _ = os.RemoveAll(filepath.Dir(b.path)) +func optimizerTime(value time.Time) string { + return value.UTC().Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z") } -func (b *hostedQueryBridge) serve(ctx context.Context, execute QueryExecutor) { - defer close(b.done) - var queries int - var mutex sync.Mutex - for { - connection, err := b.listener.Accept() - if err != nil { - return - } - queries++ - if queries > hostedQueryLimit { - _ = writeHostedQueryResponse(connection, hostedQueryResponse{ - SchemaVersion: 1, Error: "Hosted local Value exceeded its canonical query budget", - }) - _ = connection.Close() - continue - } - mutex.Lock() - b.serveConnection(ctx, connection, execute) - mutex.Unlock() - } +func optimizerRound(value float64) float64 { + return math.Floor(value*1e9+0.5) / 1e9 } -func (b *hostedQueryBridge) serveConnection( - ctx context.Context, - connection net.Conn, - execute QueryExecutor, -) { - defer func() { _ = connection.Close() }() - _ = connection.SetDeadline(time.Now().Add(30 * time.Second)) - content, err := io.ReadAll(io.LimitReader(connection, hostedQueryRequestBytes+1)) - if err != nil || len(content) > hostedQueryRequestBytes { - _ = writeHostedQueryResponse(connection, hostedQueryResponse{ - SchemaVersion: 1, Error: "Hosted local Value exceeded its canonical query byte budget", - }) - return - } - decoder := json.NewDecoder(bufio.NewReader(strings.NewReader(string(content)))) - decoder.DisallowUnknownFields() - var request hostedQueryRequest - if err := decoder.Decode(&request); err != nil || request.SchemaVersion != 1 || - len(request.Queries) == 0 || len(request.Queries) > query.MaxPlanQueries || - len(request.Requested) == 0 || len(request.Requested) > query.MaxPlanQueries { - _ = writeHostedQueryResponse(connection, hostedQueryResponse{ - SchemaVersion: 1, Error: "Hosted local Value sent an invalid canonical query", - }) - return - } - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - _ = writeHostedQueryResponse(connection, hostedQueryResponse{ - SchemaVersion: 1, Error: "Hosted local Value sent multiple canonical queries", - }) - return - } - queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - sources, err := execute(queryCtx, request.Queries, request.Requested) - response := hostedQueryResponse{SchemaVersion: 1, Sources: sources} - if err != nil { - response.Sources = nil - response.Error = fmt.Sprintf("Canonical PostgreSQL query failed: %v", err) +func safeValueNumber(value any) (float64, bool) { + number, ok := value.(json.Number) + if ok { + result, err := number.Float64() + return result, err == nil && !math.IsNaN(result) && !math.IsInf(result, 0) } - _ = writeHostedQueryResponse(connection, response) + switch number := value.(type) { + case float64: + return number, !math.IsNaN(number) && !math.IsInf(number, 0) + case int64: + return float64(number), true + case int: + return float64(number), true + } + return 0, false } -func writeHostedQueryResponse(connection net.Conn, response hostedQueryResponse) error { - content, err := json.Marshal(response) - if err != nil { - return err - } - if len(content) > hostedQueryResponseBytes { - content, _ = json.Marshal(hostedQueryResponse{ - SchemaVersion: 1, Error: "Canonical PostgreSQL query response exceeded its byte budget", - }) - } - content = append(content, '\n') - _, err = connection.Write(content) - return err +func safeValueCount(value any) (int, bool) { + number, ok := safeValueNumber(value) + return int(number), ok && number >= 0 && number <= math.MaxInt32 && math.Trunc(number) == number } diff --git a/server/internal/operationalvalue/local_test.go b/server/internal/operationalvalue/local_test.go index 91565f6bfa..89591c0754 100644 --- a/server/internal/operationalvalue/local_test.go +++ b/server/internal/operationalvalue/local_test.go @@ -3,8 +3,6 @@ package operationalvalue import ( "context" "encoding/json" - "io" - "net" "os" "path/filepath" "strings" @@ -59,99 +57,103 @@ func TestLocalCollectionHealthNeedsNeitherCollectorNorCredential(t *testing.T) { } } -func TestLocalRuntimeCancellationOwnsDescendantsAndStripsCredentials(t *testing.T) { - directory := t.TempDir() - started, leaked := filepath.Join(directory, "started"), filepath.Join(directory, "leaked") - marker, _ := json.Marshal(started) - target, _ := json.Marshal(leaked) - child := `require("node:fs").writeFileSync(` + string(marker) + `,"ready"); - setTimeout(()=>require("node:fs").writeFileSync(` + string(target) + `,"leaked"),500); - setInterval(()=>{},1000)` - childSource, _ := json.Marshal(child) - source := `if(process.env.GH_TOKEN || process.env.GITHUB_TOKEN || process.env.CAO_OPERATIONAL_VALUE_GH_TOKEN) - throw new Error("credentials crossed local boundary"); - require("node:child_process").spawn(process.execPath,["-e",` + string(childSource) + `]); - setInterval(()=>{},1000)` - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { - done <- runLocalRuntime(ctx, Config{Root: directory, WorkerTimeout: 5 * time.Second, - Environment: []string{"PATH=" + os.Getenv("PATH"), "GH_TOKEN=test-only", "GITHUB_TOKEN=test-only", - "CAO_OPERATIONAL_VALUE_GH_TOKEN=test-only"}}, - []string{"-e", source}) - }() - deadline := time.Now().Add(3 * time.Second) - for { - if _, err := os.Stat(started); err == nil { - break - } - select { - case err := <-done: - t.Fatalf("local child did not start with credentials removed: %v", err) - default: - } - if time.Now().After(deadline) { - t.Fatal("local runtime did not start within its bounded deadline") - } - time.Sleep(10 * time.Millisecond) +func TestHostedHealthPreservesActualAttemptAcrossPreparation(t *testing.T) { + root := t.TempDir() + policy := filepath.Join(root, "cao.json") + content := `{"version":1,"gh-aw-version":"v0.93.1","control-plane":{ + "scope":{"allowed-repositories":["org/repo"]},"activity":{"repositories":["org/repo"]}, + "campaigns":{"activity":{"workflow-values":[ + {"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs"}]}}}}` + if err := os.WriteFile(policy, []byte(content), 0o600); err != nil { + t.Fatal(err) } - cancel() - if err := <-done; err == nil { - t.Fatal("canceled local work must report failure") + snapshot := filepath.Join(root, "transport") + if err := os.MkdirAll(snapshot, 0o700); err != nil { + t.Fatal(err) } - time.Sleep(600 * time.Millisecond) - if _, err := os.Stat(leaked); !os.IsNotExist(err) { - t.Fatal("a descendant survived local runtime cancellation") + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { + t.Fatal(err) + } + config := Config{Root: root, Repositories: []string{"org/repo"}, NodeBinary: "/nonexistent-node"} + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + file := filepath.Join(snapshot, "gh-aw-logs-shards", "workflow-values-collections.jsonl") + data, err := os.ReadFile(file) // #nosec G304 -- file is under this test's TempDir. + if err != nil { + t.Fatal(err) + } + var envelope struct { + Record model.Row `json:"workflow_value_collection"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + t.Fatal(err) + } + attempted := time.Now().UTC().Truncate(time.Millisecond) + record, err := CollectionAttempt(envelope.Record, attempted, "available", "captured") + if err != nil { + t.Fatal(err) + } + envelopeData, err := json.Marshal(map[string]any{ + "schema_version": 2, "kind": "workflow_value_collection", + "workflow_value_collection": record, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, append(envelopeData, '\n'), 0o600); err != nil { + t.Fatal(err) + } + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + data, err = os.ReadFile(file) // #nosec G304 -- file is under this test's TempDir. + if err != nil || json.Unmarshal(data, &envelope) != nil || + envelope.Record["state"] != "available" || envelope.Record["reasonCode"] != "captured" || + envelope.Record["lastAttemptAt"] != optimizerTime(attempted) { + t.Fatalf("preparation concealed a real collector attempt: %v %s", err, data) } } func TestHostedSnapshotQueriesCanonicalDataWithoutSQLite(t *testing.T) { root := t.TempDir() - activity := filepath.Join(root, "activity") - if err := os.MkdirAll(activity, 0o700); err != nil { - t.Fatal(err) - } - script := `import { connect } from "node:net"; -import { writeFile } from "node:fs/promises"; -const args = process.argv.slice(2); -if (args.includes("--database")) throw new Error("hosted execution received a SQLite database"); -const output = args[args.indexOf("--output") + 1]; -const socket = connect(process.env.CAO_DASHBOARD_QUERY_SOCKET); -let response = ""; -socket.setEncoding("utf8"); -socket.on("data", chunk => response += chunk); -const done = new Promise((resolve, reject) => { - socket.on("end", resolve); - socket.on("error", reject); -}); -socket.end(JSON.stringify({schemaVersion: 1, queries: [{name:"rows", from:"runs"}], requested:["rows"]})); -await done; -const parsed = JSON.parse(response); -if (parsed.sources.rows.rows[0].repository !== "api") throw new Error("canonical rows unavailable"); -await writeFile(output, JSON.stringify({schema_version:2,kind:"operational_value",operational_value:{ - timestamp:"2026-01-01T00:00:00Z",repository:"octo/api",campaign:"optimization", - campaign_id:"campaign:optimization",value_id:"worker.efficiency",value:1 -}}) + "\n"); -console.log(JSON.stringify({values:1,warnings:[]}));` - if err := os.WriteFile(filepath.Join(activity, "local-values.mjs"), []byte(script), 0o600); err != nil { + policy := filepath.Join(root, "cao.json") + if err := os.WriteFile(policy, []byte(`{"version":1,"gh-aw-version":"v0.93.1", + "control-plane":{"scope":{"allowed-repositories":["githubnext/gh-cao"]}, + "activity":{"repositories":["githubnext/gh-cao"]}}}`), 0o600); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(root, "snapshot") + if err := os.MkdirAll(snapshot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { + t.Fatal(err) + } + observed, _ := time.Parse(time.RFC3339, "2026-11-05T20:00:00Z") + config := Config{Root: root, Repositories: []string{"githubnext/gh-cao"}, + NodeBinary: "/nonexistent-node", ObservedAt: observed} + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { t.Fatal(err) } output := filepath.Join(t.TempDir(), "operational-values.jsonl") queries := 0 - err := ComputeHostedSnapshot(t.Context(), Config{ - Root: root, NodeBinary: "node", Output: output, - Environment: []string{"PATH=" + os.Getenv("PATH")}, WorkerTimeout: 10 * time.Second, - }, "unused-policy", "unused-snapshot", output, + err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, output, func(_ context.Context, definitions []query.Definition, requested []string) (map[string]model.Source, error) { queries++ - if len(definitions) != 1 || definitions[0].Name != "rows" || - len(requested) != 1 || requested[0] != "rows" { + if len(definitions) != 1 || definitions[0].Name != "optimization-repository-efficiency" || + len(definitions[0].Joins) != 1 || definitions[0].Aggregate == nil || + len(requested) != 1 || requested[0] != "optimization-repository-efficiency" { t.Fatalf("unexpected canonical query: %#v %#v", definitions, requested) } - return map[string]model.Source{"rows": { - Source: "rows", Rows: []model.Row{{"repository": "api"}}, - Metadata: model.Metadata{"availability": "available"}, + return map[string]model.Source{requested[0]: { + Source: requested[0], Rows: []model.Row{{"owner": "githubnext", "repository": "gh-cao", + "declaredWorkflowPath": ".github/workflows/worker.md", "concludedRuns": 3, + "successfulRuns": 3, "successfulRunsWithAic": 3, "aicPerSuccessfulRun": 2.5, + "failedRuns": 0, "cancelledRuns": 0}}, + Metadata: model.Metadata{"availability": "available", "canonical-revision": int64(42)}, }}, nil }) if err != nil { @@ -160,6 +162,17 @@ console.log(JSON.stringify({values:1,warnings:[]}));` if queries != 1 { t.Fatalf("canonical query count = %d, want 1", queries) } + rows, err := ReadMeasurements(output) + if err != nil || len(rows) != 3 || rows[0]["repository"] != "githubnext/gh-cao" || + rows[0]["value"] != json.Number("2.5") || + rows[0]["evidence-digest"] != "33fb3591bb9abb77f977f73e85b162b3d9035554e4374334698e82fc5978d136" { + t.Fatalf("hosted result is not a canonical native assessment: %+v %v", rows, err) + } + for _, row := range rows { + if row["coverage-status"] != "unknown" || row["evaluator-digest"] == nil { + t.Fatalf("hosted Value promoted unverified coverage: %+v", row) + } + } if _, err := os.Stat(output); err != nil { t.Fatal(err) } @@ -171,44 +184,177 @@ console.log(JSON.stringify({values:1,warnings:[]}));` }); err != nil { t.Fatal(err) } + missingOutput := filepath.Join(t.TempDir(), "missing.jsonl") + if err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, missingOutput, + func(_ context.Context, definitions []query.Definition, requested []string) (map[string]model.Source, error) { + return map[string]model.Source{requested[0]: {Metadata: model.Metadata{ + "availability": "available", "canonical-revision": int64(42), + }}}, nil + }); err != nil { + t.Fatal(err) + } + missing, err := ReadMeasurements(missingOutput) + if err != nil || len(missing) != 3 { + t.Fatalf("missing canonical opportunity was not evaluated: %+v %v", missing, err) + } + for _, row := range missing { + if row["value"] != nil || row["maturity-status"] != "unavailable" || + row["missing-reason"] != "no-eligible-opportunities" { + t.Fatalf("missing opportunities became a fabricated zero: %+v", row) + } + } + if err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, + filepath.Join(t.TempDir(), "unavailable.jsonl"), + func(_ context.Context, definitions []query.Definition, requested []string) (map[string]model.Source, error) { + return map[string]model.Source{requested[0]: {Metadata: model.Metadata{ + "availability": "unavailable", "canonical-revision": int64(42), + }}}, nil + }); err == nil { + t.Fatal("unavailable canonical query produced a current Value result") + } } -func TestHostedQueryBridgeRejectsOversizedRequests(t *testing.T) { - queries := 0 - bridge, err := startHostedQueryBridge(t.Context(), +func TestHostedDependabotWithoutJSReportsMissingEvidence(t *testing.T) { + tests := []struct { + name, campaign, repository string + missing bool + }{ + {"enabled", `"dependabot":{"targets":{"github/gh-aw":{"mode":"live"}},"workers":{"update-planner":{"workflow":"dependabot-update-planner"}}}`, "github/gh-aw", true}, + {"disabled campaign", `"dependabot":{"enabled":false,"targets":{"github/gh-aw":{"mode":"live"}},"workers":{"update-planner":{"workflow":"dependabot-update-planner"}}}`, "github/gh-aw", false}, + {"disabled worker", `"dependabot":{"targets":{"github/gh-aw":{"mode":"live"}},"workers":{"update-planner":{"workflow":"dependabot-update-planner","enabled":false}}}`, "github/gh-aw", false}, + {"out of scope", `"dependabot":{"targets":{"github/gh-aw":{"mode":"live"}},"workers":{"update-planner":{"workflow":"dependabot-update-planner"}}}`, "githubnext/gh-cao", false}, + {"remote registration alone", "", "github/gh-aw", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + campaigns := `"activity":{"workflow-values":[{"repository":"github/gh-aw","workflow":"daily-file-diet",` + + `"collector":".github/value-collectors/daily-file-diet.mjs"}]}` + if tt.campaign != "" { + campaigns += "," + tt.campaign + } + policy := filepath.Join(root, "cao.json") + content := `{"version":1,"gh-aw-version":"v0.93.1","control-plane":{` + + `"scope":{"allowed-repositories":["github/gh-aw","githubnext/gh-cao"]},` + + `"activity":{"repositories":["github/gh-aw","githubnext/gh-cao"]},` + + `"campaigns":{` + campaigns + `}}}` + if err := os.WriteFile(policy, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(root, "snapshot") + if err := os.MkdirAll(snapshot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { + t.Fatal(err) + } + observed, _ := time.Parse(time.RFC3339, "2026-11-05T20:00:00Z") + config := Config{Root: root, Repositories: []string{tt.repository}, + NodeBinary: "/nonexistent-node", ObservedAt: observed} + if err := WriteCollectionHealth(t.Context(), config, policy, snapshot); err != nil { + t.Fatal(err) + } + output := filepath.Join(root, "values.jsonl") + queries := 0 + if err := ComputeHostedSnapshot(t.Context(), config, policy, snapshot, output, + func(_ context.Context, definitions []query.Definition, requested []string) (map[string]model.Source, error) { + queries++ + return map[string]model.Source{requested[0]: { + Source: requested[0], + Rows: []model.Row{{"declaredWorkflowPath": ".github/workflows/worker.md", + "concludedRuns": 3, "successfulRuns": 3, "successfulRunsWithAic": 3, + "aicPerSuccessfulRun": 2.5, "failedRuns": 0, "cancelledRuns": 0}}, + Metadata: model.Metadata{"availability": "available", "canonical-revision": int64(42)}, + }}, nil + }); err != nil { + t.Fatal(err) + } + if queries != 1 { + t.Fatalf("canonical query count = %d, want 1", queries) + } + rows, err := ReadMeasurements(output) + if err != nil { + t.Fatal(err) + } + want := 3 + if tt.missing { + want += 3 + } + if len(rows) != want { + t.Fatalf("assessments = %d, want %d: %+v", len(rows), want, rows) + } + for _, row := range rows { + if row["campaign"] != "dependabot" { + if row["value"] == nil { + t.Fatalf("supported Optimization was not evaluated: %+v", row) + } + continue + } + if row["value"] != nil || row["maturity-status"] != "unavailable" || + row["coverage-status"] != "missing" || + row["missing-reason"] != "remote-acquisition-unavailable" || + row["measurement-kind"] != "snapshot" || + row["evidence-digest"] != nil || row["rollup-numerator"] != nil { + t.Fatalf("unsupported Dependabot became a successful assessment: %+v", row) + } + } + }) + } +} + +func TestHostedLocalEvaluationHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := ComputeHostedSnapshot(ctx, Config{}, "unused", "unused", "unused", func(context.Context, []query.Definition, []string) (map[string]model.Source, error) { - queries++ + t.Fatal("canceled hosted Value evaluation queried the canonical database") return map[string]model.Source{}, nil - }) + }); err == nil { + t.Fatal("canceled hosted Value evaluation succeeded") + } +} + +func TestHostedPinnedOptimizationAdapterMatchesReviewedSource(t *testing.T) { + root, err := filepath.Abs("../../..") if err != nil { t.Fatal(err) } - defer bridge.Close() - connection, err := (&net.Dialer{}).DialContext(t.Context(), "unix", bridge.path) + if err := verifyHostedLocalAdapters(root, nil); err != nil { + t.Fatalf("checked-in JS oracle changed without a Go evaluator review: %v", err) + } +} + +func TestHostedReviewedEvidenceImportUsesPinnedOriginalPacket(t *testing.T) { + root, err := filepath.Abs("../../..") if err != nil { t.Fatal(err) } - if _, err := connection.Write([]byte(strings.Repeat(" ", hostedQueryRequestBytes+1))); err != nil { - _ = connection.Close() + snapshot := t.TempDir() + if err := os.WriteFile(filepath.Join(snapshot, "inventory-sources.json"), + []byte(`{"repositories":{"rows":[]},"workflows":{"rows":[]}}`), 0o600); err != nil { t.Fatal(err) } - if unix, ok := connection.(*net.UnixConn); ok { - if err := unix.CloseWrite(); err != nil { - _ = connection.Close() - t.Fatal(err) - } + policy := filepath.Join(root, ".github", "workflows", "cao.json") + if err := WriteCollectionHealth(t.Context(), + Config{Root: root, NodeBinary: "/nonexistent-node", Repositories: []string{"github/gh-aw"}}, + policy, snapshot); err != nil { + t.Fatal(err) } - content, err := io.ReadAll(connection) - _ = connection.Close() + content, err := os.ReadFile(filepath.Join(snapshot, "gh-aw-logs-shards", "workflow-values.jsonl")) // #nosec G304 -- snapshot is a test-owned TempDir. if err != nil { t.Fatal(err) } - var response hostedQueryResponse - if err := json.Unmarshal(content, &response); err != nil { - t.Fatal(err) + var envelope struct { + SchemaVersion int `json:"schema_version"` + Kind string `json:"kind"` + Packet model.Row `json:"operational_value_evidence"` } - if queries != 0 || !strings.Contains(response.Error, "byte budget") { - t.Fatalf("oversized request response = %#v, queries = %d", response, queries) + if err := json.Unmarshal(content, &envelope); err != nil || envelope.SchemaVersion != 2 || + envelope.Kind != "operational_value_evidence" || + envelope.Packet["repository"] != "github/gh-aw" || + envelope.Packet["workflow"] != "daily-file-diet" { + t.Fatalf("pinned reviewed evidence was not imported: %v %v", err, envelope.Packet) } } diff --git a/server/internal/operationalvalue/policy.go b/server/internal/operationalvalue/policy.go index d2e7ba5b69..23cb08fa1a 100644 --- a/server/internal/operationalvalue/policy.go +++ b/server/internal/operationalvalue/policy.go @@ -1,10 +1,18 @@ package operationalvalue import ( + "bytes" "context" + "crypto/sha256" "encoding/json" "errors" + "fmt" + "io" + "os" "path/filepath" + "regexp" + "strconv" + "strings" ) type WorkflowPolicySnapshot struct { @@ -12,33 +20,585 @@ type WorkflowPolicySnapshot struct { Registrations []WorkflowRegistration `json:"registrations"` } -// ResolveWorkflowPolicy binds registrations to the same validated, composed -// document. Callers must resolve again before publication rather than caching -// authority by the entry profile's bytes alone. -func ResolveWorkflowPolicy(ctx context.Context, config Config, policy string) (WorkflowPolicySnapshot, error) { - const resolve = ` -import { pathToFileURL } from 'node:url'; -const { loadPolicyFile } = await import(pathToFileURL(process.argv[1]).href); -const { workflowValueRegistrations } = await import(pathToFileURL(process.argv[2]).href); -const document = loadPolicyFile(process.argv[3]); -const entries = document['control-plane']?.campaigns?.activity?.['workflow-values'] ?? []; -const registrations = workflowValueRegistrations(document, entries.map(entry => entry.repository)); -process.stdout.write(JSON.stringify({ document, registrations })); -` - content, err := runLocalRuntimeOutput(ctx, config, []string{ - "--input-type=module", "--eval", resolve, - filepath.Join(config.Root, ".github", "workflows", "shared", "policy.mjs"), - filepath.Join(config.Root, "activity", "workflow-values.mjs"), policy, - }) +type WorkflowRegistration struct { + Repository string `json:"repository"` + Workflow string `json:"workflow"` + Collector string `json:"collector"` +} + +func WorkflowRegistrations(ctx context.Context, config Config, policy string) ([]WorkflowRegistration, error) { + snapshot, err := ResolveWorkflowPolicy(ctx, config, policy) + return snapshot.Registrations, err +} + +type valuePolicy struct { + Version int `json:"version"` + GHAW string `json:"gh-aw-version"` + Control struct { + Scope struct { + Owners []string `json:"allowed-owners"` + Repositories []string `json:"allowed-repositories"` + } `json:"scope"` + Activity struct { + Repositories []string `json:"repositories"` + } `json:"activity"` + Campaigns map[string]struct { + Enabled *bool `json:"enabled"` + Values []valueRegistration `json:"workflow-values"` + Workers map[string]struct { + Workflow string `json:"workflow"` + Enabled *bool `json:"enabled"` + } `json:"workers"` + Targets map[string]struct { + Mode string `json:"mode"` + } `json:"targets"` + } `json:"campaigns"` + Workflows map[string]struct { + Enabled *bool `json:"enabled"` + } `json:"workflows"` + } `json:"control-plane"` +} + +type valueRegistration struct { + Repository string `json:"repository"` + Workflow string `json:"workflow"` + Collector string `json:"collector"` + Enabled *bool `json:"enabled"` + APIRequests int `json:"api-requests"` + Evidence json.RawMessage `json:"evidence"` + Acquisition json.RawMessage `json:"acquisition"` +} + +var ( + valueRepositoryPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9._-]+$`) + valueOwnerPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`) + valueWorkflowPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + valueCollectorPattern = regexp.MustCompile(`^[A-Za-z0-9._/-]+\.mjs$`) + valueEvidencePattern = regexp.MustCompile(`^\.github/value-evidence/[A-Za-z0-9._/-]+\.jsonl(?:\.gz)?$`) + valueGHAWPattern = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$`) + valueRequirementPattern = regexp.MustCompile(`^[a-z0-9]+(?:[a-z0-9._:/-]*[a-z0-9])?$`) +) + +// orderedObject preserves JSON insertion order. The policy digest is the hash +// of JSON.stringify(composedPolicy), not Go's alphabetically sorted map JSON. +type orderedObject []orderedField +type orderedField struct { + key string + value any +} + +func (o orderedObject) get(key string) (any, bool) { + for _, field := range o { + if field.key == key { + return field.value, true + } + } + return nil, false +} + +func (o orderedObject) object(key string) orderedObject { + value, _ := o.get(key) + result, _ := value.(orderedObject) + return result +} + +func decodePolicyValue(decoder *json.Decoder) (any, error) { + token, err := decoder.Token() if err != nil { - return WorkflowPolicySnapshot{}, err + return nil, err + } + switch token { + case json.Delim('{'): + object := orderedObject{} + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, errors.New("invalid policy key") + } + if _, duplicate := object.get(key); duplicate { + return nil, fmt.Errorf("duplicate policy key %q", key) + } + value, err := decodePolicyValue(decoder) + if err != nil { + return nil, err + } + object = append(object, orderedField{key, value}) + } + _, err := decoder.Token() + return object, err + case json.Delim('['): + values := []any{} + for decoder.More() { + value, err := decodePolicyValue(decoder) + if err != nil { + return nil, err + } + values = append(values, value) + } + _, err := decoder.Token() + return values, err + } + if text, ok := token.(string); ok && strings.Contains(text, "${{") { + return nil, errors.New("policy must not contain GitHub Actions expressions") + } + return token, nil +} + +func marshalPolicyValue(buffer *bytes.Buffer, value any) error { + switch typed := value.(type) { + case orderedObject: + buffer.WriteByte('{') + for i, field := range typed { + if i != 0 { + buffer.WriteByte(',') + } + key, _ := json.Marshal(field.key) + buffer.Write(key) + buffer.WriteByte(':') + if err := marshalPolicyValue(buffer, field.value); err != nil { + return err + } + } + buffer.WriteByte('}') + case []any: + buffer.WriteByte('[') + for i, item := range typed { + if i != 0 { + buffer.WriteByte(',') + } + if err := marshalPolicyValue(buffer, item); err != nil { + return err + } + } + buffer.WriteByte(']') + case json.Number: + number, err := strconv.ParseFloat(string(typed), 64) + if err != nil || number > 9_007_199_254_740_991 || number < -9_007_199_254_740_991 { + return errors.New("reviewed policy number exceeds JSON safe integer precision") + } + buffer.WriteString(strconv.FormatFloat(number, 'f', -1, 64)) + default: + var encoded bytes.Buffer + writer := json.NewEncoder(&encoded) + writer.SetEscapeHTML(false) + err := writer.Encode(typed) + if err != nil { + return err + } + buffer.Write(bytes.TrimSuffix(encoded.Bytes(), []byte{'\n'})) + } + return nil +} + +func composeValuePolicy(base, overlay orderedObject) orderedObject { + result := append(orderedObject{}, base...) + for _, field := range overlay { + found := false + for i := range result { + if result[i].key == field.key { + child, baseObject := result[i].value.(orderedObject) + replacement, overlayObject := field.value.(orderedObject) + if baseObject && overlayObject { + result[i].value = composeValuePolicy(child, replacement) + } else { + result[i].value = field.value + } + found = true + break + } + } + if !found { + result = append(result, field) + } + } + return result +} + +func loadValuePolicy(ctx context.Context, path string) (json.RawMessage, valuePolicy, error) { + var policy valuePolicy + entry, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, policy, err + } + root := filepath.Dir(entry) + active := map[string]bool{} + var load func(string, int) (orderedObject, error) + load = func(path string, depth int) (orderedObject, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if depth > 8 { + return nil, errors.New("deployment profile import depth exceeds 8") + } + canonical, err := filepath.EvalSymlinks(path) + if err != nil { + return nil, err + } + relative, err := filepath.Rel(root, canonical) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return nil, errors.New("deployment profile imports must remain within the policy directory") + } + if active[canonical] { + return nil, errors.New("deployment profile import cycle") + } + active[canonical] = true + defer delete(active, canonical) + content, err := readBoundedValueFile(canonical, 4<<20) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(content)) + decoder.UseNumber() + value, err := decodePolicyValue(decoder) + if err != nil { + return nil, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return nil, errors.New("invalid trailing policy JSON") + } + document, ok := value.(orderedObject) + if !ok { + return nil, errors.New("policy root must be a mapping") + } + base, extends := document.get("extends") + if !extends { + return document, nil + } + if len(document) != 2 || document.object("control-plane") == nil || + len(document.object("control-plane")) != 1 || + len(document.object("control-plane").object("web")) != 1 { + return nil, errors.New("deployment profile may only extend control-plane.web.host") + } + if _, ok := document.object("control-plane").object("web").get("host"); !ok { + return nil, errors.New("deployment profile requires control-plane.web.host") + } + importPath, ok := base.(string) + if !ok || importPath == "" || filepath.IsAbs(importPath) { + return nil, errors.New("deployment profile extends must be a non-empty relative path") + } + imported, err := load(filepath.Join(filepath.Dir(canonical), importPath), depth+1) + if err != nil { + return nil, err + } + profile := orderedObject{} + for _, field := range document { + if field.key != "extends" { + profile = append(profile, field) + } + } + return composeValuePolicy(imported, profile), nil + } + document, err := load(entry, 0) + if err != nil { + return nil, policy, err + } + var buffer bytes.Buffer + if err := marshalPolicyValue(&buffer, document); err != nil { + return nil, policy, err + } + content := buffer.Bytes() + if err := json.Unmarshal(content, &policy); err != nil { + return nil, policy, err + } + if err := validateValuePolicy(document, &policy); err != nil { + return nil, policy, err + } + return content, policy, nil +} + +func readBoundedValueFile(path string, limit int64) ([]byte, error) { + file, err := os.Open(path) // #nosec G304 -- operator-selected policy or confined private transport. + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + content, err := io.ReadAll(io.LimitReader(file, limit+1)) + if err != nil { + return nil, err + } + if int64(len(content)) > limit { + return nil, errors.New("reviewed Value input exceeds its byte budget") + } + return content, nil +} + +func validateValuePolicy(document orderedObject, policy *valuePolicy) error { + if policy.Version != 1 || !valueGHAWPattern.MatchString(policy.GHAW) || document.object("control-plane") == nil { + return errors.New("value requires a reviewed version 1 control policy") + } + if _, ok := document.get("extends"); ok { + return errors.New("unresolved policy profile") + } + if value, ok := document.get("$schema"); ok && + value != "https://raw.githubusercontent.com/githubnext/gh-cao/main/.github/workflows/shared/cao.schema.json" { + return errors.New("invalid reviewed policy schema") + } + if err := allowedPolicyKeys(document, "$schema", "version", "gh-aw-version", "control-plane", "target-authority"); err != nil { + return err + } + control := document.object("control-plane") + if err := allowedPolicyKeys(control, "scope", "activity", "inventory", "web", "defaults", "campaigns", "workflows", "operations", "publishing", "marketplace", "webhook-gateway", "token-broker"); err != nil { + return err + } + for _, field := range []string{"scope", "activity", "campaigns", "workflows"} { + if value, exists := control.get(field); exists { + if _, ok := value.(orderedObject); !ok { + return fmt.Errorf("reviewed control-plane.%s must be a mapping", field) + } + } + } + if err := allowedPolicyKeys(control.object("scope"), "allowed-owners", "allowed-repositories"); err != nil { + return err + } + for _, field := range []string{"allowed-owners", "allowed-repositories"} { + if value, exists := control.object("scope").get(field); exists { + if _, ok := value.([]any); !ok { + return errors.New("reviewed scope must be a list") + } + } + } + if _, exists := control.get("activity"); !exists || len(policy.Control.Activity.Repositories) == 0 { + return errors.New("value requires explicit Activity repository scope") + } + if err := allowedPolicyKeys(control.object("activity"), "repositories", "public-only"); err != nil { + return err + } + if _, ok := control.object("activity").get("repositories"); !ok { + return errors.New("activity repositories must be explicitly reviewed") + } + if value, exists := control.object("activity").get("public-only"); exists { + if _, ok := value.(bool); !ok { + return errors.New("invalid Activity public-only setting") + } + } + for _, set := range [][]string{policy.Control.Scope.Owners, policy.Control.Scope.Repositories, policy.Control.Activity.Repositories} { + seen := map[string]bool{} + for _, item := range set { + key := strings.ToLower(item) + if seen[key] { + return errors.New("duplicate reviewed Activity scope") + } + seen[key] = true + } + } + for _, owner := range policy.Control.Scope.Owners { + if !valueOwnerPattern.MatchString(owner) { + return errors.New("invalid reviewed owner scope") + } + } + for _, repository := range append(append([]string{}, policy.Control.Scope.Repositories...), policy.Control.Activity.Repositories...) { + if !valueRepositoryPattern.MatchString(repository) { + return errors.New("invalid reviewed repository scope") + } + } + for _, repository := range policy.Control.Activity.Repositories { + if !valuePolicyInScope(policy, policy.Control.Activity.Repositories, repository) { + return errors.New("activity repository lies outside reviewed scope") + } + } + for name := range policy.Control.Campaigns { + if !valueWorkflowPattern.MatchString(name) { + return errors.New("invalid campaign identity") + } + if _, exists := control.object("campaigns").object(name).get("workflow-values"); name != "activity" && exists { + return errors.New("workflow-values is only supported for Activity") + } + if enabled, exists := control.object("campaigns").object(name).get("enabled"); exists { + if _, ok := enabled.(bool); !ok { + return errors.New("invalid campaign enabled setting") + } + } + } + activity := policy.Control.Campaigns["activity"] + if value, exists := control.object("campaigns").get("activity"); exists { + if _, ok := value.(orderedObject); !ok { + return errors.New("reviewed Activity campaign must be a mapping") + } + } + if len(activity.Values) > 1000 { + return errors.New("reviewed Value registration limit exceeded") + } + values, declared := control.object("campaigns").object("activity").get("workflow-values") + items, _ := values.([]any) + if declared && items == nil || len(items) != len(activity.Values) { + return errors.New("invalid reviewed Value registrations") + } + identities := map[string]bool{} + for i, entry := range activity.Values { + raw, ok := items[i].(orderedObject) + if !ok { + return errors.New("invalid reviewed Value registration") + } + if err := allowedPolicyKeys(raw, "repository", "workflow", "collector", "evidence", "api-requests", "enabled", "acquisition"); err != nil { + return err + } + if !valueRepositoryPattern.MatchString(entry.Repository) || !valueWorkflowPattern.MatchString(entry.Workflow) || + (entry.Enabled == nil || *entry.Enabled) && !valueCollectorPattern.MatchString(entry.Collector) || + entry.Collector != "" && (!valueCollectorPattern.MatchString(entry.Collector) || !confinedValuePath(entry.Collector)) { + return errors.New("invalid reviewed Value registration identity or collector") + } + if value, exists := raw.get("enabled"); exists { + if _, ok := value.(bool); !ok { + return errors.New("invalid Value registration enabled flag") + } + } + if value, exists := raw.get("api-requests"); exists { + number, ok := value.(json.Number) + if !ok || number.String() != fmt.Sprint(entry.APIRequests) || entry.APIRequests < 1 || entry.APIRequests > 1000 || entry.Collector == "" { + return errors.New("invalid reviewed Value API budget") + } + } + if len(entry.Evidence) != 0 { + var evidence struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + } + if err := json.Unmarshal(entry.Evidence, &evidence); err != nil { + return err + } + metadata := raw.object("evidence") + if len(metadata) != 2 || !valueEvidencePattern.MatchString(evidence.Path) || + !confinedValuePath(evidence.Path) || + (!strings.HasSuffix(evidence.Path, ".jsonl") && !strings.HasSuffix(evidence.Path, ".jsonl.gz")) || + !sha256Fingerprint.MatchString(evidence.SHA256) { + return errors.New("invalid reviewed Value evidence import") + } + } + if len(entry.Acquisition) != 0 && entry.Collector == "" { + return errors.New("value acquisition requires a reviewed collector") + } + if len(entry.Acquisition) != 0 { + if err := validateValueAcquisition(raw.object("acquisition")); err != nil { + return err + } + } + key := strings.ToLower(entry.Repository) + ":" + entry.Workflow + if identities[key] { + return errors.New("duplicate reviewed Value registration") + } + identities[key] = true } - var snapshot WorkflowPolicySnapshot - if err := json.Unmarshal(content, &snapshot); err != nil { + return nil +} + +func validateValueAcquisition(acquisition orderedObject) error { + if len(acquisition) != 1 { + return errors.New("invalid reviewed Value acquisition manifest") + } + raw, exists := acquisition.get("requirements") + requirements, ok := raw.([]any) + if !exists || !ok || len(requirements) == 0 || len(requirements) > 64 { + return errors.New("invalid reviewed Value acquisition requirements") + } + seen := map[string]bool{} + for _, value := range requirements { + requirement, ok := value.(orderedObject) + if !ok || len(requirement) != 5 || + allowedPolicyKeys(requirement, "cache", "pool", "primitive", "scope", "source") != nil { + return errors.New("invalid reviewed Value acquisition requirement") + } + fields := make([]string, 0, 5) + for _, name := range []string{"cache", "pool", "primitive", "scope", "source"} { + value, _ := requirement.get(name) + text, ok := value.(string) + if !ok || len(text) > 160 || !valueRequirementPattern.MatchString(text) { + return errors.New("invalid reviewed Value acquisition identity") + } + fields = append(fields, text) + } + cache, pool, source := fields[0], fields[1], fields[4] + valid := source == "canonical" && pool == "local" && cache == "canonical" || + source == "github" && (pool == "core" || pool == "search" || pool == "graphql") && cache == "invocation" || + source == "git" && pool == "git" && cache == "immutable" || + source == "archive" && pool == "archive" && cache == "invocation" + if !valid { + return errors.New("unsupported reviewed Value acquisition source or quota pool") + } + key := strings.Join(fields, "\x00") + if seen[key] { + return errors.New("duplicate reviewed Value acquisition requirement") + } + seen[key] = true + } + return nil +} + +func allowedPolicyKeys(object orderedObject, allowed ...string) error { + for _, field := range object { + found := false + for _, key := range allowed { + if key == field.key { + found = true + break + } + } + if !found { + return fmt.Errorf("unsupported reviewed policy field %q", field.key) + } + } + return nil +} + +func confinedValuePath(path string) bool { + if filepath.IsAbs(path) { + return false + } + for _, part := range strings.Split(path, "/") { + if part == "" || part == "." || part == ".." { + return false + } + } + return true +} + +func valuePolicyInScope(policy *valuePolicy, repositories []string, repository string) bool { + contains := func(items []string, value string) bool { + for _, item := range items { + if strings.EqualFold(item, value) { + return true + } + } + return false + } + owner, _, _ := strings.Cut(repository, "/") + return contains(repositories, repository) && + contains(policy.Control.Activity.Repositories, repository) && + (len(policy.Control.Scope.Repositories) == 0 || contains(policy.Control.Scope.Repositories, repository)) && + (len(policy.Control.Scope.Owners) == 0 || contains(policy.Control.Scope.Owners, owner)) +} + +func valuePolicyDigest(content []byte) string { + hash := sha256.Sum256(content) + return fmt.Sprintf("%x", hash) +} + +// ResolveWorkflowPolicy binds registrations to the same validated composed +// document. Re-resolve before publication: importing a profile never grants +// authority based solely on the unchanged entry file. +func ResolveWorkflowPolicy(ctx context.Context, _ Config, path string) (WorkflowPolicySnapshot, error) { + document, policy, err := loadValuePolicy(ctx, path) + if err != nil { return WorkflowPolicySnapshot{}, err } - if len(snapshot.Document) == 0 || !json.Valid(snapshot.Document) || len(snapshot.Registrations) > 1000 { - return WorkflowPolicySnapshot{}, errors.New("reviewed Value policy snapshot exceeds its contract") + snapshot := WorkflowPolicySnapshot{Document: document, Registrations: []WorkflowRegistration{}} + activity := policy.Control.Campaigns["activity"] + if activity.Enabled != nil && !*activity.Enabled { + return snapshot, nil + } + repositories := make([]string, len(activity.Values)) + for i, entry := range activity.Values { + repositories[i] = entry.Repository + } + for _, entry := range activity.Values { + if (entry.Enabled == nil || *entry.Enabled) && valuePolicyInScope(&policy, repositories, entry.Repository) { + snapshot.Registrations = append(snapshot.Registrations, WorkflowRegistration{ + Repository: entry.Repository, Workflow: entry.Workflow, Collector: entry.Collector, + }) + } } return snapshot, nil } diff --git a/server/internal/operationalvalue/policy_test.go b/server/internal/operationalvalue/policy_test.go new file mode 100644 index 0000000000..2d993393ef --- /dev/null +++ b/server/internal/operationalvalue/policy_test.go @@ -0,0 +1,113 @@ +package operationalvalue + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGoWorkflowPolicyComposesImportsAndBindsRegistrations(t *testing.T) { + dir := t.TempDir() + base := `{"version":1,"gh-aw-version":"v0.93.1","control-plane":{"scope":{ + "allowed-owners":["org"],"allowed-repositories":["org/repo"]}, + "activity":{"repositories":["org/repo"]}, + "campaigns":{"activity":{"workflow-values":[ + {"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs","api-requests":5, + "acquisition":{"requirements":[{"source":"github","pool":"core", + "primitive":"github.actions-run-provenance.v1","scope":"worker","cache":"invocation"}]}}]}}}}` + if err := os.WriteFile(filepath.Join(dir, "base.json"), []byte(base), 0o600); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "cao.json") + if err := os.WriteFile(path, []byte(`{"extends":"base.json","control-plane":{"web":{"host":{ + "target":{"module":"container"}}}}}`), 0o600); err != nil { + t.Fatal(err) + } + snapshot, err := ResolveWorkflowPolicy(context.Background(), Config{NodeBinary: "/nonexistent-node"}, path) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Registrations) != 1 || snapshot.Registrations[0].Collector != "reviewed.mjs" || + strings.Contains(string(snapshot.Document), "extends") || + !strings.Contains(string(snapshot.Document), `"allowed-owners":["org"]`) || + !strings.Contains(string(snapshot.Document), `"host":{"target":{"module":"container"}}`) { + t.Fatalf("composed reviewed authority was not retained: %+v", snapshot) + } + expected := `{"version":1,"gh-aw-version":"v0.93.1","control-plane":{"scope":{` + + `"allowed-owners":["org"],"allowed-repositories":["org/repo"]},` + + `"activity":{"repositories":["org/repo"]},` + + `"campaigns":{"activity":{"workflow-values":[{"repository":"org/repo","workflow":"worker",` + + `"collector":"reviewed.mjs","api-requests":5,"acquisition":{"requirements":[{` + + `"source":"github","pool":"core","primitive":"github.actions-run-provenance.v1",` + + `"scope":"worker","cache":"invocation"}]}}]}},"web":{"host":{"target":{"module":"container"}}}}}` + if string(snapshot.Document) != expected { + t.Fatalf("composed JSON diverged from JSON.stringify policy order:\n%s\n%s", + snapshot.Document, expected) + } + if err := os.WriteFile(filepath.Join(dir, "base.json"), []byte(strings.Replace(base, + `"allowed-repositories":["org/repo"]`, `"allowed-repositories":["else/target"]`, 1)), 0o600); err != nil { + t.Fatal(err) + } + if _, err := ResolveWorkflowPolicy(context.Background(), Config{}, path); err == nil { + t.Fatal("reviewed base-scope revocation was ignored") + } +} + +func TestGoWorkflowPolicyRejectsUnreviewedAuthority(t *testing.T) { + valid := `{"version":1,"gh-aw-version":"v0.93.1","control-plane":{ + "scope":{"allowed-repositories":["org/repo"]},"activity":{"repositories":["org/repo"]}, + "campaigns":{"activity":{"workflow-values":[ + {"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs"}]}}}}` + for name, content := range map[string]string{ + "duplicate key": strings.Replace(valid, `"version":1`, `"version":1,"version":1`, 1), + "unreviewed": strings.Replace(valid, `"collector":"reviewed.mjs"`, `"collector":"../unreviewed.mjs"`, 1), + "expression": strings.Replace(valid, `"collector":"reviewed.mjs"`, `"collector":"${{ secrets.TOKEN }}"`, 1), + "unsupported manifest": strings.Replace(valid, `"collector":"reviewed.mjs"`, + `"collector":"reviewed.mjs","acquisition":{"requirements":[{"source":"git","pool":"core",`+ + `"primitive":"git.commit-inventory.v1","scope":"worker","cache":"immutable"}]}`, 1), + "duplicate identity": strings.Replace(valid, + `{"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs"}`, + `{"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs"},`+ + `{"repository":"org/repo","workflow":"worker","collector":"reviewed.mjs"}`, 1), + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "cao.json") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := ResolveWorkflowPolicy(t.Context(), Config{}, path); err == nil { + t.Fatal("accepted unreviewed Value policy") + } + }) + } +} + +func TestGoWorkflowPolicyResolvesCurrentCatalogAndHostedProfile(t *testing.T) { + root, err := filepath.Abs("../../..") + if err != nil { + t.Fatal(err) + } + for _, file := range []string{"cao.json", "cao.azure.json"} { + t.Run(file, func(t *testing.T) { + path := filepath.Join(root, ".github", "workflows", file) + snapshot, err := ResolveWorkflowPolicy(t.Context(), + Config{NodeBinary: "/nonexistent-node"}, path) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Registrations) != 2 || + snapshot.Registrations[0].Repository != "github/gh-aw" || + snapshot.Registrations[0].Workflow != "daily-file-diet" || + snapshot.Registrations[1].Repository != "github/gh-aw" || + snapshot.Registrations[1].Workflow != "dependabot-update-planner" || + !strings.Contains(string(snapshot.Document), `"git.commit-inventory.v1"`) || + !strings.Contains(string(snapshot.Document), `"github.dependabot-alert-lifecycle.v1"`) || + strings.Contains(string(snapshot.Document), `"extends"`) { + t.Fatalf("checked-in reviewed registration was not resolved: %+v", snapshot.Registrations) + } + }) + } +} diff --git a/server/internal/operationalvalue/workflow.go b/server/internal/operationalvalue/workflow.go deleted file mode 100644 index c91b8d6172..0000000000 --- a/server/internal/operationalvalue/workflow.go +++ /dev/null @@ -1,134 +0,0 @@ -package operationalvalue - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/githubnext/gh-cao/server/internal/model" -) - -type WorkflowRegistration struct { - Repository string `json:"repository"` - Workflow string `json:"workflow"` - Collector string `json:"collector"` -} - -func WorkflowRegistrations(ctx context.Context, config Config, policy string) ([]WorkflowRegistration, error) { - node := config.NodeBinary - if node == "" { - node = "node" - } - worker, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - command := exec.CommandContext(worker, node, filepath.Join(config.Root, "activity", "workflow-values.mjs"), "--registrations", policy) // #nosec G204 -- trusted runtime. - command.Dir, command.Env = config.Root, config.Environment - configureProcessTree(command) - command.Cancel = func() error { return terminateProcessTree(command.Process) } - command.WaitDelay = 5 * time.Second - stdout, stderr := newLimitedBuffer(1024*1024, cancel), newLimitedBuffer(4096, cancel) - command.Stdout, command.Stderr = stdout, stderr - err := command.Run() - _ = terminateProcessTree(command.Process) - if err != nil || stdout.Exceeded() { - return nil, errors.New("cannot resolve reviewed workflow Value registrations") - } - var entries []WorkflowRegistration - if err := json.Unmarshal(stdout.Bytes(), &entries); err != nil { - return nil, err - } - return entries, nil -} - -// CollectWorkflows executes only reviewed exact opt-ins through the shared -// Activity producer. previous contains canonical accepted envelopes, not code. -func CollectWorkflows(ctx context.Context, config Config, policy, workspace, previous string) ([]model.Row, []string, error) { - if config.RateLimitReserve > 0 { - remaining, err := githubAPIRemaining(ctx, config) - if err != nil { - return nil, nil, err - } - if remaining <= config.RateLimitReserve { - return nil, []string{"workflow Value acquisition skipped: API budget reserve"}, nil - } - } - if err := os.MkdirAll(workspace, 0o700); err != nil { - return nil, nil, err - } - scratch, err := os.MkdirTemp(workspace, ".invocation-") - if err != nil { - return nil, nil, err - } - defer func() { _ = os.RemoveAll(scratch) }() - output := filepath.Join(scratch, "packets.jsonl") - arguments := []string{ - filepath.Join(config.Root, "activity", "workflow-values.mjs"), - "--root", config.Root, "--policy", policy, "--database", config.Database, - "--workspace", workspace, "--previous", previous, "--output", output, - "--observedAt", formatTimestamp(config.ObservedAt), - } - for _, repository := range config.Repositories { - arguments = append(arguments, "--repository", repository) - } - timeout := config.WorkerTimeout - if timeout <= 0 { - timeout = 10*time.Minute + 10*time.Second - } - worker, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - node := config.NodeBinary - if node == "" { - node = "node" - } - command := exec.CommandContext(worker, node, arguments...) // #nosec G204 -- reviewed runtime and bounded arguments. - command.Dir = config.Root - command.Env = append(append([]string{}, config.Environment...), "CAO_WORKFLOW_VALUE_SHARED_PROCESS_GROUP=1") - configureProcessTree(command) - command.Cancel = func() error { return terminateProcessTree(command.Process) } - command.WaitDelay = 5 * time.Second - stdout, stderr := newLimitedBuffer(maxWorkerOutput, cancel), newLimitedBuffer(maxWorkerOutput, cancel) - command.Stdout, command.Stderr = stdout, stderr - err = command.Run() - _ = terminateProcessTree(command.Process) - if err != nil || stdout.Exceeded() || stderr.Exceeded() { - return nil, nil, errors.New("reviewed workflow Value producer failed or exceeded its budget; accepted evidence retained") - } - var report struct { - Warnings []string `json:"warnings"` - } - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - return nil, nil, fmt.Errorf("decode workflow Value result: %w", err) - } - file, err := os.Open(output) // #nosec G304 -- fixed filename in this invocation's private scratch directory. - if err != nil { - return nil, nil, err - } - defer func() { _ = file.Close() }() - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 64*1024), 8*1024*1024+1) - packets := []model.Row{} - for scanner.Scan() { - var envelope struct { - SchemaVersion int `json:"schema_version"` - Kind string `json:"kind"` - Packet model.Row `json:"operational_value_evidence"` - } - decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes())) - decoder.UseNumber() - if err := decoder.Decode(&envelope); err != nil || envelope.SchemaVersion != 2 || envelope.Kind != "operational_value_evidence" { - return nil, nil, errors.New("invalid workflow Value producer envelope") - } - packets = append(packets, envelope.Packet) - if len(packets) > 1000 { - return nil, nil, errors.New("workflow Value packet count exceeds budget") - } - } - return packets, report.Warnings, scanner.Err() -} diff --git a/server/internal/operationalvalue/workflow_test.go b/server/internal/operationalvalue/workflow_test.go deleted file mode 100644 index 4f130490c6..0000000000 --- a/server/internal/operationalvalue/workflow_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package operationalvalue - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "testing" - "time" -) - -func TestWorkflowProducerUsesReviewedExactRegistrationAndOriginalPacket(t *testing.T) { - root, err := filepath.Abs("../../..") - if err != nil { - t.Fatal(err) - } - directory, err := os.MkdirTemp(root, ".hosted-workflow-value-test-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(directory) }) - packet := map[string]any{ - "schemaVersion": 1, "repository": "org/repo", "workflow": "daily-review", - "id": "operational-value-evidence:org%2Frepo%3Adaily-review%3Alines%3Av1", - "sourcePath": ".github/workflows/daily-review.md", "observedAt": "2026-10-20T12:00:00.000Z", - "windowStart": "2026-10-01T00:00:00.000Z", "windowEnd": "2026-10-20T00:00:00.000Z", - "contractDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "facts": []any{ - map[string]any{ - "fact-kind": "contract", "fact-id": "contract", "goal": "Remove waste", "acceptance": "Accepted removal", - "workflow-name": "Review", "metric-id": "lines", "metric-name": "Lines", "metric-unit": "lines", - "metric-direction": "increase", "metric-version": "v1", "metric-formula": "accepted lines removed", - "measurement-basis": "workflow-contribution", "cadence-days": 1, - "outputs-available": false, "repository-state-available": false, "checks-available": false, - }, - map[string]any{"fact-kind": "coverage", "fact-id": "window", "at": "2026-10-01T00:00:00.000Z", - "end-at": "2026-10-20T00:00:00.000Z", "coverage-status": "complete"}, - }, - } - envelope := map[string]any{"schema_version": 2, "kind": "operational_value_evidence", "operational_value_evidence": packet} - content, _ := json.Marshal(envelope) - module := `export const definition={schemaVersion:3,repository:"org/repo",slug:"daily-review",sourcePath:".github/workflows/daily-review.md",adoption:{adoptedAt:"2026-10-01T00:00:00.000Z"},evidence:{repositories:["org/repo"],window:{cadenceDays:1,maturationDays:0}}}; export async function collectValueEvidence(){return ` + string(content) + `;}` - collector := filepath.Join(directory, "collector.mjs") - if err := os.WriteFile(collector, []byte(module), 0o600); err != nil { - t.Fatal(err) - } - relative, _ := filepath.Rel(root, collector) - policy := map[string]any{ - "version": 1, "gh-aw-version": "v0.92.5", "control-plane": map[string]any{ - "scope": map[string]any{"allowed-owners": []string{"org"}, "allowed-repositories": []string{"org/repo"}}, - "activity": map[string]any{"repositories": []string{"org/repo"}}, - "campaigns": map[string]any{"activity": map[string]any{"workflow-values": []any{ - map[string]any{"repository": "org/repo", "workflow": "daily-review", "collector": filepath.ToSlash(relative)}, - }}}, - }, - } - policyPath := filepath.Join(directory, "cao.json") - policyJSON, _ := json.Marshal(policy) - if err := os.WriteFile(policyPath, policyJSON, 0o600); err != nil { - t.Fatal(err) - } - previous := filepath.Join(directory, "previous.jsonl") - if err := os.WriteFile(previous, nil, 0o600); err != nil { - t.Fatal(err) - } - config := Config{Root: root, Database: filepath.Join(directory, "activity.sqlite"), Repositories: []string{"org/repo"}, ObservedAt: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), Environment: []string{"PATH=" + os.Getenv("PATH")}} - entries, err := WorkflowRegistrations(context.Background(), config, policyPath) - if err != nil || len(entries) != 1 || entries[0].Workflow != "daily-review" { - t.Fatalf("registrations=%+v err=%v", entries, err) - } - packets, warnings, err := CollectWorkflows(context.Background(), config, policyPath, filepath.Join(directory, "proofs"), previous) - if err != nil || len(packets) != 1 || len(warnings) != 0 || packets[0]["observedAt"] != packet["observedAt"] { - t.Fatalf("packets=%+v warnings=%v err=%v", packets, warnings, err) - } - if err := os.WriteFile(previous, append(content, '\n'), 0o600); err != nil { - t.Fatal(err) - } - packets, warnings, err = CollectWorkflows(context.Background(), config, policyPath, filepath.Join(directory, "proofs"), previous) - if err != nil || len(packets) != 0 || len(warnings) != 0 { - t.Fatalf("cadence was not satisfied by accepted canonical evidence: packets=%+v warnings=%v err=%v", packets, warnings, err) - } - config.Repositories = []string{"org/outside"} - packets, warnings, err = CollectWorkflows(context.Background(), config, policyPath, filepath.Join(directory, "proofs"), previous) - if err != nil || len(packets) != 0 || len(warnings) != 0 { - t.Fatalf("runtime repository scope was enlarged: packets=%+v warnings=%v err=%v", packets, warnings, err) - } -} diff --git a/server/internal/postgresx/acquisition_value_batch.go b/server/internal/postgresx/acquisition_value_batch.go new file mode 100644 index 0000000000..f97ace1ffd --- /dev/null +++ b/server/internal/postgresx/acquisition_value_batch.go @@ -0,0 +1,301 @@ +package postgresx + +import ( + "context" + "errors" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/githubnext/gh-cao/server/internal/collectnative" + "github.com/githubnext/gh-cao/server/internal/model" + "github.com/githubnext/gh-cao/server/internal/query" +) + +// ReadValueBatch restores all metric packets and their shared proof from one +// enrollment-pinned snapshot. A workflow with no packets is not an error. +func (a Acquisition) ReadValueBatch( + ctx context.Context, scope collectnative.EnrollmentScope, workflow string, +) ([]model.Row, []ValueProofFile, error) { + if scope.Epoch <= 0 || scope.InstallationID <= 0 || !validEnrollmentRepository(scope.Repository) || + !evidenceSlug.MatchString(workflow) { + return nil, nil, collectnative.ErrIdentity + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + connection, err := pgx.ConnectConfig(ctx, a.store.config.Copy()) + if err != nil { + return nil, nil, err + } + defer func() { _ = connection.Close(context.WithoutCancel(ctx)) }() + tx, err := connection.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + return nil, nil, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if err := a.store.fenceValueScope(ctx, tx, scope); err != nil { + return nil, nil, err + } + identity := model.Row{ + "repository": scope.Repository, "sourcePath": ".github/workflows/" + workflow + ".md", + "repositoryId": "repository:" + evidenceComponent(scope.Repository), + "workflowId": "workflow:" + evidenceComponent(scope.Repository+":.github/workflows/"+workflow+".md"), + } + known, err := valueParentKnown(ctx, tx, a.store.namespace, identity) + if err != nil { + return nil, nil, err + } + if !known { + return nil, nil, ErrValueParentMissing + } + rows, err := tx.Query(ctx, `SELECT id FROM operational_value_evidence WHERE namespace=$1 + AND repository_id=$2 AND workflow_id=$3 ORDER BY id LIMIT 4`, + a.store.namespace, identity["repositoryId"], identity["workflowId"]) + if err != nil { + return nil, nil, err + } + ids := []string{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return nil, nil, err + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, nil, err + } + if len(ids) > 3 { + return nil, nil, ErrValueInvalid + } + packets := make([]model.Row, 0, len(ids)) + for _, id := range ids { + packet, err := readValuePacket(ctx, tx, a.store.namespace, id) + if err != nil { + return nil, nil, err + } + packets = append(packets, packet) + } + proofRows, err := tx.Query(ctx, `SELECT path,digest,content FROM cao_value_proof_files + WHERE namespace=$1 AND repository=$2 AND workflow=$3 ORDER BY path COLLATE "C" LIMIT $4`, + a.store.namespace, scope.Repository, workflow, ValueProofMaxFiles+1) + if err != nil { + return nil, nil, err + } + files := []ValueProofFile{} + var total int + for proofRows.Next() { + var file ValueProofFile + if err := proofRows.Scan(&file.Path, &file.Digest, &file.Content); err != nil { + proofRows.Close() + return nil, nil, err + } + total += len(file.Content) + if len(file.Content) > ValueProofFileMax || total > ValueProofMaxBytes || len(files) >= ValueProofMaxFiles { + proofRows.Close() + return nil, nil, collectnative.ErrCapacity + } + files = append(files, file) + } + err = proofRows.Err() + proofRows.Close() + if err != nil { + return nil, nil, err + } + files, err = validateValueProofFiles(files) + if err != nil { + return nil, nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, nil, err + } + return packets, files, nil +} + +// PublishValueBatch replaces all metric packets and their proof in one +// transaction, checking the enrollment epoch and every starting packet first. +func (a Acquisition) PublishValueBatch( + ctx context.Context, scope collectnative.EnrollmentScope, previous, packets []model.Row, + files []ValueProofFile, updatedAt time.Time, +) (ValuePublication, error) { + if scope.Epoch <= 0 || scope.InstallationID <= 0 || !validEnrollmentRepository(scope.Repository) || + len(packets) != 3 || updatedAt.IsZero() { + return ValuePublication{}, collectnative.ErrIdentity + } + files, err := validateValueProofFiles(files) + if err != nil { + return ValuePublication{}, err + } + type item struct { + row, old model.Row + id, digest string + oldDigest string + } + items := make([]item, 0, len(packets)) + seen := map[string]bool{} + workflow, capture, cutoff := "", "", "" + for _, packet := range packets { + row, digest, err := normalizeValuePublication(packet) + if err != nil { + return ValuePublication{}, err + } + path := evidenceText(row["sourcePath"]) + name := strings.TrimSuffix(strings.TrimPrefix(path, ".github/workflows/"), ".md") + id := evidenceText(row["id"]) + if row["repository"] != scope.Repository || !evidenceSlug.MatchString(name) || + seen[id] || workflow != "" && (workflow != name || + capture != row["observedAt"] || cutoff != row["windowEnd"]) { + return ValuePublication{}, ErrValueInvalid + } + seen[id] = true + workflow, capture, cutoff = name, evidenceText(row["observedAt"]), evidenceText(row["windowEnd"]) + items = append(items, item{row: row, id: id, digest: digest}) + } + if len(previous) != 0 && len(previous) != len(items) { + return ValuePublication{}, ErrValueConflict + } + for _, packet := range previous { + old, digest, err := normalizeValuePublication(packet) + if err != nil { + return ValuePublication{}, err + } + found := false + for i := range items { + if items[i].id == old["id"] { + items[i].old, items[i].oldDigest = old, digest + found = true + break + } + } + if !found { + return ValuePublication{}, ErrValueConflict + } + } + sort.Slice(items, func(i, j int) bool { return items[i].id < items[j].id }) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + connection, err := pgx.ConnectConfig(ctx, a.store.config.Copy()) + if err != nil { + return ValuePublication{}, err + } + defer func() { _ = connection.Close(context.WithoutCancel(ctx)) }() + tx, err := connection.Begin(ctx) + if err != nil { + return ValuePublication{}, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + for _, lock := range []string{ + `SELECT pg_advisory_xact_lock_shared(712083241, 17484)`, + `SELECT pg_advisory_xact_lock(hashtextextended(current_schema() || ':' || $1, 0))`, + } { + args := []any{} + if strings.Contains(lock, "$1") { + args = append(args, a.store.namespace) + } + if _, err := tx.Exec(ctx, lock, args...); err != nil { + return ValuePublication{}, err + } + } + if err := a.store.fenceValueScope(ctx, tx, scope); err != nil { + return ValuePublication{}, err + } + result := ValuePublication{ID: items[0].id, ObservedAt: capture} + if err := tx.QueryRow(ctx, `SELECT revision FROM cao_state WHERE namespace=$1 FOR UPDATE`, + a.store.namespace).Scan(&result.Revision); err != nil { + return ValuePublication{}, err + } + allSame := true + for _, entry := range items { + known, err := valueParentKnown(ctx, tx, a.store.namespace, entry.row) + if err != nil { + return ValuePublication{}, err + } + if !known { + return ValuePublication{}, ErrValueParentMissing + } + current, err := readValuePacket(ctx, tx, a.store.namespace, entry.id) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return ValuePublication{}, err + } + currentDigest := "" + if current != nil { + _, currentDigest, err = normalizeValuePublication(current) + if err != nil { + return ValuePublication{}, err + } + } + if currentDigest != entry.oldDigest && currentDigest != entry.digest { + return ValuePublication{}, ErrValueConflict + } + if currentDigest != entry.digest { + allSame = false + if current != nil { + if err := validateValueReplacement(current, entry.row); err != nil { + return ValuePublication{}, err + } + } + } + } + proof := &valueProofPublication{repository: scope.Repository, workflow: workflow, files: files} + if allSame { + if err := a.store.sameValueProof(ctx, tx, proof); err != nil { + return ValuePublication{}, err + } + if err := tx.Commit(ctx); err != nil { + return ValuePublication{}, err + } + return result, nil + } + writer := &Writer{store: a.store, tx: tx, ordinals: map[string]int64{}, batches: map[string][][]any{}} + for _, source := range []string{"$operationalValueEvidence", "$operationalValueEvidence.facts"} { + var ordinal int64 + if err := tx.QueryRow(ctx, "SELECT coalesce(max(ordinal)+1,0) FROM "+ + query.SQLIdentifier(entityTables[source].name)+" WHERE namespace=$1", + a.store.namespace).Scan(&ordinal); err != nil { + return ValuePublication{}, err + } + writer.ordinals[source] = ordinal + } + for _, entry := range items { + for _, statement := range []string{ + `DELETE FROM operational_value_evidence_facts WHERE namespace=$1 AND packet_id=$2`, + `DELETE FROM operational_value_evidence WHERE namespace=$1 AND id=$2`, + } { + if _, err := tx.Exec(ctx, statement, a.store.namespace, entry.id); err != nil { + return ValuePublication{}, err + } + } + if err := writer.Append(ctx, "$operationalValueEvidence", entry.row); err != nil { + return ValuePublication{}, err + } + if _, err := tx.Exec(ctx, `INSERT INTO cao_value_publications(namespace,packet_id,digest,published_at) + VALUES($1,$2,$3,now()) ON CONFLICT(namespace,packet_id) DO UPDATE SET digest=excluded.digest,published_at=excluded.published_at`, + a.store.namespace, entry.id, entry.digest); err != nil { + return ValuePublication{}, err + } + } + if err := writer.Flush(ctx); err != nil { + return ValuePublication{}, err + } + if err := writer.valueQuality(ctx); err != nil { + return ValuePublication{}, err + } + if err := a.store.replaceValueProof(ctx, tx, scope.Repository, workflow, files, updatedAt); err != nil { + return ValuePublication{}, err + } + at, _ := evidenceTime(capture) + if err := tx.QueryRow(ctx, `UPDATE cao_state SET revision=revision+1,evaluated_at=greatest(evaluated_at,$2) + WHERE namespace=$1 RETURNING revision`, a.store.namespace, at).Scan(&result.Revision); err != nil { + return ValuePublication{}, err + } + if err := tx.Commit(ctx); err != nil { + return ValuePublication{}, err + } + result.Updated = true + return result, nil +} diff --git a/server/internal/postgresx/acquisition_value_batch_test.go b/server/internal/postgresx/acquisition_value_batch_test.go new file mode 100644 index 0000000000..60afaf6156 --- /dev/null +++ b/server/internal/postgresx/acquisition_value_batch_test.go @@ -0,0 +1,108 @@ +package postgresx + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "testing" + "time" + + "github.com/githubnext/gh-cao/server/internal/collectnative" + "github.com/githubnext/gh-cao/server/internal/model" +) + +func testValueBatch(t *testing.T, capture string) []model.Row { + t.Helper() + packets := make([]model.Row, 3) + for index := range packets { + content, err := json.Marshal(evidencePacket(t)) + if err != nil { + t.Fatal(err) + } + var row model.Row + if err := json.Unmarshal(content, &row); err != nil { + t.Fatal(err) + } + metric := "linked-" + string(rune('a'+index)) + row["id"] = "operational-value-evidence:" + url.QueryEscape("octo/example:verify-bugs:"+metric+":v1") + row["observedAt"] = capture + for _, raw := range row["facts"].([]any) { + fact := raw.(map[string]any) + if fact["fact-kind"] == "contract" { + fact["metric-id"] = metric + } + } + packets[index] = row + } + return packets +} + +func TestNativeValueBatchPublicationIsAtomicAndEpochFenced(t *testing.T) { + store, config := nativeTestStore(t) + replica, err := NewConfig(t.Context(), config) + if err != nil { + t.Fatal(err) + } + defer func() { _ = replica.Close() }() + packets := testValueBatch(t, "2026-10-20T12:00:00Z") + seedPublicationParents(t, store, packets[0]) + scope := nativeValueScope(t, store) + starting, files, err := replica.Acquisition().ReadValueBatch(t.Context(), scope, "verify-bugs") + if err != nil || len(starting) != 0 || len(files) != 0 { + t.Fatalf("new workflow had prior evidence: %d %d %v", len(starting), len(files), err) + } + proof := []ValueProofFile{{Path: "lifecycle.json", Content: []byte("complete")}} + accepted, err := store.Acquisition().PublishValueBatch(t.Context(), scope, nil, packets, proof, time.Now()) + if err != nil || !accepted.Updated { + t.Fatalf("complete metric batch rejected: %+v %v", accepted, err) + } + retry, err := replica.Acquisition().PublishValueBatch(t.Context(), scope, nil, packets, proof, time.Now()) + if err != nil || retry.Updated || retry.Revision != accepted.Revision { + t.Fatalf("exact committed retry failed: %+v %v", retry, err) + } + if _, err := replica.Acquisition().PublishValueBatch(t.Context(), scope, nil, packets, + []ValueProofFile{{Path: "lifecycle.json", Content: []byte("different")}}, time.Now()); !errors.Is(err, ErrValueConflict) { + t.Fatalf("committed evidence accepted different proof: %v", err) + } + starting, files, err = replica.Acquisition().ReadValueBatch(t.Context(), scope, "verify-bugs") + if err != nil || len(starting) != 3 || len(files) != 1 || string(files[0].Content) != "complete" { + t.Fatalf("replica did not read an atomic packet/proof snapshot: %d %+v %v", len(starting), files, err) + } + next := testValueBatch(t, "2026-10-20T13:00:00Z") + next[2]["contractDigest"] = "not-a-digest" + if _, err := store.Acquisition().PublishValueBatch(t.Context(), scope, starting, next, proof, time.Now()); err == nil { + t.Fatal("invalid third packet partially replaced the first two") + } + unchanged, files, err := store.Acquisition().ReadValueBatch(t.Context(), scope, "verify-bugs") + if err != nil || len(unchanged) != 3 || len(files) != 1 { + t.Fatalf("failed batch changed publication: %d %v", len(unchanged), err) + } + for _, row := range unchanged { + if row["observedAt"] != "2026-10-20T12:00:00Z" { + t.Fatalf("failed batch advanced a packet: %+v", row) + } + } + next[2]["contractDigest"] = packets[2]["contractDigest"] + updated, err := replica.Acquisition().PublishValueBatch(t.Context(), scope, starting, next, proof, time.Now()) + if err != nil || !updated.Updated { + t.Fatalf("complete batch did not advance: %+v %v", updated, err) + } + if _, err := store.Acquisition().PublishValueBatch(t.Context(), scope, starting, + testValueBatch(t, "2026-10-20T14:00:00Z"), proof, time.Now()); !errors.Is(err, ErrValueConflict) { + t.Fatalf("stale starting packets replaced newer evidence: %v", err) + } + authority := replica.NativeEnrollment() + if err := authority.WithMutation(t.Context(), func(ctx context.Context) error { + _, err := authority.Withdraw(ctx, scope.InstallationID, []string{scope.Repository}) + return err + }); err != nil { + t.Fatal(err) + } + if _, _, err := store.Acquisition().ReadValueBatch(t.Context(), scope, "verify-bugs"); !errors.Is(err, collectnative.ErrClaimLost) { + t.Fatalf("withdrawn scope read accepted evidence: %v", err) + } + if _, err := store.Acquisition().PublishValueBatch(t.Context(), scope, starting, next, proof, time.Now()); !errors.Is(err, collectnative.ErrClaimLost) { + t.Fatalf("withdrawn scope published accepted evidence: %v", err) + } +} diff --git a/server/internal/server/adx.go b/server/internal/server/adx.go index 9f77db450b..6f60474927 100644 --- a/server/internal/server/adx.go +++ b/server/internal/server/adx.go @@ -94,46 +94,22 @@ func newTraceWorker(ctx context.Context, config Config, collector *Collector) (* return nil, err } namespace := adxtrace.Namespace(config.ADX.Endpoint, config.ADX.Database) - if collector.native != nil { - database, ok := config.Database.(*postgresx.Store) - if !ok || database == nil || !config.ADXTraces.SharedStorage { - return nil, errors.New("native trace admission requires PostgreSQL shared storage and atomic enrollment withdrawal") - } - var readers []sessionexport.MiningReader - if config.ADXTraces.MiningEnabled { - readers = append(readers, database) - } - authority, worker, err := database.NativeEnrollmentWithTrace(ctx, *config.ADXTraces, namespace, client, readers...) - if err != nil { - return nil, err - } - collector.nativeAuthority(authority) - collector.native.collector.Destinations = append(collector.native.collector.Destinations, - collectnative.TraceDestination{Admission: worker}) - return worker, nil - } - if config.ADXTraces.SharedStorage { - database, ok := config.Database.(*postgresx.Store) - if !ok || database == nil { - return nil, errors.New("shared trace admission requires canonical PostgreSQL") - } - storage, err := database.TraceStorage(namespace, config.ADXTraces.MaxSpoolBytes, config.ADXTraces.MaxEntries) - if err != nil { - return nil, err - } - if config.ADXTraces.MiningEnabled { - return adxtrace.NewShared(ctx, *config.ADXTraces, namespace, client, storage, database) - } - return adxtrace.NewShared(ctx, *config.ADXTraces, namespace, client, storage) + database, ok := config.Database.(*postgresx.Store) + if !ok || database == nil || !config.ADXTraces.SharedStorage { + return nil, errors.New("native trace admission requires PostgreSQL shared storage and atomic enrollment withdrawal") } + var readers []sessionexport.MiningReader if config.ADXTraces.MiningEnabled { - database, ok := config.Database.(*postgresx.Store) - if !ok || database == nil { - return nil, errors.New("trace mining requires canonical PostgreSQL") - } - return adxtrace.New(ctx, *config.ADXTraces, namespace, client, database) + readers = append(readers, database) + } + authority, worker, err := database.NativeEnrollmentWithTrace(ctx, *config.ADXTraces, namespace, client, readers...) + if err != nil { + return nil, err } - return adxtrace.New(ctx, *config.ADXTraces, namespace, client) + collector.nativeAuthority(authority) + collector.native.collector.Destinations = append(collector.native.collector.Destinations, + collectnative.TraceDestination{Admission: worker}) + return worker, nil } func (a *App) ADXTraceStatus() adxtrace.Status { diff --git a/server/internal/server/backfill_stress_test.go b/server/internal/server/backfill_stress_test.go index 92f59fb219..b4cd4df69f 100644 --- a/server/internal/server/backfill_stress_test.go +++ b/server/internal/server/backfill_stress_test.go @@ -1,9 +1,14 @@ package server import ( + "bytes" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -14,6 +19,7 @@ import ( "time" "github.com/githubnext/gh-cao/server/internal/collect" + operationalpostgres "github.com/githubnext/gh-cao/server/internal/operational/postgres" "github.com/githubnext/gh-cao/server/internal/query" "github.com/githubnext/gh-cao/server/internal/simulator" ) @@ -43,10 +49,19 @@ func TestPostgresBackfillStress(t *testing.T) { h := newSyntheticBackfill(t, ctx, scenario) h.backfill.Queue.Debounce = time.Hour secret := "synthetic-backfill-webhook-signing-secret" - app, err := New(ctx, h.ops, Config{ + postgresURL := os.Getenv("CAO_POSTGRES_URL") + if postgresURL == "" { + postgresURL = os.Getenv("POSTGRES_URL") + } + nativeOps, err := operationalpostgres.New(ctx, postgresURL, service+"-native", operationalpostgres.Config{}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = nativeOps.Close() }() + app, err := New(ctx, nativeOps, Config{ Database: h.data, DatabaseQueriesPath: backfillDatabaseQueries, Listen: "127.0.0.1:0", SiteDirectory: t.TempDir(), AccessToken: testAccessToken, - Collector: &CollectorConfig{AppID: 1, AdmitOnly: true, QueueMaxLength: 5_000_000}, + Collector: &CollectorConfig{NativeAcquisition: true, AppID: 1, AdmitOnly: true, QueueMaxLength: 5_000_000}, WebhookSecret: secret, QueryCache: QueryCacheConfig{Disabled: true}, RateLimits: RateLimitConfig{ General: RateLimitPolicy{Capacity: 2_000_000, Window: time.Minute}, @@ -56,12 +71,38 @@ func TestPostgresBackfillStress(t *testing.T) { if err != nil { t.Fatal(err) } + names := make([]string, repositories) + for index := range names { + names[index] = fmt.Sprintf("simulator/repo-%05d", index+1) + } + if err := app.Collector().enrollment.AddRepositories(ctx, 1, names); err != nil { + t.Fatal(err) + } handler := app.Handler() var active atomic.Bool var overlapping atomic.Int64 local := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.URL.Path == "/api/github/webhook" && active.Load() { - overlapping.Add(1) + if request.URL.Path == "/api/github/webhook" { + if active.Load() { + overlapping.Add(1) + } + if request.Header.Get("X-GitHub-Event") == "workflow_run" { + payload, err := io.ReadAll(request.Body) + if err != nil || !validWebhookSignature(payload, request.Header.Get("X-Hub-Signature-256"), []byte(secret)) { + http.Error(writer, "invalid simulator delivery", http.StatusBadRequest) + return + } + payload, err = nativeSimulatorRunPayload(payload) + if err != nil { + http.Error(writer, "invalid simulator workflow", http.StatusBadRequest) + return + } + mac := hmac.New(sha256.New, []byte(secret)) + _, _ = mac.Write(payload) + request.Body = io.NopCloser(bytes.NewReader(payload)) + request.ContentLength = int64(len(payload)) + request.Header.Set("X-Hub-Signature-256", "sha256="+hex.EncodeToString(mac.Sum(nil))) + } } handler.ServeHTTP(writer, request) })) @@ -119,6 +160,10 @@ func TestPostgresBackfillStress(t *testing.T) { if err != nil || depth != int64(expected) { t.Fatalf("durable run stream = %d, want %d: %v", depth, expected, err) } + nativeStatus, err := app.Collector().reporter.Snapshot(ctx) + if err != nil || nativeStatus.QueueDepth == 0 || nativeStatus.Counters["webhookDuplicate"] == 0 { + t.Fatalf("native admission lost concurrent simulator work: %+v %v", nativeStatus, err) + } backfillReadiness(t, ctx, local, http.StatusOK) traceID := backfillStressSamples(t, ctx, local, scenario, horizon) if flush != nil { diff --git a/server/internal/server/collectenv.go b/server/internal/server/collectenv.go index bb166d96ca..e29c746ea1 100644 --- a/server/internal/server/collectenv.go +++ b/server/internal/server/collectenv.go @@ -71,12 +71,15 @@ func CollectorConfigFromEnv() (*CollectorConfig, error) { if err != nil || appID <= 0 { return nil, errors.New("CAO_COLLECT_APP_ID must be a positive GitHub App identifier") } - admitOnly := envBool("CAO_COLLECT_ADMIT_ONLY") - if native { - admitOnly, err = collectorFeatureBool("CAO_COLLECT_ADMIT_ONLY") - if err != nil { - return nil, err - } + if !native { + return nil, errors.New("app-backed collection requires CAO_COLLECT_NATIVE=true; the filesystem collector is retired") + } + if err := rejectRetiredCollectorEnvironment(os.Getenv); err != nil { + return nil, err + } + admitOnly, err := collectorFeatureBool("CAO_COLLECT_ADMIT_ONLY") + if err != nil { + return nil, err } var privateKey []byte if !admitOnly { @@ -96,13 +99,9 @@ func CollectorConfigFromEnv() (*CollectorConfig, error) { PrivateKeyPEM: privateKey, BaseURL: strings.TrimSpace(os.Getenv("CAO_COLLECT_GITHUB_API_URL")), UploadURL: strings.TrimSpace(os.Getenv("CAO_COLLECT_GITHUB_UPLOAD_URL")), - LakeDirectory: strings.TrimSpace(os.Getenv("CAO_COLLECT_LAKE_DIRECTORY")), CatalogRoot: strings.TrimSpace(os.Getenv("CAO_COLLECT_CATALOG_ROOT")), ControlRepository: strings.TrimSpace(os.Getenv("CAO_COLLECT_CONTROL_REPOSITORY")), PolicyPath: strings.TrimSpace(os.Getenv("CAO_POLICY_PATH")), - StaticInventoryPath: strings.TrimSpace(os.Getenv("CAO_COLLECT_STATIC_INVENTORY")), - NodeBinary: strings.TrimSpace(os.Getenv("CAO_COLLECT_NODE_BINARY")), - GitHubBinary: strings.TrimSpace(os.Getenv("CAO_COLLECT_GH_BINARY")), Consumer: strings.TrimSpace(os.Getenv("CAO_COLLECT_CONSUMER")), WindowDays: envInt("CAO_COLLECT_WINDOW_DAYS"), RunLimit: envInt("CAO_COLLECT_RUN_LIMIT"), diff --git a/server/internal/server/collecting.go b/server/internal/server/collecting.go index d20d4bf7ab..7c227fbb2a 100644 --- a/server/internal/server/collecting.go +++ b/server/internal/server/collecting.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "strconv" "strings" "sync" "time" @@ -40,7 +39,7 @@ type CollectorConfig struct { // instead of requiring a local inventory file on every replica. NativeInventoryDigest string // NativeValues separately selects the existing reviewed Value execution - // lane. Only this feature requires Node, CatalogRoot and PolicyPath. + // lane. Only this feature requires its reviewed catalog and policy. NativeValues bool // AppID and PrivateKeyPEM authenticate the GitHub App whose installations // define ingestion scope. @@ -50,20 +49,12 @@ type CollectorConfig struct { UploadURL string Transport http.RoundTripper - // LakeDirectory holds collected evidence in the published snapshot layout. - LakeDirectory string - // CatalogRoot contains activity/cao.mjs. + // CatalogRoot identifies the reviewed Value policy imports when selected. CatalogRoot string - // ControlRepository names the control repository for inventory discovery. + // ControlRepository names the reviewed inventory's control repository. ControlRepository string - // PolicyPath is the reviewed deployment policy resolved for inventory. + // PolicyPath is the reviewed Value policy when selected. PolicyPath string - // StaticInventoryPath is the source-bound inventory packaged with the - // collector image. - StaticInventoryPath string - - NodeBinary string - GitHubBinary string WindowDays int RunLimit int @@ -91,8 +82,8 @@ type CollectorConfig struct { QueueMaxLength int // AdmitOnly runs the admission half of the profile alone: the process // verifies deliveries and enqueues work, and never collects or projects. - // It therefore needs no App private key and no evidence lake, which keeps - // the internet-facing front end from holding credentials it cannot use. + // It therefore needs no App private key, which keeps the internet-facing + // front end from holding credentials it cannot use. AdmitOnly bool } @@ -104,78 +95,24 @@ func (config *CollectorConfig) Validate() error { if config.NativeAcquisition { return config.validateNative() } - if config.NativeValues || config.NativeInventoryDigest != "" || config.NativeInventory != nil { - return errors.New("native inventory and Value features require native acquisition") - } - if config.AppID <= 0 { - return errors.New("collection requires a GitHub App identifier") - } - if config.RepositoryRefreshInterval < 0 { - return errors.New("repository refresh interval must not be negative") - } - if config.AdmitOnly { - // Admission verifies deliveries against the webhook secret and writes - // to Redis. Requiring collection credentials here would place the App - // private key in a process that never calls GitHub. - if len(config.PrivateKeyPEM) > 0 { - return errors.New( - "admission-only collection must not be given the App private key; " + - "it never calls GitHub") - } - if config.Workers > 0 { - return errors.New("admission-only collection cannot run workers") - } - if config.RecoverDeliveries { - return errors.New("admission-only collection cannot replay deliveries") - } - } else { - if len(config.PrivateKeyPEM) == 0 { - return errors.New("collection requires a GitHub App private key") - } - if strings.TrimSpace(config.LakeDirectory) == "" { - return errors.New("collection requires an evidence lake directory") - } - if strings.TrimSpace(config.CatalogRoot) == "" { - return errors.New("collection requires the catalog root containing activity/cao.mjs") - } - if strings.TrimSpace(config.ControlRepository) == "" { - return errors.New("collection requires a control repository for inventory discovery") - } - if strings.TrimSpace(config.PolicyPath) == "" { - return errors.New("collection requires the reviewed control policy path") - } - if strings.TrimSpace(config.StaticInventoryPath) == "" { - return errors.New("collection requires the packaged static control-plane inventory") - } - } - if config.QueueMaxLength <= 0 { - config.QueueMaxLength = defaultQueueMaxLength - } - return nil + return fmt.Errorf("%w: hosted collection requires CAO_COLLECT_NATIVE=true; the filesystem collector is retired", + collectnative.ErrUnsupported) } -// Collector is the assembled collection profile. It satisfies Reconciler, so -// the existing rebuild and webhook plumbing works unchanged, and additionally -// satisfies EventAdmitter, so deliveries are queued instead of projected -// inline. +// Collector is the assembled native collection profile. It satisfies +// Reconciler and EventAdmitter so deliveries are queued, not projected inline. type Collector struct { - volatile bool - bootstrapDone chan struct{} - bootstrapOnce sync.Once - backfillMu sync.Mutex - config CollectorConfig - client *githubapp.Client - quota *githubquota.Service - enrollment collect.Enrollment - queue collect.Queue - lake collect.Lake - runner collect.Runner - projector collect.Projector - admitter collect.Admitter - backfill collect.Backfill - reporter collect.Reporter - replayer collect.DeliveryReplayer - native *nativeCollection + backfillMu sync.Mutex + config CollectorConfig + client *githubapp.Client + quota *githubquota.Service + enrollment collect.Enrollment + queue collect.Queue + admitter collect.Admitter + backfill collect.Backfill + reporter collect.Reporter + replayer collect.DeliveryReplayer + native *nativeCollection } var _ Reconciler = (*Collector)(nil) @@ -221,196 +158,39 @@ func NewCollector( if err != nil { return nil, fmt.Errorf("configure github quota: %w", err) } - searchQuota, err := githubquota.New(services.GitHubQuota, githubquota.Options{SafetyReserve: 1}) - if err != nil { - return nil, fmt.Errorf("configure search quota: %w", err) - } - quotaApp := "github-app-" + strconv.FormatInt(config.AppID, 10) - enrollment := collect.Enrollment{Metadata: services.Collection, Leases: services.Leases} queue := collect.Queue{ Tasks: services.Queue, Admission: services.Admission, Metadata: services.Collection, Leases: services.Leases, Deliveries: services.Deliveries, Metrics: services.IngestionMetrics, MaxLength: int64(config.QueueMaxLength), } - if config.NativeAcquisition { - return newNativeCollector(ctx, services, capabilities, data, config, quota, queue) - } - if config.AdmitOnly { - // Erasure is enqueued rather than performed, because this process has - // no evidence lake to erase from. - backfill := collect.Backfill{StateStore: services.State, Metadata: services.Collection, Quota: quota, QuotaApp: quotaApp} - collector := &Collector{ - volatile: volatile, - config: config, - quota: quota, - enrollment: enrollment, - queue: queue, - backfill: backfill, - admitter: collect.Admitter{Enrollment: enrollment, Queue: queue, Mining: true}, - reporter: collect.Reporter{ - Enrollment: enrollment, Queue: queue, Backfill: backfill, Metrics: services.IngestionMetrics, Data: data, - }, - } - return collector, nil - } - client, err := githubapp.New(githubapp.Config{ - AppID: config.AppID, - PrivateKeyPEM: config.PrivateKeyPEM, - BaseURL: config.BaseURL, - UploadURL: config.UploadURL, - Transport: config.Transport, - }) - if err != nil { - return nil, err - } - if err := client.ValidateRepositoryAccess(ctx, config.ControlRepository); err != nil { - return nil, fmt.Errorf("validate repository visibility: %w", err) - } - lake := collect.Lake{Directory: config.LakeDirectory} - if err := lake.Prepare(); err != nil { - return nil, err - } - budget := &githubapp.Budget{Metadata: services.Collection, Store: services.RateLimits, Floor: config.RateLimitFloor} - runner := collect.Runner{ - Lake: lake, - CatalogRoot: config.CatalogRoot, - NodeBinary: config.NodeBinary, - GitHubBinary: config.GitHubBinary, - GitHubAPIURL: config.BaseURL, - WindowDays: config.WindowDays, - RunLimit: config.RunLimit, - MaxStorageMB: config.MaxStorageMB, - RequestTimeoutMinutes: config.RequestTimeoutMinutes, - Timeout: config.CollectionTimeout, - Tokens: client, - Budget: budget, - } - if err := runner.Validate(); err != nil { - return nil, err - } - projector := collect.Projector{State: services.State, Leases: services.Leases, Data: data, - Lake: lake, - Enrollment: enrollment, - CatalogRoot: config.CatalogRoot, - NodeBinary: config.NodeBinary, - GitHubBinary: config.GitHubBinary, - GitHubAPIURL: config.BaseURL, - Tokens: client, - Budget: budget, - OutcomeClient: client, - Quota: quota, - QuotaApp: quotaApp, - RemoteQuota: quota, - RemoteSearchQuota: searchQuota, - RemoteHTTP: &http.Client{Transport: config.Transport, Timeout: 65 * time.Second, - CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, - RemoteValuePlan: &collect.RemoteValuePlanCache{}, - WindowDays: config.WindowDays, - DatabaseQueriesPath: databaseQueriesPath, - ControlRepository: config.ControlRepository, - PolicyPath: config.PolicyPath, - StaticInventoryPath: config.StaticInventoryPath, - MinInterval: config.MinProjectionInterval, - InventoryRepositoryLimit: config.InventoryLimit, - } - backfill := collect.Backfill{StateStore: services.State, Metadata: services.Collection, Enrollment: enrollment, Queue: queue, - Projector: projector, Lake: lake, Enumerator: client, RunEnumerator: client, - Quota: quota, QuotaApp: quotaApp, - WindowDays: config.WindowDays, - } - collector := &Collector{ - volatile: volatile, - config: config, - client: client, - quota: quota, - enrollment: enrollment, - queue: queue, - lake: lake, - runner: runner, - projector: projector, - admitter: collect.Admitter{ - Enrollment: enrollment, Queue: queue, Mining: true, - Projection: projector, - }, - backfill: backfill, - reporter: collect.Reporter{ - Enrollment: enrollment, Queue: queue, Backfill: backfill, - Budget: budget, Metrics: services.IngestionMetrics, Data: data, - }, - replayer: collect.DeliveryReplayer{State: services.State, Metadata: services.Collection, Client: client, Enabled: config.RecoverDeliveries}, - } - if volatile { - collector.bootstrapDone = make(chan struct{}) - collector.backfill.ReconstructScope = true - collector.backfill.ScopeLimit = config.InventoryLimit - if collector.backfill.ScopeLimit <= 0 { - collector.backfill.ScopeLimit = 100_000 - } - collector.backfill.ScopeReady = func(ctx context.Context) error { - if err := ctx.Err(); err != nil { - return err - } - collector.bootstrapOnce.Do(func() { close(collector.bootstrapDone) }) - return nil - } - } - return collector, nil + return newNativeCollector(ctx, services, capabilities, data, config, quota, queue) } -// Rebuild reprojects the evidence lake. Cold start and recovery both reuse the -// existing administrative rebuild endpoint through this method. +// Rebuild starts a native backfill through the existing administrative endpoint. func (c *Collector) Rebuild(ctx context.Context) (ingest.Result, error) { if c.config.AdmitOnly { return ingest.Result{}, ErrAdmitOnly } - if c.native != nil { - state, err := c.runBackfill(ctx) - return ingest.Result{Revision: state.Revision}, err - } - if c.volatile { - state, err := c.runBackfill(ctx) - return ingest.Result{Revision: state.Revision}, err - } - populated, err := c.lake.Populated() - if err != nil { - return ingest.Result{}, err - } - if populated { - return c.projector.Rebuild(ctx) - } - state, err := c.backfill.Run(ctx) - if err != nil { - return ingest.Result{}, err - } - return ingest.Result{Revision: state.Revision}, nil + state, err := c.runBackfill(ctx) + return ingest.Result{Revision: state.Revision}, err } -// Reconcile is the lease-taking path the default profile uses. Collection -// never needs it because Admit is preferred, but implementing it keeps the -// Reconciler contract total. +// Reconcile implements the Reconciler contract for callers that do not use +// delivery admission directly. func (c *Collector) Reconcile(ctx context.Context, event GitHubWebhook) (ingest.Result, error) { - if !c.RecoveryReady() { - return ingest.Result{}, ErrCollectionBootstrapping - } if _, err := c.admitter.Admit(ctx, event.Event, event.Payload); err != nil { if errors.Is(err, collect.ErrNotEnrolled) { return ingest.Result{}, nil } return ingest.Result{}, err } - if c.config.AdmitOnly || c.native != nil { - // Admission is complete; a collection worker projects. - return ingest.Result{}, nil - } - return c.projector.Project(ctx) + // Admission is complete; a native collection worker projects. + return ingest.Result{}, nil } // Admit queues collection for one verified delivery without taking the global // projection lease, so concurrent deliveries never contend. func (c *Collector) Admit(ctx context.Context, event GitHubWebhook) (map[string]any, error) { - if !c.RecoveryReady() { - return nil, ErrCollectionBootstrapping - } admission, err := c.admitter.AdmitDelivery(ctx, event.Event, event.Payload, event.Delivery, deliveryTTL) if err != nil { if errors.Is(err, collect.ErrNotEnrolled) { @@ -436,11 +216,7 @@ func (c *Collector) Start(ctx context.Context, onProjection func(revision int64) } func (c *Collector) start(startupCtx, ctx context.Context, onProjection func(revision int64), launch func(func())) error { - ensure := c.queue.Ensure - if c.native != nil { - ensure = c.queue.EnsureRuns - } - if err := ensure(startupCtx); err != nil { + if err := c.queue.EnsureRuns(startupCtx); err != nil { return err } if c.config.AdmitOnly { @@ -454,9 +230,6 @@ func (c *Collector) start(startupCtx, ctx context.Context, onProjection func(rev return } serverLog.Printf("cold start failed") - if c.native == nil && (!c.volatile || c.RecoveryReady()) { - return - } timer := time.NewTimer(30 * time.Second) select { case <-ctx.Done(): @@ -470,13 +243,6 @@ func (c *Collector) start(startupCtx, ctx context.Context, onProjection func(rev worker := c.Worker(fmt.Sprintf("%s-%d", c.consumer(), index)) worker.OnProjection = onProjection launch(func() { - if c.volatile { - select { - case <-ctx.Done(): - return - case <-c.bootstrapDone: - } - } if err := worker.Run(ctx); err != nil && ctx.Err() == nil { serverLog.Printf("collection worker stopped") } @@ -484,13 +250,6 @@ func (c *Collector) start(startupCtx, ctx context.Context, onProjection func(rev } if c.config.RecoverDeliveries { launch(func() { - if c.volatile { - select { - case <-ctx.Done(): - return - case <-c.bootstrapDone: - } - } c.recoverDeliveries(ctx) }) } @@ -498,18 +257,8 @@ func (c *Collector) start(startupCtx, ctx context.Context, onProjection func(rev return nil } -var ErrCollectionBootstrapping = errors.New("collection scope is being reconstructed; retry delivery") - func (c *Collector) RecoveryReady() bool { - if !c.volatile { - return true - } - select { - case <-c.bootstrapDone: - return true - default: - return false - } + return true } func (c *Collector) runBackfill(ctx context.Context) (collect.BackfillState, error) { @@ -542,39 +291,20 @@ func (c *Collector) recoverDeliveries(ctx context.Context) { } } -// ErrAdmitOnly reports an operation that requires collection credentials and -// an evidence lake on a process configured to admit deliveries only. +// ErrAdmitOnly reports an operation that requires collection credentials on a +// process configured to admit deliveries only. var ErrAdmitOnly = errors.New( "this process admits deliveries only; run the collect or backfill role") // Worker builds a standalone collection worker for the collect role. func (c *Collector) Worker(consumer string) collect.Worker { - if c.native != nil { - worker := c.native.worker - worker.Consumer = consumer - return collect.Worker{Native: &worker} - } - return collect.Worker{ - Queue: c.queue, Runner: c.runner, Projector: c.projector, - Enrollment: c.enrollment, Consumer: consumer, Project: true, - ScopeRefresher: c.scopeRefresher(), - Discovery: c.RepositoryDiscovery(), - } -} - -func (c *Collector) scopeRefresher() collect.ScopeRefresher { - return collect.ScopeRefresher{Client: c.client, Quota: c.quota, - QuotaApp: "github-app-" + strconv.FormatInt(c.config.AppID, 10), - RepositoryLimit: c.config.InventoryLimit} + worker := c.native.worker + worker.Consumer = consumer + return collect.Worker{Native: &worker} } func (c *Collector) RepositoryDiscovery() collect.RepositoryDiscovery { - if c.native != nil { - return collect.RepositoryDiscovery{Native: &c.native.worker} - } - return collect.RepositoryDiscovery{Client: c.client, Enrollment: c.enrollment, - Queue: c.queue, State: c.projector.State, Leases: c.enrollment.Leases, - Interval: c.config.RepositoryRefreshInterval} + return collect.RepositoryDiscovery{Native: &c.native.worker} } // Backfill exposes cold start for the backfill role. diff --git a/server/internal/server/collecting_test.go b/server/internal/server/collecting_test.go index 53cb2acce6..4a39e2cb18 100644 --- a/server/internal/server/collecting_test.go +++ b/server/internal/server/collecting_test.go @@ -136,10 +136,9 @@ func TestCollectionStatusRequiresHostedAdministrator(t *testing.T) { func TestProfilesAreMutuallyExclusive(t *testing.T) { collector := &CollectorConfig{ - AppID: 1, - PrivateKeyPEM: []byte("key"), - LakeDirectory: t.TempDir(), - CatalogRoot: t.TempDir(), + NativeAcquisition: true, + AppID: 1, + PrivateKeyPEM: []byte("key"), } t.Run("published snapshots and collection cannot both be configured", func(t *testing.T) { err := validateProfileExclusivity(Config{ @@ -206,8 +205,28 @@ func TestCollectorConfigFromEnvSelectsTheDefaultProfileWhenUnset(t *testing.T) { } func TestCollectorConfigFromEnvFailsClosed(t *testing.T) { + t.Run("an App without native acquisition is rejected before reading credentials", func(t *testing.T) { + t.Setenv("CAO_COLLECT_APP_ID", "12345") + t.Setenv("CAO_COLLECT_NATIVE", "false") + if _, err := CollectorConfigFromEnv(); err == nil || !strings.Contains(err.Error(), "CAO_COLLECT_NATIVE=true") { + t.Fatalf("expected the retired collector to be rejected explicitly, got %v", err) + } + }) + t.Run("retired collector settings are rejected", func(t *testing.T) { + for _, name := range []string{"CAO_COLLECT_LAKE_DIRECTORY", "CAO_COLLECT_STATIC_INVENTORY", "CAO_COLLECT_GH_BINARY", "CAO_COLLECT_NODE_BINARY"} { + t.Run(name, func(t *testing.T) { + t.Setenv("CAO_COLLECT_APP_ID", "12345") + t.Setenv("CAO_COLLECT_NATIVE", "true") + t.Setenv(name, "/retired") + if _, err := CollectorConfigFromEnv(); err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("expected retired setting %s to be rejected, got %v", name, err) + } + }) + } + }) t.Run("an incomplete configuration is rejected", func(t *testing.T) { t.Setenv("CAO_COLLECT_APP_ID", "12345") + t.Setenv("CAO_COLLECT_NATIVE", "true") t.Setenv("CAO_COLLECT_PRIVATE_KEY", "") t.Setenv("CAO_COLLECT_PRIVATE_KEY_BASE64", "") t.Setenv("CAO_COLLECT_PRIVATE_KEY_FILE", "") @@ -224,15 +243,11 @@ func TestCollectorConfigFromEnvFailsClosed(t *testing.T) { } func TestCollectorConfigFromEnvReadsTheCollectionProfile(t *testing.T) { - lake := t.TempDir() - catalog := t.TempDir() t.Setenv("CAO_COLLECT_APP_ID", "12345") + t.Setenv("CAO_COLLECT_NATIVE", "true") t.Setenv("CAO_COLLECT_PRIVATE_KEY", "-----BEGIN PRIVATE KEY-----") - t.Setenv("CAO_COLLECT_LAKE_DIRECTORY", lake) - t.Setenv("CAO_COLLECT_CATALOG_ROOT", catalog) + t.Setenv("CAO_COLLECT_NATIVE_INVENTORY_DIGEST", strings.Repeat("a", 64)) t.Setenv("CAO_COLLECT_CONTROL_REPOSITORY", "octo/control") - t.Setenv("CAO_POLICY_PATH", "/app/.github/workflows/cao.hosted.json") - t.Setenv("CAO_COLLECT_STATIC_INVENTORY", "/app/catalog/control-plane-inventory.json") t.Setenv("CAO_COLLECT_WORKERS", "4") t.Setenv("CAO_COLLECT_PROJECTION_INTERVAL", "90s") t.Setenv("CAO_COLLECT_RECOVER_DELIVERIES", "true") @@ -256,8 +271,7 @@ func TestCollectorConfigFromEnvReadsTheCollectionProfile(t *testing.T) { if config.RepositoryRefreshInterval.Minutes() != 30 { t.Fatalf("repository refresh interval = %s, want 30m", config.RepositoryRefreshInterval) } - if config.PolicyPath != "/app/.github/workflows/cao.hosted.json" || - config.StaticInventoryPath != "/app/catalog/control-plane-inventory.json" { + if !config.NativeAcquisition || config.NativeInventoryDigest != strings.Repeat("a", 64) { t.Fatalf("unexpected inventory configuration: %+v", config) } } @@ -333,19 +347,19 @@ func TestCollectorPrivateKeyDecodedContentStillRequiresAValidPEM(t *testing.T) { // the App private key would place a credential it cannot use in the // internet-facing process. func TestAdmitOnlyRefusesCollectionCredentials(t *testing.T) { - config := CollectorConfig{AppID: 42, AdmitOnly: true} + config := CollectorConfig{NativeAcquisition: true, AppID: 42, AdmitOnly: true} if err := config.Validate(); err != nil { t.Fatalf("admission-only configuration was rejected: %v", err) } - withKey := CollectorConfig{AppID: 42, AdmitOnly: true, PrivateKeyPEM: []byte("key")} + withKey := CollectorConfig{NativeAcquisition: true, AppID: 42, AdmitOnly: true, PrivateKeyPEM: []byte("key")} if err := withKey.Validate(); err == nil { t.Fatal("admission-only accepted the App private key") } - withWorkers := CollectorConfig{AppID: 42, AdmitOnly: true, Workers: 1} + withWorkers := CollectorConfig{NativeAcquisition: true, AppID: 42, AdmitOnly: true, Workers: 1} if err := withWorkers.Validate(); err == nil { t.Fatal("admission-only accepted collection workers") } - collecting := CollectorConfig{AppID: 42} + collecting := CollectorConfig{NativeAcquisition: true, AppID: 42} if err := collecting.Validate(); err == nil { t.Fatal("the collection profile was accepted without a private key") } diff --git a/server/internal/server/native_collecting.go b/server/internal/server/native_collecting.go index 4cf74bc334..511d26054d 100644 --- a/server/internal/server/native_collecting.go +++ b/server/internal/server/native_collecting.go @@ -116,7 +116,7 @@ func newNativeCollector(ctx context.Context, services operational.OperationalSer } values := &collect.NativeValueService{Projector: collect.Projector{ Data: data, State: services.State, Leases: services.Leases, Enrollment: enrollment, - CatalogRoot: config.CatalogRoot, PolicyPath: config.PolicyPath, NodeBinary: config.NodeBinary, + CatalogRoot: config.CatalogRoot, PolicyPath: config.PolicyPath, GitHubAPIURL: config.BaseURL, Tokens: client, QuotaApp: quotaApp, RemoteQuota: quota, RemoteSearchQuota: search, RemoteValuePlan: &collect.RemoteValuePlanCache{}, RemoteHTTP: &http.Client{Transport: config.Transport, Timeout: 65 * time.Second, diff --git a/server/internal/server/native_collecting_test.go b/server/internal/server/native_collecting_test.go index b094790f1e..ce1ec90bfa 100644 --- a/server/internal/server/native_collecting_test.go +++ b/server/internal/server/native_collecting_test.go @@ -17,6 +17,7 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "strings" "testing" "time" @@ -139,14 +140,14 @@ func TestNativeHostedAssemblyRunsWithoutLegacyFilesOrExecutables(t *testing.T) { config := CollectorConfig{NativeAcquisition: true, NativeInventory: &snapshot, AppID: 42, Workers: 1, WindowDays: 7, PrivateKeyPEM: pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), BaseURL: api.URL + "/api/v3/", ControlRepository: "octo/api", - LakeDirectory: "/must-not-read", CatalogRoot: "/must-not-read", PolicyPath: "/must-not-read", StaticInventoryPath: "/must-not-read"} + CatalogRoot: "/must-not-read", PolicyPath: "/must-not-read"} t.Setenv("PATH", "/nonexistent") collector, err := NewCollector(t.Context(), operational, data, config, "") if err != nil { t.Fatal(err) } - if collector.lake.Directory != "" || collector.runner.CatalogRoot != "" || collector.projector.State != nil { - t.Fatal("native hosted assembly constructed the legacy filesystem pipeline") + if collector.native == nil || collector.native.inventory == nil || collector.native.worker.Runs.Collector == nil { + t.Fatal("native hosted assembly omitted the acquisition worker") } replicaConfig := config replicaConfig.NativeInventory = nil @@ -158,8 +159,14 @@ func TestNativeHostedAssemblyRunsWithoutLegacyFilesOrExecutables(t *testing.T) { } valueConfig := replicaConfig valueConfig.NativeValues = true - if _, err := NewCollector(t.Context(), operational, data, valueConfig, ""); err == nil || !strings.Contains(err.Error(), "requires Node") { - t.Fatalf("selected Value execution silently ignored its missing Node dependency: %v", err) + valueConfig.CatalogRoot, err = filepath.Abs("../../..") + if err != nil { + t.Fatal(err) + } + valueConfig.PolicyPath = filepath.Join(valueConfig.CatalogRoot, ".github", "workflows", "cao.json") + valueCollector, err := NewCollector(t.Context(), operational, data, valueConfig, "") + if err != nil || valueCollector.native.values == nil { + t.Fatalf("selected Go Value execution required Node or omitted the worker: %v", err) } if _, err := collector.Rebuild(t.Context()); err != nil { t.Fatal("native cold-start assembly:", err) @@ -204,8 +211,9 @@ func TestNativeHostedAssemblyRunsWithoutLegacyFilesOrExecutables(t *testing.T) { } admission, err := NewCollector(t.Context(), operational, data, CollectorConfig{NativeAcquisition: true, AppID: 42, AdmitOnly: true}, "") - if err != nil || admission.client != nil || admission.native.inventory != nil || admission.runner.CatalogRoot != "" { - t.Fatalf("native admission acquired worker credentials/files: %v", err) + if err != nil || admission.native == nil || admission.client != nil || + admission.native.inventory != nil || admission.native.worker.Client != nil { + t.Fatalf("native admission acquired worker credentials or inventory: %v", err) } unsupported := config unsupported.MaxStorageMB = 1 diff --git a/server/internal/server/native_collection_config.go b/server/internal/server/native_collection_config.go index 7bce54737c..316e1644b0 100644 --- a/server/internal/server/native_collection_config.go +++ b/server/internal/server/native_collection_config.go @@ -32,6 +32,9 @@ func (config *CollectorConfig) ApplyNativeEnvironment(lookup func(string) string if !config.NativeAcquisition { return nil } + if err := rejectRetiredCollectorEnvironment(lookup); err != nil { + return err + } admitOnly, err := collectorFeatureBoolWithLookup("CAO_COLLECT_ADMIT_ONLY", lookup) if err != nil { return err @@ -82,6 +85,15 @@ func (config *CollectorConfig) ApplyNativeEnvironment(lookup func(string) string return nil } +func rejectRetiredCollectorEnvironment(lookup func(string) string) error { + for _, name := range []string{"CAO_COLLECT_LAKE_DIRECTORY", "CAO_COLLECT_STATIC_INVENTORY", "CAO_COLLECT_GH_BINARY", "CAO_COLLECT_NODE_BINARY"} { + if strings.TrimSpace(lookup(name)) != "" { + return fmt.Errorf("%s is unsupported by native collection", name) + } + } + return nil +} + func (config *CollectorConfig) validateNative() error { if config.AppID <= 0 { return errors.New("native collection requires a GitHub App identifier") diff --git a/server/internal/server/native_collection_config_test.go b/server/internal/server/native_collection_config_test.go index 1bb0962b2d..09db625487 100644 --- a/server/internal/server/native_collection_config_test.go +++ b/server/internal/server/native_collection_config_test.go @@ -12,9 +12,9 @@ import ( func TestNativeCollectionConfigurationHasFeatureSpecificDependencies(t *testing.T) { config := CollectorConfig{NativeAcquisition: true, AppID: 7, PrivateKeyPEM: []byte("validated by App adapter"), ControlRepository: "octo/api", NativeInventoryDigest: strings.Repeat("a", 64), - LakeDirectory: "/must-not-read", CatalogRoot: "/must-not-read", PolicyPath: "/must-not-read"} + CatalogRoot: "/must-not-read", PolicyPath: "/must-not-read"} if err := config.validateNative(); err != nil || config.QueueMaxLength != defaultQueueMaxLength { - t.Fatalf("native-only configuration required Activity files: %v", err) + t.Fatalf("native configuration required Value files without Value enabled: %v", err) } admission := CollectorConfig{NativeAcquisition: true, AppID: 7, AdmitOnly: true} if err := admission.validateNative(); err != nil { diff --git a/server/internal/server/operational_services_test.go b/server/internal/server/operational_services_test.go index 1839504b83..f8106f5dc6 100644 --- a/server/internal/server/operational_services_test.go +++ b/server/internal/server/operational_services_test.go @@ -51,7 +51,7 @@ func focusedTestStore(store operational.Store) focusedOperationalStore { } } -func TestFocusedServicesComposeApplicationAndCollector(t *testing.T) { +func TestFocusedServicesComposeApplicationAndValidateCollection(t *testing.T) { store := focusedTestStore(redisx.NewStore(emptyRedisClient{}, "focused-services")) app, err := New(t.Context(), store, Config{ Database: constructorDatabase(), SiteDirectory: t.TempDir(), @@ -63,21 +63,9 @@ func TestFocusedServicesComposeApplicationAndCollector(t *testing.T) { if app.services != store.services { t.Fatal("application substituted the selected focused services") } - collector, err := NewCollector(t.Context(), store, constructorDatabase(), CollectorConfig{ - AppID: 1, AdmitOnly: true, - }, "") - if err != nil { - t.Fatal(err) - } - if collector.enrollment.Metadata != store.services.Collection || - collector.enrollment.Leases != store.services.Leases || - collector.queue.Tasks != store.services.Queue || - collector.queue.Admission != store.services.Admission || - collector.queue.Deliveries != store.services.Deliveries || - collector.queue.Metrics != store.services.IngestionMetrics || - collector.backfill.StateStore != store.services.State || - collector.reporter.Metrics != store.services.IngestionMetrics { - t.Fatal("collector substituted focused collection services") + if err := operational.ValidateOperationalServices(store.Capabilities(), store.services, + operational.Requirements{SingleProcess: true, AllowVolatile: true, Collection: true}); err != nil { + t.Fatalf("focused collection services were rejected: %v", err) } } @@ -95,9 +83,8 @@ func TestCollectorRejectsMissingFocusedServices(t *testing.T) { t.Run(name, func(t *testing.T) { store := focusedTestStore(redisx.NewStore(emptyRedisClient{}, "missing-focused-services")) remove(&store.services) - _, err := NewCollector(t.Context(), store, constructorDatabase(), CollectorConfig{ - AppID: 1, AdmitOnly: true, - }, "") + err := operational.ValidateOperationalServices(store.Capabilities(), store.services, + operational.Requirements{SingleProcess: true, AllowVolatile: true, Collection: true}) if !errors.Is(err, operational.ErrUnsupported) { t.Fatalf("incomplete collector services accepted: %v", err) } diff --git a/server/internal/server/operational_test.go b/server/internal/server/operational_test.go index b35345f1fd..4c8950126d 100644 --- a/server/internal/server/operational_test.go +++ b/server/internal/server/operational_test.go @@ -1,25 +1,13 @@ package server import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "errors" - "io" "net/http" "net/http/httptest" "net/url" - "path/filepath" "strings" "testing" - "time" - "github.com/githubnext/gh-cao/server/internal/audit" - "github.com/githubnext/gh-cao/server/internal/githubapp" - "github.com/githubnext/gh-cao/server/internal/ingest" "github.com/githubnext/gh-cao/server/internal/operational/memory" - "github.com/githubnext/gh-cao/server/internal/simulator" ) func memoryOAuthApp(t *testing.T, githubURL string) *App { @@ -115,193 +103,15 @@ func TestMemoryRequiresOAuthAndExplicitVolatility(t *testing.T) { } } -type unavailableMemoryScope struct{ client *githubapp.Client } - -func (s unavailableMemoryScope) ListInstallations(ctx context.Context) ([]githubapp.Installation, error) { - return s.client.ListInstallations(ctx) -} -func (unavailableMemoryScope) ListRepositories(context.Context, int64) ([]githubapp.Repository, error) { - return nil, errors.New("scope unavailable") -} - -func TestMemoryCollectorHTTPBootstrapAndPopulatedAdminBackfill(t *testing.T) { - data := integrationDatabase(t) +func TestMemoryCannotAdmitNativeCollectionWithoutDurableScope(t *testing.T) { store, err := memory.New(memory.Config{}) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) - scenario := simulator.Scenario{ - Name: "memory-collector", Repositories: 1, - History: &simulator.History{Days: 1, RunsPerDay: 1, AsOf: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)}, - } - api, err := simulator.NewAPIHandler(scenario, 1) - if err != nil { - t.Fatal(err) - } - endpoint := httptest.NewServer(api) - t.Cleanup(endpoint.Close) - transport, err := simulator.NewLocalTransport(endpoint.URL) - if err != nil { - t.Fatal(err) - } - t.Cleanup(transport.CloseIdleConnections) - key, err := stressAppKey() - if err != nil { - t.Fatal(err) - } - lake := t.TempDir() - if _, err := scenario.WriteLake(t.Context(), lake, 1); err != nil { - t.Fatal(err) - } - if _, err := ingest.Run(t.Context(), data, lake, ingest.Options{DatabaseQueriesPath: backfillDatabaseQueries}); err != nil { - t.Fatal(err) - } - root, err := filepath.Abs("../../..") - if err != nil { - t.Fatal(err) - } - host, err := memoryHostPolicy().resolve(mapLookup(nil)) - if err != nil { - t.Fatal(err) - } - github := fakeGitHub(t, fakeGitHubOptions{membershipState: "active", accessExpiresIn: 3600}) - auditStore, err := newHostedCLISQLiteTestStore(t, filepath.Join(t.TempDir(), "audit.sqlite")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = auditStore.Close() }) - const secret = "memory-integration-webhook-secret" - app, err := New(t.Context(), focusedTestStore(store), Config{ - Database: data, DatabaseQueriesPath: backfillDatabaseQueries, SiteDirectory: hostedTestSite(t), - HostProfile: host.Profile, Listen: "0.0.0.0:8443", CertFile: "certificate.pem", KeyFile: "key.pem", - SingleReplicaConfirmed: true, AllowVolatile: true, - Proxy: ProxyPolicy{AllowedHosts: []string{"dashboard.example.com"}, RequireHTTPS: true}, - GitHubOAuth: validOAuthConfig(github.URL), WebhookSecret: secret, AdminUsers: []string{"octocat"}, - Audit: audit.New(io.Discard).WithStore(auditStore), - Collector: &CollectorConfig{ - AppID: 1, PrivateKeyPEM: key, BaseURL: endpoint.URL + "/", Transport: transport, - LakeDirectory: lake, CatalogRoot: root, ControlRepository: "simulator/repo-00001", - PolicyPath: filepath.Join(root, ".github/workflows/cao.json"), - StaticInventoryPath: filepath.Join(lake, "inventory-sources.json"), Workers: 1, WindowDays: 1, - InventoryLimit: 10, QueueMaxLength: 20, - }, - }) - if err != nil { - t.Fatal(err) - } - collector := app.Collector() - if !collector.volatile || collector.RecoveryReady() || !collector.backfill.ReconstructScope || - collector.backfill.ScopeReady == nil || collector.backfill.ScopeLimit != 10 { - t.Fatal("production constructor did not wire volatile recovery") - } - // The simulator publishes a compacted lake, so use the existing no-recompaction replay mode. - collector.backfill.Projector.CatalogRoot = "" - webhook := func() *httptest.ResponseRecorder { - payload := `{"action":"completed","workflow_run":{"id":1},"repository":{"full_name":"simulator/repo-00001"}}` - request := azureRequest(t, http.MethodPost, "/api/github/webhook") - request.Body = io.NopCloser(strings.NewReader(payload)) - request.Header.Set("X-GitHub-Delivery", "same-retryable-memory-delivery") - request.Header.Set("X-GitHub-Event", "workflow_run") - mac := hmac.New(sha256.New, []byte(secret)) - _, _ = mac.Write([]byte(payload)) - request.Header.Set("X-Hub-Signature-256", "sha256="+hex.EncodeToString(mac.Sum(nil))) - response := httptest.NewRecorder() - app.Handler().ServeHTTP(response, request) - return response - } - readiness := func(want int) { - t.Helper() - response := httptest.NewRecorder() - app.Handler().ServeHTTP(response, azureRequest(t, http.MethodGet, "/api/readiness")) - if response.Code != want { - t.Fatalf("readiness = %d, want %d: %s", response.Code, want, response.Body.String()) - } - } - readiness(http.StatusServiceUnavailable) - collector.backfill.Enumerator = unavailableMemoryScope{collector.client} - if _, err := collector.runBackfill(t.Context()); err == nil || collector.RecoveryReady() { - t.Fatal("partial reconstruction declared ready") - } - if response := webhook(); response.Code != http.StatusServiceUnavailable { - t.Fatalf("partial bootstrap admitted delivery: %d: %s", response.Code, response.Body.String()) - } - collector.backfill.Enumerator = collector.client - state, err := collector.runBackfill(t.Context()) - if err != nil || !state.LakeReplayed || state.QueuedRepositories != 1 || state.QueuedRunTasks != 1 { - t.Fatalf("memory backfill: %+v, %v", state, err) - } - readiness(http.StatusOK) - if response := webhook(); response.Code != http.StatusAccepted || strings.Contains(response.Body.String(), `"duplicate":true`) { - t.Fatalf("retry was lost or acknowledged before bootstrap: %d: %s", response.Code, response.Body.String()) - } - if response := webhook(); response.Code != http.StatusAccepted || !strings.Contains(response.Body.String(), `"duplicate":true`) { - t.Fatalf("delivery was not deduplicated: %d: %s", response.Code, response.Body.String()) - } - session, csrf := callbackSession(t, app) - request := azureRequest(t, http.MethodPost, "/api/admin/rebuild") - request.AddCookie(session) - request.AddCookie(csrf) - request.Header.Set("X-CSRF-Token", csrf.Value) - before := api.Stats().Requests - response := httptest.NewRecorder() - app.Handler().ServeHTTP(response, request) - if response.Code != http.StatusAccepted { - t.Fatalf("admin backfill rejected: %d: %s", response.Code, response.Body.String()) - } - deadline := time.Now().Add(10 * time.Second) - for { - status, err := app.readRebuildStatus(t.Context()) - if err != nil || status.State == "failed" { - t.Fatalf("admin backfill failed: %+v, %v", status, err) - } - if status.State == "succeeded" { - if status.AuditState != "recorded" { - t.Fatalf("admin backfill audit was not persisted: %+v", status) - } - break - } - if time.Now().After(deadline) { - t.Fatal("admin backfill did not finish") - } - time.Sleep(5 * time.Millisecond) - } - if api.Stats().Requests <= before { - t.Fatal("populated admin rebuild skipped fresh GitHub enumeration") - } - leases, err := collector.queue.Lease(t.Context(), "lifecycle-test", 20, 0) - if err != nil { - t.Fatal(err) - } - for _, lease := range leases { - if err := collector.queue.Complete(t.Context(), lease); err != nil { - t.Fatal(err) - } - } - collector.backfill.Enumerator = unavailableMemoryScope{collector.client} - runtimeCtx, cancelRuntime := context.WithCancel(t.Context()) - defer cancelRuntime() - if err := app.Start(runtimeCtx); err != nil { - t.Fatal(err) - } - stopCtx, cancelStop := context.WithTimeout(t.Context(), 2*time.Second) - defer cancelStop() - if err := app.Stop(stopCtx); err != nil { - t.Fatalf("memory worker, projector, or revocation work outlived shutdown: %v", err) - } -} - -func TestVolatileCollectionAdmissionIsRetryableDuringBootstrap(t *testing.T) { - collector := &Collector{volatile: true, bootstrapDone: make(chan struct{})} - result, err := collector.Admit(t.Context(), GitHubWebhook{Delivery: "retryable", Event: "workflow_run"}) - if !errors.Is(err, ErrCollectionBootstrapping) || result != nil { - t.Fatalf("bootstrap acknowledged unknown scope: %#v, %v", result, err) - } - if collector.RecoveryReady() { - t.Fatal("bootstrap reported ready") - } - close(collector.bootstrapDone) - if !collector.RecoveryReady() { - t.Fatal("reconstruction did not release admission") + _, err = NewCollector(t.Context(), focusedTestStore(store), integrationDatabase(t), + CollectorConfig{NativeAcquisition: true, AppID: 1, AdmitOnly: true}, "") + if err == nil || !strings.Contains(err.Error(), "volatile collection requires co-resident workers") { + t.Fatalf("native admission accepted volatile scope without durable recovery: %v", err) } } diff --git a/server/internal/server/operations_test.go b/server/internal/server/operations_test.go index 331a8f0e15..fc77bdba74 100644 --- a/server/internal/server/operations_test.go +++ b/server/internal/server/operations_test.go @@ -6,20 +6,21 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "fmt" "net/http" "net/http/httptest" + "net/url" "os" "reflect" - "strconv" "strings" "testing" "time" "github.com/githubnext/gh-cao/server/internal/collect" "github.com/githubnext/gh-cao/server/internal/ingest" + operationalpostgres "github.com/githubnext/gh-cao/server/internal/operational/postgres" "github.com/githubnext/gh-cao/server/internal/redisx" "github.com/githubnext/gh-cao/server/internal/simulator" + "github.com/githubnext/gh-cao/server/internal/testutil" ) type testReconciler struct { @@ -262,158 +263,170 @@ func TestHostedRebuildRequiresExplicitAdministrator(t *testing.T) { } } -func TestSimulatorUsesProductionWebhookAdmissionAndCollectionQueue(t *testing.T) { - database := integrationDatabase(t) - var client *redisx.Client - var err error - if endpoint := os.Getenv("CAO_SIMULATOR_REDIS_URL"); endpoint != "" { - client, err = redisx.New(endpoint) - if err == nil { - _, err = client.Do(t.Context(), "PING") - } - if err != nil { - t.Fatalf("connect to CAO_SIMULATOR_REDIS_URL: %v", err) - } - } else { - address, closeServer := fakeRedis(t) - t.Cleanup(closeServer) - client, err = redisx.New("redis://" + address) - if err != nil { - t.Fatal(err) - } +func TestSimulatorUsesProductionNativeWebhookAdmissionAndCollectionQueue(t *testing.T) { + database, pgconfig := testutil.PostgresWithConfig(t, t.Context(), "CAO_TEST_POSTGRES_URL", "POSTGRES_URL") + endpoint, err := url.Parse(pgconfig.ConnString()) + if err != nil { + t.Fatal(err) + } + parameters := endpoint.Query() + parameters.Set("search_path", pgconfig.RuntimeParams["search_path"]) + endpoint.RawQuery = parameters.Encode() + store, err := operationalpostgres.New(t.Context(), endpoint.String(), "simulator-native-admission", operationalpostgres.Config{}) + if err != nil { + t.Fatal(err) } + t.Cleanup(func() { _ = store.Close() }) site := t.TempDir() if err := os.WriteFile(site+"/index.html", []byte(""), 0o600); err != nil { t.Fatal(err) } const secret = "simulator-webhook-secret" - namespace := "simulator-test-" + strconv.FormatInt(time.Now().UnixNano(), 10) - app, err := New(t.Context(), redisx.NewStore(client, namespace), Config{ + app, err := New(t.Context(), store, Config{ Database: database, Listen: "127.0.0.1:0", SiteDirectory: site, AccessToken: strings.Repeat("x", 32), WebhookSecret: secret, - Collector: &CollectorConfig{AppID: 1, AdmitOnly: true}, + Collector: &CollectorConfig{NativeAcquisition: true, AppID: 1, AdmitOnly: true}, }) if err != nil { t.Fatal(err) } + repositories := []string{"simulator/repo-00001", "simulator/repo-00002", + "simulator/repo-00003", "simulator/repo-00004"} + if err := app.Collector().enrollment.AddRepositories(t.Context(), 1, repositories); err != nil { + t.Fatal(err) + } server := httptest.NewServer(app.Handler()) defer server.Close() - result, err := (simulator.Scenario{ + deliveries, err := (simulator.Scenario{ Name: "production-webhook-path", Repositories: 4, EventsPerRepository: 2, Seed: 3, Distribution: "uniform", DuplicateEvery: 2, - }).Deliver(t.Context(), server.Client(), server.URL+"/api/github/webhook", secret, 4) + }).Generate() if err != nil { t.Fatal(err) } - if result.Accepted != result.Attempts || result.Failed != 0 || result.Duplicates == 0 { - t.Fatalf("simulator did not exercise successful deduplicating admission: %#v", result) + var completed *simulator.Delivery + for _, delivery := range deliveries { + status, result := postNativeSimulatorDelivery(t, server, secret, delivery) + if status != http.StatusAccepted { + t.Fatalf("native simulator delivery %s: %d %#v", delivery.Event, status, result) + } + if delivery.Event == "workflow_run" { + if result["queued"] != true || result["duplicate"] != false { + t.Fatalf("native workflow was not durably admitted: %#v", result) + } + if completed == nil { + copy := delivery + completed = © + } + } } -} - -func TestSimulatorExercisesGoServerWithRedis(t *testing.T) { - endpoint := os.Getenv("CAO_SIMULATOR_REDIS_URL") - if endpoint == "" { - t.Skip("set CAO_SIMULATOR_REDIS_URL to run the Go server simulator integration") + if completed == nil { + t.Fatal("simulator did not generate a completed workflow") } - database := integrationDatabase(t) - client, err := redisx.New(endpoint) - if err != nil { - t.Fatal(err) + status, duplicate := postNativeSimulatorDelivery(t, server, secret, *completed) + if status != http.StatusAccepted || duplicate["duplicate"] != true || duplicate["queued"] != false { + t.Fatalf("native simulator delivery was not deduplicated: %d %#v", status, duplicate) } - if _, err := client.Do(t.Context(), "PING"); err != nil { - t.Fatalf("connect to CAO_SIMULATOR_REDIS_URL: %v", err) + stats, err := app.Collector().queue.Tasks.QueueStats(t.Context(), "collect:run-tasks", "") + if err != nil || stats.Length == 0 { + t.Fatalf("native admission lost queued run attempts: %+v %v", stats, err) } - - site := t.TempDir() - if err := os.WriteFile(site+"/index.html", []byte(""), 0o600); err != nil { + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL+"/api/v1/ingestion/health", nil) + if err != nil { t.Fatal(err) } - const secret = "simulator-webhook-secret" - namespace := "simulator-server-" + strconv.FormatInt(time.Now().UnixNano(), 10) - app, err := New(t.Context(), redisx.NewStore(client, namespace), Config{ - Database: database, - Listen: "127.0.0.1:0", - SiteDirectory: site, - AccessToken: testAccessToken, - WebhookSecret: secret, - Collector: &CollectorConfig{AppID: 1, AdmitOnly: true}, - }) + request.Header.Set("Authorization", "Bearer "+strings.Repeat("x", 32)) + response, err := server.Client().Do(request) // #nosec G704 -- the request targets the local httptest server. if err != nil { t.Fatal(err) } - if err := app.Collector().Start(t.Context(), nil); err != nil { - t.Fatalf("start Go server collector: %v", err) - } - // Admission-only servers queue scope changes for a separate worker. - repositories := make([]string, 8) - for index := range repositories { - repositories[index] = fmt.Sprintf("simulator/repo-%05d", index+1) + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + t.Fatalf("native ingestion health returned %d", response.StatusCode) } - if err := app.Collector().enrollment.AddRepositories(t.Context(), 1, repositories); err != nil { + var health collect.Status + if err := json.NewDecoder(response.Body).Decode(&health); err != nil { t.Fatal(err) } - server := httptest.NewServer(app.Handler()) - defer server.Close() - - ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) - defer cancel() - result, err := (simulator.Scenario{ - Name: "go-server-recovery-burst", - Repositories: 8, - EventsPerRepository: 5, - Seed: 29, - Distribution: "hot", - OutOfOrder: true, - DuplicateEvery: 3, - DropEvery: 7, - DelayEvery: 4, - Delay: "1ms", - ReplayCount: 3, - RemoveRepositories: true, - }).Deliver(ctx, server.Client(), server.URL+"/api/github/webhook", secret, 8) - if err != nil { - t.Fatalf("deliver simulator workload to Go server: %v", err) - } - if result.Accepted != result.Attempts || result.Failed != 0 || result.Duplicates == 0 || result.Dropped == 0 || result.Replayed == 0 { - t.Fatalf("simulator did not exercise burst, duplicate, drop, and replay traffic: %#v", result) + if health.QueueDepth == 0 || health.Counters["webhookReceived"] != int64(len(deliveries)+1) || + health.Counters["webhookDuplicate"] != 1 { + t.Fatalf("native ingestion health lost admitted simulator work: %+v", health) } +} - request, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/api/v1/ingestion/health", nil) // #nosec G704 -- httptest binds this endpoint to loopback. +func postNativeSimulatorDelivery(t *testing.T, server *httptest.Server, secret string, delivery simulator.Delivery) (int, map[string]any) { + t.Helper() + payload := delivery.Payload + if delivery.Event == "workflow_run" { + var err error + payload, err = nativeSimulatorRunPayload(payload) + if err != nil { + t.Fatal(err) + } + } + request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+"/api/github/webhook", strings.NewReader(string(payload))) if err != nil { t.Fatal(err) } - request.Header.Set("Authorization", "Bearer "+testAccessToken) + mac := hmac.New(sha256.New, []byte(secret)) + _, _ = mac.Write(payload) + request.Header.Set("X-Hub-Signature-256", "sha256="+hex.EncodeToString(mac.Sum(nil))) + request.Header.Set("X-GitHub-Delivery", delivery.ID) + request.Header.Set("X-GitHub-Event", delivery.Event) response, err := server.Client().Do(request) // #nosec G704 -- the request targets the local httptest server. if err != nil { - t.Fatalf("read Go server ingestion health: %v", err) + t.Fatal(err) } defer func() { if err := response.Body.Close(); err != nil { - t.Errorf("close ingestion health response: %v", err) + t.Errorf("close simulator response: %v", err) } }() - if response.StatusCode != http.StatusOK { - t.Fatalf("ingestion health returned %d", response.StatusCode) - } - var status collect.Status - if err := json.NewDecoder(response.Body).Decode(&status); err != nil { + var result map[string]any + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { t.Fatal(err) } - if status.Health != "recovering" || status.QueueDepth == 0 || status.PendingTasks != 0 { - t.Fatalf("simulator workload did not leave queued recovery work: %#v", status) + return response.StatusCode, result +} + +func nativeSimulatorRunPayload(payload []byte) ([]byte, error) { + // The simulator predates native admission and omits the workflow identity required for run tasks. + var event map[string]json.RawMessage + if err := json.Unmarshal(payload, &event); err != nil { + return nil, err } - if status.Counters["webhookReceived"] != int64(result.Attempts) { - t.Fatalf("health snapshot counted %d webhooks, simulator sent %d", status.Counters["webhookReceived"], result.Attempts) + var run map[string]any + if err := json.Unmarshal(event["workflow_run"], &run); err != nil { + return nil, err + } + run["path"] = ".github/workflows/test.lock.yml" + run["run_attempt"] = 1 + content, err := json.Marshal(run) + if err != nil { + return nil, err + } + event["workflow_run"] = content + return json.Marshal(event) +} + +func TestNativeCollectorRejectsRedisWithoutFencedQueue(t *testing.T) { + address, closeServer := fakeRedis(t) + t.Cleanup(closeServer) + client, err := redisx.New("redis://" + address) + if err != nil { + t.Fatal(err) } - if status.Counters["webhookDuplicate"] == 0 || status.Counters["taskQueued"] == 0 { - t.Fatalf("health snapshot is missing simulator ingestion counters: %#v", status.Counters) + _, err = NewCollector(t.Context(), redisx.NewStore(client, "native-redis-rejected"), + integrationDatabase(t), CollectorConfig{NativeAcquisition: true, AppID: 1, AdmitOnly: true}, "") + if err == nil || !strings.Contains(err.Error(), "fenced task receipts") { + t.Fatalf("native admission accepted a queue without fenced receipts: %v", err) } } diff --git a/server/internal/server/server.go b/server/internal/server/server.go index b7ad244661..f767c0e023 100644 --- a/server/internal/server/server.go +++ b/server/internal/server/server.go @@ -400,9 +400,6 @@ func New(ctx context.Context, store operational.Store, config Config) (*App, err return nil, err } app.adxTraces = worker - if app.Collector().native == nil { - app.Collector().runner.Traces = worker - } } if err := app.configureSessionArchive(ctx); err != nil { return nil, fmt.Errorf("configure session archive: %w", err) diff --git a/server/internal/server/session_archive.go b/server/internal/server/session_archive.go index fd3ff3e304..c1d908ded3 100644 --- a/server/internal/server/session_archive.go +++ b/server/internal/server/session_archive.go @@ -515,18 +515,10 @@ func (a *App) configureSessionArchive(ctx context.Context) error { repositories: slices.Clone(config.Repositories), read: read, } if collector := a.Collector(); collector != nil { - if config.NativeProducer { - if collector.native == nil { - return errors.New("native session archive requires assembled native collection") - } - collector.native.archive = a.sessionArchive - } else { - if collector.runner.Traces == nil { - collector.runner.Traces = a.sessionArchive - } else { - collector.runner.Traces = collect.TraceCaptures{collector.runner.Traces, a.sessionArchive} - } + if collector.native == nil { + return errors.New("native session archive requires assembled native collection") } + collector.native.archive = a.sessionArchive } return nil } diff --git a/specs/server-ingestion.md b/specs/server-ingestion.md index 66a393da52..a7546e3a05 100644 --- a/specs/server-ingestion.md +++ b/specs/server-ingestion.md @@ -21,7 +21,7 @@ server collects Activity evidence directly from a GitHub App, driven by webhooks, instead of consuming a snapshot published by GitHub Actions. It defines profile selection and exclusivity, enrollment, event admission, -collection, the evidence lake, projection, cold start, gap recovery, rate-limit +native acquisition and publication, cold start, gap recovery, rate-limit governance, authority, and failure behavior. Its rate-limit governance applies to outbound GitHub API capacity. Inbound @@ -45,16 +45,12 @@ A **conforming server ingestion implementation** satisfies every MUST in this document. An implementation that does not implement this profile at all remains conforming to CAO; this profile is OPTIONAL. -The default deployment uses PostgreSQL operational storage and preserves this -document's durability and multi-process requirements. Explicit Redis deployments -preserve those same requirements. An explicitly acknowledged single-process memory -deployment MAY use the volatile exception in -[`server-operational-storage.md`](server-operational-storage.md). Only restart -durability and deployment-wide coordination are relaxed: atomic webhook -admission, scope/ownership checks, signatures, OAuth, quota governance, bounded -work, and explicit incomplete recovery remain mandatory. Its startup gates -admission on complete fresh scope enumeration and reuses the same evidence lake, -projector, collector, and backfill, not a second ingestion implementation. +App-backed collection requires `CAO_COLLECT_NATIVE=true`, durable +deployment-scoped PostgreSQL operational ownership, and canonical PostgreSQL. +An App identifier without native selection MUST fail at startup; the former +filesystem collector and volatile memory collector are not fallback profiles. +Without an App identifier the existing Actions snapshot profile remains the +default. ## 2. Profiles @@ -82,11 +78,9 @@ A deployment MUST select exactly one acquisition profile. Both profiles MUST produce identical canonical records for the same observation window and the same underlying GitHub activity. -- The legacy filesystem collector MUST acquire logs with the same - `gh aw logs --audit` invocation and compact them with the same - `activity/cao.mjs` commands the Actions profile uses. A native acquisition - adapter MAY replace those subprocesses under the boundary in section 2.4; - that does not waive source coverage, canonical equivalence or activation gates. +- The native Go acquisition adapter MUST preserve the source coverage, + canonical equivalence and activation gates of the Actions profile without + shelling out to the Activity or gh-aw runtime. - The collector profile MUST reconstruct retained operational-value observations with a native Go implementation conforming to `specs/operational-value-history.md`, using the same campaign adapter contract, @@ -99,8 +93,8 @@ window and the same underlying GitHub activity. Go query engine. The shard format and payload-hash manifest remain shared contracts, not separately invented native formats. - A conforming implementation MUST provide a profile equivalence test that - asserts identical canonical records from an Actions-published directory and a - server-collected evidence lake covering the same window. + asserts identical canonical records from Actions-published evidence and + native exact-attempt acquisition covering the same window. ### 2.3 HTTP host independence @@ -226,10 +220,10 @@ for workers, the exact reviewed inventory digest already provisioned through immutable reviewed bytes directly. Replacing the shared configuration MUST use the explicit expected-digest CAS; replicas MUST NOT choose a newer configuration implicitly. Native activation MUST NOT require an Activity directory, Node or a -local inventory file. Optional existing Node Value execution is separately -selected and retains its explicit catalog/policy/runtime dependencies. Native +local inventory file. Optional Go Value execution is separately +selected and retains its explicit reviewed catalog and policy dependencies. Native selection MUST reject unsupported legacy limits and malformed settings rather -than silently ignore them or fall back to the legacy producer. +than silently ignore them or fall back to the retired filesystem producer. Native Actions job acquisition uses the exact-attempt jobs endpoint and scoped job-log downloads with shared quota observations, credential-free @@ -577,14 +571,11 @@ A collection task acquires evidence for exactly one repository. - A worker MUST lease a task before collecting and MUST NOT collect a repository that another worker holds. -- A worker MUST bound each collection with a timeout and an output size cap. - The legacy filesystem collector MUST own and remove its working directory. - Native acquisition MUST instead use bounded in-memory processing and the - durable handoff in section 2.4, without staging an evidence directory. -- A legacy collection subprocess MUST receive an explicit GitHub API - rate-limit reserve. Native provider requests MUST reserve and reconcile the - existing shared installation quota directly. Both paths MUST fail closed - rather than exhaust the installation. +- A worker MUST bound each acquisition with a timeout and byte caps, use bounded + in-memory processing and the durable handoff in section 2.4, and MUST NOT + stage a filesystem evidence directory. +- Native provider requests MUST reserve and reconcile the existing shared + installation quota directly, failing closed rather than exhausting it. - On failure a worker MUST retry with bounded attempts and jittered backoff, and MUST dead-letter a task that exhausts them. A failed task MUST NOT corrupt or partially replace a repository's existing evidence. @@ -593,45 +584,15 @@ A collection task acquires evidence for exactly one repository. equivalent delayed-work mechanism; repository-lock contention MUST also be retried without consuming the collection attempt budget. -## 6. The evidence lake - -Collected evidence MUST be durably replayable. The legacy filesystem collector -persists it in an **evidence lake** with the directory obligations below. -The native boundary in section 2.4 instead persists safe, exact-attempt -publication intents in PostgreSQL. Those intents are not an Activity directory, -a local overlay, or approved-content trace storage. Native hosted activation -MUST enforce cold start, retention and scope erasure through that shared durable -representation, and reject backends unable to satisfy those contracts. - -- The lake MUST use the Activity snapshot directory layout: compacted run shards - under `gh-aw-logs-runs/`, record shards under `gh-aw-logs-records/`, a - `payload-hashes.json` manifest, and `inventory-sources.json`. A source-built - collector MUST also retain the resolved `control-settings.json` and - source-bound `control-plane-inventory.json` that produced those inventory - sources. -- Shard writes MUST be atomic per repository: a reader MUST observe either the - previous shard or the complete new shard. -- Repository shard prefixes MUST preserve the complete normalized coordinate, - be prefix-free, and occupy a namespace distinct from legacy slash-to-hyphen - filenames. Acquisition, cache selection, and compaction MUST use the same - identity. The collector uses `repo-v2~owner~repository~-logs-` with lowercase - segments; `~` cannot occur in repository segments or legacy coordinates, - making both separators and the terminator unambiguous without expanding - maximum-length GitHub coordinates beyond filesystem basename limits. -- Existing raw shards MAY migrate into that namespace only after validating - all declared repository identities in their content. Migration MUST preserve - unrelated colliding repositories, MUST NOT overwrite an existing destination, - and MUST support retry after interrupted publication. Legacy and versioned - retained evidence MUST remain replayable without GitHub requests. -- Missing, malformed, or conflicting legacy identity MUST fail collection or - erasure closed, preserving the evidence for operator repair rather than - guessing ownership from filenames or enrollment. Compaction MUST reject - conflicting identities across or within records before mutating any group. -- The manifest MUST be regenerated whenever shards change, and MUST list a - SHA-256 hash for every shard it names. -- The lake MUST NOT contain prompts, tokens, secrets, or credentials. -- The lake MUST be sufficient to repopulate an empty canonical database with no - GitHub requests. +## 6. Durable evidence + +Collected safe evidence MUST be durably replayable. Native acquisition persists +exact-attempt publication intents in PostgreSQL as specified in section 2.4, +not in an Activity directory, local overlay, or approved-content trace store. +Native activation MUST enforce cold start, retention and scope erasure through +that shared durable representation, and reject backends unable to satisfy +those contracts. The Actions snapshot layout remains specified separately in +`specs/activity.md`; it is not an App-backed collector fallback. ## 7. Projection @@ -639,16 +600,13 @@ Projection converts retained evidence into the current canonical PostgreSQL dataset. Native publication remains subject to section 2.4's capability and ownership contract. -- Projection MUST preserve the Actions profile's canonical mapping. The legacy - filesystem collector MUST reuse its ingestion implementation over the evidence - lake directory. The native Go adapter MUST satisfy the independent-oracle and - production-query equivalence requirements in sections 2.2 and 2.4; it MUST NOT - introduce alternate canonical semantics. -- Legacy full-dataset projection MUST coalesce collections that complete within - a configured debounce window into one replacement. Native publication MUST - process bounded pending batches under its exact digest and epoch fences. - Snapshot replacement and native publication MUST NOT be interleaved as - compatibility paths. +- Projection MUST preserve the Actions profile's canonical mapping. The native + Go adapter MUST satisfy the independent-oracle and production-query + equivalence requirements in sections 2.2 and 2.4; it MUST NOT introduce + alternate canonical semantics. +- Native publication MUST process bounded pending batches under its exact + digest and epoch fences. Snapshot replacement and native publication MUST + NOT be interleaved as compatibility paths. - A replacement MUST commit all current sources, source documents, diagnostics, revision state, and evaluation time in one PostgreSQL transaction. A failed projection MUST leave the previously committed dataset serving. @@ -656,11 +614,9 @@ ownership contract. rather than commit a dataset known to be inconsistent. - On successful activation the implementation MUST increment the revision and notify connected clients through the existing revision channel. -- Legacy projection MUST short-circuit when the lake's content-addressed data - revision already matches the current dataset. Native publication MUST NOT - reactivate a consumed digest in the same enrollment epoch. An explicit operator - rebuild MAY bypass the unchanged-evidence short-circuit, but MUST NOT bypass - current scope, retention, identity or publication fences. +- Native publication MUST NOT reactivate a consumed digest in the same + enrollment epoch. An explicit operator rebuild MUST NOT bypass current + scope, retention, identity or publication fences. - PostgreSQL MUST be the only dashboard entity and query store. Operational adapters MAY hold projection coordination and health state, but MUST NOT hold entity rows, canonical source documents, query indexes, persistent query projections, or @@ -677,55 +633,47 @@ ownership contract. local-eligibility request, or exact enrollment set. A successful publication atomically replaces canonical `operationalValues`, advances the ordinary canonical revision without changing the Activity data revision, and notifies - existing revision consumers. Accepted JSONL MAY be mirrored into the evidence - lake for cold-start recovery, but that mirror MUST NOT become the hosted query - input. Actions/static reconstruction remains an independent SQLite path. + existing revision consumers. Actions/static reconstruction remains an + independent SQLite path. - Inventory discovery MUST NOT silently truncate the enrolled repository set. An implementation MAY enforce a configured bound, but exceeding it MUST fail the projection rather than publish a partial inventory. -- A source-built collector MUST generate its static control-plane inventory - deterministically from the exact source revision during image build. Runtime - inventory discovery MUST resolve the reviewed deployment policy, reject an - unavailable policy resolution, replace only its repository scope with the - exact operational enrollment set, and enrich the source-bound inventory with a - short-lived installation token for the control repository. +- Native activation MUST select reviewed inventory bytes or an exact + previously provisioned digest. Runtime inventory discovery MUST reject + unavailable policy resolution, enforce the exact operational enrollment + set, and validate control-repository visibility with App credentials. - The first canonical activation MUST fail when it cannot produce valid inventory evidence with at least one campaign. After one valid inventory has been retained, a transient policy, GitHub, or enrichment failure MAY reuse that previous inventory while projecting newer run evidence; an empty or malformed inventory seed MUST NOT qualify as a previous valid inventory. - Reusing previous inventory after a refresh failure MUST retain a pending - refresh request for the next coalesced projection interval. + refresh request for the next reconciliation interval. ## 8. Cold start Cold start MUST be resumable and MUST report progress through the existing administrative rebuild surface. -An implementation MUST support cold start in this order: - -1. **Replay the evidence lake.** When a lake is present, the implementation MUST - repopulate from it without issuing GitHub requests. This is the normal - recovery path. -2. **Enumerate installations** and their repositories into the enrollment set, - checkpointed per page cursor. -3. **Seed backfill tasks**, ordered so that recently active repositories become - queryable first. -4. **Collect per repository** for the retention window, with a per-repository - cursor, so an interruption loses at most one repository's in-flight work. +An implementation MUST support cold start by replaying durable PostgreSQL +publication intents before acquiring missing evidence, reconciling App +installations and their repositories under enrollment ownership, and seeding +receipt-fenced historical discovery and exact-attempt tasks within a frozen +window. Interrupted discovery MUST resume from its durable cursor without +losing accepted attempts or changing its window. During cold start the implementation MUST continue to accept webhooks and MUST apply them without loss or double application. Window completeness and backfilled repository count MUST be published as source metadata. -An empty source-built lake MUST produce and retain valid inventory evidence -before its first canonical projection. A retained lake with valid inventory -evidence remains replayable without GitHub requests. +First activation MUST use reviewed inventory evidence; replicas MAY restore +the exact previously provisioned digest without local inventory files. An explicitly configured historical backfill window MUST be sent as an inclusive UTC `created=START..END` range to GitHub's run-list API; a provider that cannot apply that range MUST fail closed. An unset window preserves -unbounded discovery. Windowed admission MUST NOT truncate retained lake replay. +unbounded discovery. Windowed admission MUST NOT truncate retained publication +replay. Transient contention with a webhook-held enrollment mutation lease MUST wait cancellation-safely rather than abort cold start. Enrollment ownership updates MUST preserve transfer/withdrawal semantics when pipelined in bounded batches. @@ -747,7 +695,7 @@ independent of activity replay. refreshes and missing-installation withdrawals. Failed App enumeration MUST NOT infer withdrawals or advance the successful-admission timestamp. - An operator MUST be able to force the same repository-list repair without - replaying the evidence lake or enumerating historical workflow runs: + replaying publication intents or enumerating historical workflow runs: `cao-dashboard backfill --repositories-only`. The command reports admitted work; collection workers apply the queued reconciliation. Unchanged active repositories MUST NOT be recollected just because their list was refreshed. @@ -769,10 +717,10 @@ GitHub meters App traffic per installation. An implementation MUST govern budget per installation. - Installation tokens MUST be minted per installation, cached no longer than - shortly before expiry, and never logged or persisted to the evidence lake. + shortly before expiry, and never logged or persisted to canonical evidence. - Live inventory enrichment MUST use a short-lived token for the control - repository installation and MUST pass the same explicit rate-limit reserve as - other collector GitHub subprocesses. + repository installation and MUST use the same shared quota reserve as + other native provider requests. - The implementation MUST maintain a per-installation budget corrected from `x-ratelimit-remaining` and `x-ratelimit-reset` on every response. - A worker MUST reserve budget before starting a task and MUST stop at a @@ -792,13 +740,13 @@ per installation. An implementation MUST support admitting deliveries without collection credentials, and MUST refuse a private key, workers, or delivery replay in that mode rather than accept an unusable credential. -- Prompts, tokens, and secrets MUST NOT be written to the evidence lake, the - canonical database, or telemetry. +- Prompts, tokens, and secrets MUST NOT be written to canonical evidence, + operational publication intents, or telemetry. - Missing credentials, missing installation access, an exhausted rate limit, or a - failed subprocess MUST fail closed. An implementation MUST NOT infer broader + failed acquisition MUST fail closed. An implementation MUST NOT infer broader authority or present partial evidence as complete. -- A misconfigured collector MUST fail startup or degrade to the Actions profile. - It MUST NOT degrade to a partial collector. +- A misconfigured collector MUST fail startup. It MUST NOT degrade to the + Actions profile or a partial collector. - Before collection begins, the collector MUST enumerate the complete App installation scope and validate repository visibility: when the control repository is public, it MUST fail startup if the credential's scope @@ -808,8 +756,9 @@ per installation. ## 11b. Retention and erasure -The evidence lake is retained evidence rather than a cache, so retention is a -governed property and not an accident of disk usage. +Native publication intents and accepted canonical evidence are retained +evidence rather than a disposable local cache. Retention and erasure are +governed by source ownership and enrollment epochs in section 2.4. - A signed `repository` delivery with action `created`, `deleted`, `archived`, `unarchived`, `renamed`, or `transferred` and a valid repository ID and @@ -821,8 +770,8 @@ governed property and not an accident of disk usage. archive, rename, and delete deliveries MUST NOT revive stale coordinates or evict a repository that is currently active. - Deleted, archived, transferred-out, and renamed-away coordinates MUST be - evicted from enrollment and the evidence lake, including retained repository - and workflow inventory. Canonical projection MUST remove their evidence; + evicted from enrollment and canonical evidence, including retained repository + and workflow inventory. Canonical publication MUST remove their evidence; lifecycle metadata MUST NOT recreate absent canonical repository rows. Unarchive and rename discovery MUST enroll and collect the current verified coordinate. Repository deletion or archiving changes collection eligibility, @@ -831,23 +780,15 @@ governed property and not an accident of disk usage. `installation_repositories.removed`) still takes precedence and MUST erase evidence immediately under the rules below. - Leaving ingestion scope MUST erase retained evidence. When an installation is - deleted or suspended, or repositories are removed from it, an implementation - MUST delete that repository's shards from the evidence lake and MUST request a - projection so the canonical database stops reporting it. -- Collector-owned shared operational-value observations MUST be filtered by - exact, case-insensitive repository identity, not by shard filename. Withdrawal - MUST preserve unrelated raw observations, including their extension metadata. - Affected disposable normalized shards and normalization-cache files MUST be - invalidated; compaction regenerates their unrelated rows from retained raw - evidence. A snapshot whose manifested payload was invalidated MUST NOT replay - successfully until recompaction. -- Shared-file erasure and collector projection MUST serialize across processes - sharing the evidence lake. The Go collector uses an advisory filesystem lock; - the configured lake filesystem MUST support that lock across its writers. - Before best-effort operational-value acquisition, projection MUST filter - collector-owned observations against current enrollment, including an empty - enrollment set. Credential or adapter failures MUST NOT bypass that filtering - or restore withdrawn observations through the retained-value merge. + deleted or suspended, or repositories are removed from it, the enrollment + ownership transaction MUST invalidate their pending publications, claims and + canonical graph under the same source epoch. It MUST preserve unrelated + repositories; a later delivery for the displaced installation MUST NOT erase + a repository already transferred to another installation. +- Collector-owned operational-value observations MUST be filtered by exact + repository identity and current enrollment before publication. Credential or + adapter failures MUST NOT bypass that filtering or restore withdrawn + observations through retained-value reconciliation. - Successful canonical replacement advances the revision. Queries depending on collector operational values MUST include that revision in their result-cache identity so an older cached result cannot serve after replacement. Unreachable @@ -859,31 +800,11 @@ governed property and not an accident of disk usage. facts. Their repository scope, disposal authority, retention and any hold obligations require an explicit operator decision; this rule does not invent a hold or authorize indefinite retention. -- Repository-prefixed shard erasure MUST preflight candidate ownership across - raw, run, and record directories before deleting any file. A legacy filename - match alone MUST NOT authorize deletion: unambiguously unrelated evidence - MUST remain untouched, and ambiguous evidence MUST block successful erasure. -- Erasure MUST be driven by the same webhook admission path as enrollment, so - withdrawing consent takes effect without operator action. A process without an - evidence lake MUST queue erasure rather than skip it. -- Removal MUST be scoped to the installation that currently covers a repository. - A repository transferred between installations MUST keep its enrollment and - its evidence when a later removal or deletion event arrives for the - installation that no longer covers it; that event MUST clear only its own - installation membership. -- The implementation MUST durably enqueue erasure work before removing - enrollment, so a queue failure cannot make a withdrawal unrecoverable on - delivery replay. Erasure work MUST be serialized with collection by the - repository lease and MUST wait until the repository is no longer covered by - the installation that requested removal. A worker MUST recheck current - enrollment after acquiring the lease and MUST erase, rather than collect, an - already-queued task for a repository that has left scope. Enrollment additions - and removals MUST be serialized so the membership snapshot used to queue - erasure is the same snapshot that is removed. -- A synchronous erasure failure MUST fail the delivery. A durably queued - erasure MAY be acknowledged before completion, but MUST remain visible and - recoverable until deletion and projection succeed; it MUST NOT be treated as - completed merely because it was queued. +- Erasure MUST be driven by the same authenticated enrollment reconciliation as + scope admission, without waiting for a filesystem worker or operator action. + Stale tasks MUST fail their source-epoch and ownership checks rather than + publish withdrawn evidence. Failure to commit an atomic withdrawal MUST fail + the reconciliation; an accepted delivery alone is not proof of erasure. - Collection MUST be at-least-once. Retry, deferral, blocked-repository requeue, malformed-entry handling, and dead-letter transitions MUST persist the replacement before acknowledging the original in one atomic storage operation. @@ -920,7 +841,7 @@ count. profile, operational connectivity and guarantees, PostgreSQL readiness and revision state, canonical integrity, query-definition validity, and, when collection is configured, enrollment, queue, cold-start, - rate-limit, evidence-lake, and tooling state. + rate-limit, durable publication, and tooling state. - Each check MUST have a stable machine-readable identifier, severity, human-readable summary, observed facts, and a remedy for non-passing states. The same observations MUST be available in a versioned structured format. @@ -1032,10 +953,10 @@ work, dead letters, processing lag, and the collection-health status surface during and after each run. The Go server integration test MUST deliver signed scenario traffic through the -production webhook handler with a real Redis instance, then verify the health -API reports persisted webhook counters and queued recovery work. This test MUST -run in the Redis-backed server integration job; unit tests without Redis MAY -skip it. +production webhook handler with durable deployment-scoped PostgreSQL +operational storage, then verify the health API reports persisted webhook +counters and queued recovery work. A Redis-only admission test cannot establish +native publication or source-epoch ownership. ## 14. Conformance checklist @@ -1045,11 +966,11 @@ A conforming implementation: 2. leaves the Actions profile's behavior unchanged when the collector is absent; 3. derives scope from App installations, never from `cao.json`; 4. admits and enqueues webhooks without projecting in the request path; -5. collects with the shared `gh aw logs --audit` and `activity/cao.mjs` - implementations; -6. persists a snapshot-compatible evidence lake sufficient for cold start; -7. projects by reusing the Actions profile's ingestion implementation, coalesced - and atomically activated; +5. acquires bounded, exact-attempt GitHub evidence with the native Go adapter, + without spawning Activity or gh-aw processes; +6. persists receipt-fenced safe publication intents sufficient for recovery; +7. activates canonical evidence atomically under inventory and enrollment + ownership fences; 8. recovers activity gaps by webhook delivery replay and repairs repository lists through periodic and operator-triggered authenticated discovery; 9. governs GitHub budget per installation and fails closed; @@ -1058,5 +979,5 @@ A conforming implementation: bounds its queues; 12. admits deliveries without collection credentials, and fails closed when an admission-only process is asked to collect or project; -13. short-circuits projection for an unchanged lake, transactionally replaces - the current PostgreSQL dataset, and refuses to publish a truncated inventory. +13. rejects duplicate activation in the same epoch, transactionally updates + canonical PostgreSQL, and refuses to publish a truncated inventory. diff --git a/specs/value.md b/specs/value.md index c721a757c3..a929f9c27b 100644 --- a/specs/value.md +++ b/specs/value.md @@ -200,12 +200,26 @@ These private aggregates MAY be exposed to authorized operators for capacity calibration; they MUST NOT expose credentials, request bodies, response bodies, proof content, or arbitrary stderr. -The native hosted acquisition profile MAY separately select this existing Value -execution lane with `CollectorConfig.NativeValues`. That selection retains the -reviewed Node collector programs and their catalog and policy assets; it does -not make them dependencies of native run acquisition, canonical projection or -trace admission. An unselected Value execution lane MUST NOT inspect those -assets or start a Node process. +The native hosted acquisition profile MAY separately select Value execution +with `CollectorConfig.NativeValues`. That lane MUST resolve the reviewed +catalog and policy and execute supported acquisition and local computation in +Go; it MUST reject unsupported collector registrations rather than invoke a +Node program. Git subprocesses MAY be used only for reviewed Git transport +requirements. An unselected Value execution lane MUST NOT inspect its catalog +or policy assets. + +The Go hosted local evaluator supports the reviewed Optimization canonical-query +measurement and consumes Dependabot lifecycle evidence only after the separately +registered, quota-admitted Go remote collector publishes three complete +cutoff-aligned metric packets atomically. The Dependabot collector is restricted +to `github/gh-aw`, at most 100 alert pages and ten pull-request search pages, +with a reviewed ceiling of 111 requests per attempt, including one optional +Search-quota preflight. It MUST reject incomplete +pagination, inaccessible evidence and malformed lifecycles, preserving all +previously accepted packets and proof. A selected Dependabot measurement +without three complete packets for its cutoff MUST publish explicit +missing/unavailable measurements, not omit them or infer zero. The separate +Actions snapshot pipeline retains its own reviewed collector. Selected native remote invocations MUST restore the starting packet and proof from one PostgreSQL snapshot pinned to their original installation and enrollment diff --git a/tests/unit/cao-operational-value-file-diet-replay.test.mjs b/tests/unit/cao-operational-value-file-diet-replay.test.mjs index 9fd8c32292..04b6637bdb 100644 --- a/tests/unit/cao-operational-value-file-diet-replay.test.mjs +++ b/tests/unit/cao-operational-value-file-diet-replay.test.mjs @@ -95,6 +95,9 @@ test("the real reviewed File Diet collector loads in the isolated worker and reu const ledger = JSON.parse(readFileSync(path.join(root, "tests/fixtures/operational-value/file-diet-events.json"), "utf8")); const output = path.join(directory, "evidence.jsonl"); const policy = JSON.parse(readFileSync(path.join(root, ".github/workflows/cao.json"), "utf8")); + policy["control-plane"].campaigns.activity["workflow-values"] = + policy["control-plane"].campaigns.activity["workflow-values"] + .filter((entry) => entry.repository !== definition.repository || entry.workflow === definition.slug); const registration = policy["control-plane"].campaigns.activity["workflow-values"] .find((entry) => entry.repository === definition.repository && entry.workflow === definition.slug); delete registration.evidence; diff --git a/tests/unit/cao-package-contract.test.mjs b/tests/unit/cao-package-contract.test.mjs index bf5a860b9e..f6cf8a8277 100644 --- a/tests/unit/cao-package-contract.test.mjs +++ b/tests/unit/cao-package-contract.test.mjs @@ -139,7 +139,7 @@ test("CAO server image supports downstream Compose role selection", async () => assert.ok(dashboardRuntime); assert.ok(collectorRuntime); assert.match(dashboardRuntime, /RUN apk upgrade --no-cache \\\n && apk add --no-cache ca-certificates/); - assert.match(collectorRuntime, /RUN apk upgrade --no-cache \\\n && apk add --no-cache ca-certificates github-cli/); + assert.match(collectorRuntime, /RUN apk upgrade --no-cache \\\n && apk add --no-cache ca-certificates git/); assert.match(dockerfile, /ENTRYPOINT \["\/app\/cao-dashboard"\]/); assert.match(dockerfile, /CMD \["serve-hosted"/); assert.match(dockerfile, /VOLUME \["\/app\/source"\]/); @@ -197,6 +197,7 @@ test("Docker dashboard builder installs production tooling without shipping buil assert.match(runtime, /\/workspace\/dashboard\/site\/dist\/ \/app\/site\//); } const collector = stages.find((stage) => stage.includes(" AS collector-runtime")); - assert.match(collector, /control-plane-inventory\.json \/app\/catalog\/control-plane-inventory\.json/); + assert.match(collector, /\/workspace\/\.github\/workflows\/cao\.json \/app\/\.github\/workflows\/cao\.json/); + assert.doesNotMatch(collector, /node_modules|control-plane-inventory\.json|\/app\/catalog\//); assert.doesNotMatch(collector, /COPY .*\/workspace\/dashboard\/site\/node_modules\/ \/app\//); }); diff --git a/tests/unit/dashboard-container-contract.test.mjs b/tests/unit/dashboard-container-contract.test.mjs index 9ac361cf11..2c488bf65c 100644 --- a/tests/unit/dashboard-container-contract.test.mjs +++ b/tests/unit/dashboard-container-contract.test.mjs @@ -32,14 +32,10 @@ test("server and native CLI builds require the CA-compromise-patched Go toolchai test("dashboard image is multi-stage, non-root, versioned, and health checked", async () => { const dockerfile = await text("server/Dockerfile"); - const dockerignore = await text(".dockerignore"); - const policy = JSON.parse(await text(".github/workflows/cao.json")); - const ghAwVersion = policy["gh-aw-version"]; assert.match(dockerfile, /FROM node:24-alpine@sha256:[0-9a-f]{64} AS dashboard-build/); assert.match(dockerfile, /FROM golang:1\.27\.3-alpine@sha256:[0-9a-f]{64} AS server-build/); assert.match(dockerfile, /FROM alpine:3\.22@sha256:[0-9a-f]{64} AS dashboard-runtime/); - assert.match(dockerfile, /FROM node:24-alpine@sha256:[0-9a-f]{64} AS collector-runtime/); assert.match(dockerfile, /ARG CAO_PROFILE=cao\.json/); assert.match(dockerfile, /ARG VERSION=dev/); assert.match(dockerfile, /ARG REVISION=unknown/); @@ -53,34 +49,24 @@ test("dashboard image is multi-stage, non-root, versioned, and health checked", assert.match(dockerfile, /CMD \["sh", "-c"/); assert.doesNotMatch(dockerfile, /CMD \["CMD-SHELL"/); assert.match(dockerfile, /ENTRYPOINT \["\/app\/cao-dashboard"\]/); - assert.match(dockerfile, new RegExp(`ARG GH_AW_VERSION=${ghAwVersion.replaceAll(".", "\\.")}`)); - assert.match(dockerfile, /github\/gh-aw\/releases\/download\/\$\{GH_AW_VERSION\}\/linux-\$\{TARGETARCH\}/); - assert.match(dockerfile, /sha256sum -c -/); - assert.match(dockerfile, /\/app\/\.local\/share\/gh\/extensions\/gh-aw\/gh-aw/); - assert.match(dockerfile, /\/workspace\/activity\/ \/app\/catalog\/activity\//); - assert.match(dockerfile, /REPORT_INVENTORY_STATIC=true/); - assert.match( - dockerfile, - /\/tmp\/control-plane-inventory\.json \/app\/catalog\/control-plane-inventory\.json/, - ); - assert.match( - dockerfile, - /\/workspace\/\.github\/workflows\/shared\/control\.mjs \/app\/catalog\/\.github\/workflows\/shared\/control\.mjs/, - ); - assert.match( - dockerfile, - /\/workspace\/\.github\/workflows\/shared\/policy\.mjs \/app\/catalog\/\.github\/workflows\/shared\/policy\.mjs/, - ); - assert.match( - dockerfile, - /\/workspace\/\.github\/workflows\/\*\.lock\.yml \/app\/catalog\/\.github\/workflows\//, - ); - assert.match(dockerignore, /^!\.github\/workflows\/\*\.lock\.yml$/m); - assert.match(dockerfile, /\/workspace\/dashboard\/site\/node_modules\/yaml\/ \/app\/catalog\/node_modules\/yaml\//); - assert.match(dockerfile, /VOLUME \["\/app\/evidence"\]/); assert.match(dockerfile, /FROM dashboard-runtime AS final/); }); +test("collector image contains the Go worker, reviewed policy and Git without a Node runtime or lake", async () => { + const dockerfile = await text("server/Dockerfile"); + const collector = dockerfile.split(/(?=^FROM )/m).find((stage) => stage.includes(" AS collector-runtime")); + assert.ok(collector); + assert.match(collector, /^FROM alpine:3\.22@sha256:[0-9a-f]{64} AS collector-runtime$/m); + assert.match(collector, /apk add --no-cache ca-certificates git/); + assert.match(collector, /COPY --from=server-build --chown=cao:cao \/out\/cao-dashboard \/app\/cao-dashboard/); + assert.match(collector, /\/workspace\/\.github\/workflows\/cao\.json \/app\/\.github\/workflows\/cao\.json/); + assert.match(collector, /\/workspace\/\.github\/value-evidence\/daily-file-diet\.jsonl\.gz \/app\/\.github\/value-evidence\/daily-file-diet\.jsonl\.gz/); + assert.match(collector, /\/workspace\/dashboard\/site\/src\/data\/queries\/database\.json \/app\/queries\/database\.json/); + assert.match(collector, /USER 65532:65532/); + assert.match(collector, /CMD \["collect", "--database-queries", "\/app\/queries\/database\.json"\]/); + assert.doesNotMatch(collector, /\bnode\b|github-cli|gh-aw|\/app\/evidence|VOLUME|HEALTHCHECK|\/app\/site/); +}); + test("Hosted CLI relay exports a separate source-bound static artifact", async () => { const dockerfile = await text("server/Dockerfile"); const build = dockerfile.split("FROM server-build AS hosted-cli-relay-build\n")[1] diff --git a/tests/unit/server-azure-bicep.test.mjs b/tests/unit/server-azure-bicep.test.mjs index 2717378a73..7664ace5d7 100644 --- a/tests/unit/server-azure-bicep.test.mjs +++ b/tests/unit/server-azure-bicep.test.mjs @@ -234,7 +234,8 @@ test('Azure selects PostgreSQL operational storage without Redis', () => { assert.equal((collector.match(/registries:\s*registries/g) ?? []).length, 2); assert.equal((collector.match(/dependsOn:\s*\[roles\]/g) ?? []).length, 2); assert.equal((collector.match(/'--database-queries', '\/app\/queries\/database.json'/g) ?? []).length, 2); - assert.match(collector, /name:\s*'CAO_COLLECT_CATALOG_ROOT'\s+value:\s*'\/app\/catalog'/); + assert.match(collector, /name:\s*'CAO_COLLECT_NATIVE'\s+value:\s*'true'/); + assert.match(collector, /name:\s*'CAO_COLLECT_NATIVE_INVENTORY_DIGEST'\s+value:\s*nativeInventoryDigest/); assert.match(collector, /name:\s*'CAO_POSTGRES_RUN_RETENTION_DAYS'/); }); @@ -272,6 +273,62 @@ test('Azure collection uses fixed PostgreSQL-backed workers', () => { assert.doesNotMatch(collector, /taskStreamKey|streamKey|consumerGroup/); }); +test('Azure Bicep and compiled collector select native Go acquisition and require provisioned inventory', () => { + assert.match(bicep, /param collectorNativeInventoryDigest string = ''/); + assert.match(bicep, /nativeInventoryDigest:\s*collectorNativeInventoryDigest/); + assert.match(collector, /@minLength\(64\)\s+@maxLength\(64\)[\s\S]*?param nativeInventoryDigest string/); + assert.doesNotMatch(bicep, /collectorLakeStorageSku/); + assert.doesNotMatch(collector, /lakeStorageSku|lakeAccount|evidence-lake|CAO_COLLECT_LAKE_DIRECTORY/); + assert.match(bicep, /name:\s*'CAO_COLLECT_NATIVE'\s+value:\s*'true'/); + assert.equal(generated.variables.collectionEnabled, "[not(empty(parameters('collectorImage')))]"); + assert.equal(generated.parameters.collectorNativeInventoryDigest.defaultValue, ''); + const app = resources.find((resource) => resource.type === 'Microsoft.Web/sites'); + const admission = app.properties.siteConfig.appSettings; + assert.match(admission, /if\(not\(variables\('collectionEnabled'\)\), createArray\(\), createArray\(/); + assert.match(admission, /'CAO_COLLECT_APP_ID', 'value', parameters\('collectorGithubAppId'\).*?'CAO_COLLECT_NATIVE', 'value', 'true'.*?'CAO_COLLECT_ADMIT_ONLY', 'value', 'true'/); + assert.doesNotMatch(admission, /CAO_COLLECT_NATIVE_INVENTORY_DIGEST|CAO_COLLECT_PRIVATE_KEY/); + + const workers = resources.find((resource) => resource.name === 'collection-workers'); + assert.equal(workers.condition, "[variables('collectionEnabled')]"); + assert.equal(workers.properties.parameters.nativeInventoryDigest.value, + "[parameters('collectorNativeInventoryDigest')]"); + const template = workers.properties.template; + assert.equal(template.parameters.nativeInventoryDigest.minLength, 64); + assert.equal(template.parameters.nativeInventoryDigest.maxLength, 64); + const environment = Object.fromEntries(template.variables.collectionBaseEnvironment.map(({ name, ...setting }) => [name, setting])); + assert.deepEqual(environment.CAO_COLLECT_NATIVE, { value: 'true' }); + assert.deepEqual(environment.CAO_COLLECT_NATIVE_INVENTORY_DIGEST, { + value: "[parameters('nativeInventoryDigest')]", + }); + assert.deepEqual(environment.CAO_COLLECT_APP_ID, { value: "[parameters('githubAppId')]" }); + assert.deepEqual(environment.CAO_POSTGRES_URL, { secretRef: 'cao-postgres-url' }); + assert.deepEqual(environment.CAO_OPERATIONAL_NAMESPACE, { value: "[parameters('operationalNamespace')]" }); + assert.equal(workers.properties.parameters.operationalNamespace.value, 'azure-dashboard-operational'); + assert.equal(template.resources.filter((resource) => resource.type === 'Microsoft.Storage/storageAccounts').length, 0); + for (const role of template.resources.filter((resource) => + ['Microsoft.App/jobs', 'Microsoft.App/containerApps'].includes(resource.type))) { + assert.equal(role.properties.template.containers[0].env, "[variables('collectionEnvironment')]"); + assert.equal(role.properties.template.containers[0].command[0], '/app/cao-dashboard'); + assert.ok(['collect', 'backfill'].includes(role.properties.template.containers[0].args[0])); + assert.equal(role.properties.template.volumes, undefined); + assert.equal(role.properties.template.containers[0].volumeMounts, undefined); + } + for (const retired of ['CAO_COLLECT_LAKE_DIRECTORY', 'CAO_COLLECT_STATIC_INVENTORY', 'CAO_COLLECT_GH_BINARY', + 'CAO_COLLECT_CATALOG_ROOT', 'collectorLakeStorageSku', 'lakeStorageAccountName', 'evidence-lake']) { + assert.doesNotMatch(JSON.stringify(generated), new RegExp(retired)); + } + const runtime = dockerfile.split(' AS collector-runtime\n')[1]?.split('\nFROM ')[0]; + assert.match(dockerfile, /^FROM alpine:3\.22@sha256:[0-9a-f]{64} AS collector-runtime$/m); + assert.match(runtime, /^COPY --from=server-build --chown=cao:cao \/out\/cao-dashboard \/app\/cao-dashboard$/m); + assert.match(runtime, /^COPY --from=dashboard-build --chown=cao:cao \/workspace\/dashboard\/site\/src\/data\/queries\/database\.json \/app\/queries\/database\.json$/m); + for (const policy of ['cao.json', 'cao.azure.json']) { + assert.ok(runtime.includes(`COPY --from=dashboard-build --chown=cao:cao /workspace/.github/workflows/${policy} /app/.github/workflows/${policy}`)); + } + assert.doesNotMatch(runtime, /node:|gh-aw|\/app\/catalog|\/app\/evidence/); + assert.match(runtime, /^ENTRYPOINT \["\/app\/cao-dashboard"\]$/m); + assert.match(runtime, /^CMD \["collect", "--database-queries", "\/app\/queries\/database\.json"\]$/m); +}); + test('Azure dashboard Bicep does not output secrets or PAT-based configuration', () => { const outputLines = bicep.split('\n').filter((line) => line.trim().startsWith('output ')); assert.ok(outputLines.length > 0, 'expected deterministic non-secret outputs'); diff --git a/tests/unit/workflow-contract-activity-dispatch.test.mjs b/tests/unit/workflow-contract-activity-dispatch.test.mjs index b384d64518..17074e65d1 100644 --- a/tests/unit/workflow-contract-activity-dispatch.test.mjs +++ b/tests/unit/workflow-contract-activity-dispatch.test.mjs @@ -60,18 +60,26 @@ test('fast collection preserves Value history and publishes a deferred decision }); const result = JSON.parse(output); assert.equal(result.imported, 1); - assert.equal(result.deferred, 1); + assert.equal(result.deferred, 2); assert.equal(readFileSync(path.join(shards, 'operational-values-example.jsonl'), 'utf8'), 'retained evidence\n'); - const evidence = JSON.parse(readFileSync(path.join(shards, 'workflow-values-example.jsonl'), 'utf8')); - assert.equal(evidence.operational_value_evidence.observedAt, '2026-10-18T12:05:52.580Z'); - assert.equal(evidence.operational_value_evidence.windowEnd, '2026-10-18T11:54:09Z'); - const health = JSON.parse(readFileSync(path.join(shards, 'workflow-values-example-collections.jsonl'), 'utf8')); - assert.equal(health.workflow_value_collection.repository, 'github/gh-aw'); - assert.equal(health.workflow_value_collection.workflow, 'daily-file-diet'); - assert.equal(health.workflow_value_collection.registration, 'enabled'); - assert.equal(health.workflow_value_collection.state, 'deferred'); - assert.equal(health.workflow_value_collection.reasonCode, 'fast-mode'); - assert.equal(health.workflow_value_collection.lastAttemptAt, undefined); + const evidence = readFileSync(path.join(shards, 'workflow-values-example.jsonl'), 'utf8') + .trim().split('\n').map(line => JSON.parse(line)); + assert.equal(evidence.length, 1); + const dailyFileDiet = evidence.find(envelope => + envelope.operational_value_evidence.workflow === 'daily-file-diet').operational_value_evidence; + assert.equal(dailyFileDiet.observedAt, '2026-10-18T12:05:52.580Z'); + assert.equal(dailyFileDiet.windowEnd, '2026-10-18T11:54:09Z'); + const collections = readFileSync(path.join(shards, 'workflow-values-example-collections.jsonl'), 'utf8') + .trim().split('\n').map(line => JSON.parse(line).workflow_value_collection); + assert.equal(collections.length, 2); + assert.ok(collections.every(collection => collection.state === 'deferred' && collection.reasonCode === 'fast-mode')); + const health = collections.find(collection => collection.workflow === 'daily-file-diet'); + assert.equal(health.repository, 'github/gh-aw'); + assert.equal(health.registration, 'enabled'); + assert.equal(health.state, 'deferred'); + assert.equal(health.reasonCode, 'fast-mode'); + assert.equal(health.lastAttemptAt, undefined); + assert.equal(collections.find(collection => collection.workflow === 'dependabot-update-planner').reasonCode, 'fast-mode'); assert.equal(JSON.parse(readFileSync(path.join(shards, 'workflow-value-local.json'), 'utf8'))[0].eligible, false); }); @@ -92,9 +100,9 @@ test('ordinary Activity leaves the reviewed Value grant above its protected API t.after(() => rmSync(directory, { recursive: true, force: true })); const output = path.join(directory, 'capacity'); for (const [repositories, base, expected] of [ - [['github/gh-aw'], '2000', 'log-floor=-2700\nvalue-reserve=2000\n'], + [['github/gh-aw'], '2000', 'log-floor=-2811\nvalue-reserve=2000\n'], [['githubnext/gh-cao'], '2000', 'log-floor=-2000\nvalue-reserve=2000\n'], - [['github/gh-aw'], '100', 'log-floor=-800\nvalue-reserve=100\n'], + [['github/gh-aw'], '100', 'log-floor=-911\nvalue-reserve=100\n'], [['githubnext/gh-cao'], '100', 'log-floor=-100\nvalue-reserve=100\n'], ]) { writeFileSync(output, ''); diff --git a/tests/unit/workflow-values-recurring.test.mjs b/tests/unit/workflow-values-recurring.test.mjs index dd88946e83..9a790d9255 100644 --- a/tests/unit/workflow-values-recurring.test.mjs +++ b/tests/unit/workflow-values-recurring.test.mjs @@ -164,7 +164,8 @@ test('dogfood File Diet registration pins existing effective Activity scope with [repository, `GH_AW_GITHUB_READ_PAT_${repository.split('/')[0].toUpperCase()}`]))) }; assert.deepEqual(activityCollectionPlan(controlSettings(document, auth.controlRepository), auth), activityCollectionPlan(controlSettings(before, auth.controlRepository), auth)); - const registrations = workflowValueRegistrations(document, ['github/gh-aw']); + const registrations = workflowValueRegistrations(document, ['github/gh-aw']) + .filter(registration => registration.workflow === 'daily-file-diet'); assert.equal(registrations.length, 1); assert.deepEqual({ ...registrations[0], acquisition: undefined }, { 'api-requests': 700, @@ -190,11 +191,11 @@ test('dogfood File Diet registration pins existing effective Activity scope with 'github.workflow-source-history.v1', ]); assert.equal(control.campaigns.dashboard.deploy, false); - assert.equal(workflowValueApiHeadroom(document, ['github/gh-aw']), 700); + assert.equal(workflowValueApiHeadroom(document, ['github/gh-aw']), 811); assert.equal(workflowValueApiHeadroom(document, ['githubnext/gh-cao']), 0); const disabled = structuredClone(document); disabled['control-plane'].campaigns.activity['workflow-values'][0].enabled = false; - assert.equal(workflowValueApiHeadroom(disabled, ['github/gh-aw']), 0); + assert.equal(workflowValueApiHeadroom(disabled, ['github/gh-aw']), 111); }); test('reviewed remote collection uses reserved capacity after ordinary Activity reaches its floor', async t => {