From 5e1ea9feef4c58e9afe6c2b561088c2c37960807 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:42:29 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m0/ci/nix-cache-5xx.md Quest-Claim: 88c800847418ed62 From 5218a58a53104abf4cc82ca5cf6215ec95f0086a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:46:42 -0800 Subject: [PATCH 2/4] fix(nix): treat R2 service failures as cache misses Co-Authored-By: GPT-6.1 Sol --- infra/README.md | 6 +++ infra/nix/src/worker.test.ts | 80 ++++++++++++++++++++++++++++++++++++ infra/nix/src/worker.ts | 13 +++++- quest/m0/ci/nix-cache-5xx.md | 26 ++++++++---- 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 infra/nix/src/worker.test.ts diff --git a/infra/README.md b/infra/README.md index b7859fca626..052d37774af 100644 --- a/infra/README.md +++ b/infra/README.md @@ -318,6 +318,12 @@ after upload; the next warm re-uploads what it still needs. `flake.nix` sets `fallback = true` because that expiry can remove a path before one that depends on it. +The Worker logs R2 internal errors (10001) and service-unavailable errors +(10043), then returns a non-cacheable 404 so Nix can build from source. +Other errors, including invalid object names, permissions, and bucket +configuration, still fail the request. A failure after the archive body +starts streaming remains a failed download, covered by `fallback`. + Nix only trusts paths signed by the key in `flake.nix`, so the signing key is the trust boundary, not the bucket keys. It lives in the `cache-write` Environment beside one R2 token for both buckets. Only `main` and the five diff --git a/infra/nix/src/worker.test.ts b/infra/nix/src/worker.test.ts new file mode 100644 index 00000000000..3799196df92 --- /dev/null +++ b/infra/nix/src/worker.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, mock, spyOn, test } from "bun:test"; + +import worker from "./worker"; + +const key = "00000000000000000000000000000000.narinfo"; +const request = (path = key, method = "GET") => new Request(`https://nix.moq.dev/${path}`, { method }); +const bucket = (get = mock(async (_key: string): Promise => null)) => + ({ get }) as unknown as R2Bucket; + +describe("bucket reads", () => { + for (const binding of ["RELEASE", "CI"] as const) { + for (const code of [10001, 10043]) { + test(`${binding} service error ${code} makes GET and HEAD a non-cacheable miss`, async () => { + const error = new Error(`get: Service failed (${code})`); + const failed = mock(async () => { + throw error; + }); + const env = { RELEASE: bucket(), CI: bucket(), [binding]: bucket(failed) }; + const log = spyOn(console, "error").mockImplementation(() => {}); + try { + for (const method of ["GET", "HEAD"]) { + for (const path of [key, "nar/archive.nar.zst"]) { + const response = await worker.fetch(request(path, method), env); + expect(response.status).toBe(404); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + } + } + expect(log).toHaveBeenCalledTimes(4); + expect(failed).toHaveBeenCalledTimes(4); + if (binding === "RELEASE") expect(env.CI.get).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }); + } + + test.each([ + new Error("get: Unauthorized (10002)"), + new Error("get: Access denied (10003)"), + new Error("get: No such bucket (10006)"), + new Error("get: Invalid object name (10020)"), + new Error("get: Unknown failure (0)"), + new Error("unrelated 10001"), + new TypeError("bad binding"), + ])(`${binding} preserves %s`, async (error) => { + const env = { + RELEASE: bucket(), + CI: bucket(), + [binding]: bucket(mock(async () => Promise.reject(error))), + }; + await expect(worker.fetch(request(), env)).rejects.toBe(error); + }); + } + + test("a missing binding is a configuration error", async () => { + await expect(worker.fetch(request(), { RELEASE: undefined, CI: bucket() } as never)).rejects.toThrow( + TypeError, + ); + }); + + test("a release miss checks CI and preserves the object's headers and body", async () => { + const body = "signed metadata"; + const release = mock(async () => null); + const ci = mock(async () => ({ size: body.length, body: new Response(body).body }) as R2ObjectBody); + const response = await worker.fetch(request(), { RELEASE: bucket(release), CI: bucket(ci) }); + expect(await response.text()).toBe(body); + expect(response.headers.get("Content-Type")).toBe("text/x-nix-narinfo"); + expect(response.headers.get("Content-Length")).toBe(String(body.length)); + expect(response.headers.get("Cache-Control")).toBe("public, max-age=300"); + expect(release).toHaveBeenCalledWith(key); + expect(ci).toHaveBeenCalledWith(key); + }); + + test("cache metadata needs no bucket read", async () => { + const env = { RELEASE: bucket(), CI: bucket() }; + expect((await worker.fetch(request("nix-cache-info"), env)).status).toBe(200); + expect(env.RELEASE.get).not.toHaveBeenCalled(); + expect(env.CI.get).not.toHaveBeenCalled(); + }); +}); diff --git a/infra/nix/src/worker.ts b/infra/nix/src/worker.ts index 088955e80ca..95334941ba4 100644 --- a/infra/nix/src/worker.ts +++ b/infra/nix/src/worker.ts @@ -57,7 +57,18 @@ export default { // Releases first, since they never expire. A NAR is named by its hash, so // whichever bucket holds it serves the same bytes. for (const bucket of [env.RELEASE, env.CI]) { - const object = await bucket.get(key); + let object: R2ObjectBody | null; + try { + object = await bucket.get(key); + } catch (error) { + // R2 exposes its error code in the message, not as a typed field: + // https://developers.cloudflare.com/r2/api/error-codes/ + if (!(error instanceof Error && /^get: .* \((10001|10043)\)$/.test(error.message))) throw error; + // A cache miss lets Nix build from source without a failed lookup. + // Keep intermediaries from caching a transient failure as missing. + console.error(`Nix cache read unavailable: ${key}`, error); + return new Response("Not Found", { status: 404, headers: { "Cache-Control": "no-store" } }); + } if (!object) continue; const headers = { diff --git a/quest/m0/ci/nix-cache-5xx.md b/quest/m0/ci/nix-cache-5xx.md index 52c5841c9f1..f484877ce31 100644 --- a/quest/m0/ci/nix-cache-5xx.md +++ b/quest/m0/ci/nix-cache-5xx.md @@ -11,13 +11,25 @@ substitute. Decided 2026-10-31 (user, /quest-plan of the follow-ups from moq-dev/quest's host-gate cutover for moq.pro and rust2 and its fixer repairs). - On 2026-10-31 one HTTP 500 for a `.narinfo` failed `check flake` and - ejected a seven-PR batch in the host land gate. Nix stops on a failed - path-info lookup even with `fallback = true`, which covers only failed - downloads. -- Prefer fixing it at the source: find what made the cache Worker return 500 - for a lookup and have it answer 404 (a miss) when its bucket read fails. - If Nix still sees 5xx from the edge, retry the lookup in the Worker or - through Nix's `download-attempts`. + ejected a seven-PR batch in the host land gate. +- [The Worker fix](https://github.com/moq-dev/moq/pull/7537) turns R2 internal + errors (10001) and service-unavailable errors (10043) into logged, + non-cacheable 404 misses. Malformed input, bad bindings, authorization + errors, and unknown failures still fail. No retries or client settings + change. Both buckets and metadata/archive reads have local test coverage. +- A local HTTP fault-injection probe verified that the real dev shell + evaluates and builds a fresh dependency from source with these 404s. + Nix 2.35.2 also completed that probe against the original 500 response, + while logging the error and disabling the cache. This does not reproduce + the historical `check flake` failure. The available Cloudflare OAuth token + received `Forbidden` when querying historical Worker logs, so the original + incident's R2 code or edge origin remains unconfirmed. +- Remaining: a maintainer deploys the Worker with `just infra nix deploy`, + then verifies cache misses and dev-shell evaluation/build through + `nix.moq.dev`. Deployment was not authorized for the implementation task. + Inspect the original incident's logs with Workers Observability access; + distinguish R2 service errors from configuration errors or edge failures + before broadening the handling. - The [Nix cache verification](/quest/m0/ci/nix-cache-verify.md) owns the release side of the cache. From 80062370cb1936f66bb62a64bea738b7dc6f2007 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:34:04 -0800 Subject: [PATCH 3/4] chore(quest): claim quest/m0/ci/nix-cache-5xx.md Quest-Claim: 25396aa5c18d85c8 From 61630f9f2a1aa08e84ff1fcf36f71ca0d8de2a31 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:39:34 -0800 Subject: [PATCH 4/4] chore(nix): finish deployed cache service-error quest Co-Authored-By: GPT-6.1 Sol --- quest/m0/ci/README.md | 1 - quest/m0/ci/nix-cache-5xx.md | 38 ------------------------------------ 2 files changed, 39 deletions(-) delete mode 100644 quest/m0/ci/nix-cache-5xx.md diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 6b84f40ebd1..d874bdbb2d0 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -175,7 +175,6 @@ The historical samples locate candidates; they are not controlled benchmarks. ## Required - [Read-only pull-request CI](/quest/m0/ci/read-only-secrets.md) - write secrets and publisher identities reach only protected branches and tags, never a pushed branch -- [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci` - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check`, test phases included, finishes within a worker's 5-minute prompt cache, admission wait included - [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - confirm the hosted Nightly jobs reuse the previous Nightly, then measure restore time and expiry - [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - confirm releases use `nix.moq.dev`, check its bucket sizes, and retire Cachix diff --git a/quest/m0/ci/nix-cache-5xx.md b/quest/m0/ci/nix-cache-5xx.md deleted file mode 100644 index 421a557b62f..00000000000 --- a/quest/m0/ci/nix-cache-5xx.md +++ /dev/null @@ -1,38 +0,0 @@ -# [S] A nix.moq.dev error does not fail a check - -## Goal - -A transient 5xx from `nix.moq.dev` costs a cache miss, not a failed -`just ci`: the dev shell still evaluates and builds what it could not -substitute. - -## Plan - -Decided 2026-10-31 (user, /quest-plan of the follow-ups from moq-dev/quest's host-gate cutover for moq.pro and rust2 and its fixer repairs). - -- On 2026-10-31 one HTTP 500 for a `.narinfo` failed `check flake` and - ejected a seven-PR batch in the host land gate. -- [The Worker fix](https://github.com/moq-dev/moq/pull/7537) turns R2 internal - errors (10001) and service-unavailable errors (10043) into logged misses: - the other bucket is still tried, and a resulting 404 is non-cacheable. - Malformed input, bad bindings, authorization - errors, and unknown failures still fail. No retries or client settings - change. Both buckets and metadata/archive reads have local test coverage. -- A local HTTP fault-injection probe verified that the real dev shell - evaluates and builds a fresh dependency from source with these 404s. - Nix 2.35.2 also completed that probe against the original 500 response, - while logging the error and disabling the cache. This does not reproduce - the historical `check flake` failure. The available Cloudflare OAuth token - received `Forbidden` when querying historical Worker logs, so the original - incident's R2 code or edge origin remains unconfirmed. -- Decided 2026-11-01 (maintainer): land the narrow service-code handling - without waiting for Workers Observability access to the original incident. -- Remaining: after the fix merges, the maintainer deploys the Worker with - `just infra nix deploy`, then verifies cache misses and dev-shell - evaluation/build through `nix.moq.dev`. -- The [Nix cache verification](/quest/m0/ci/nix-cache-verify.md) owns the - release side of the cache. - -## Related - -- [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - releases and bucket sizes