From 52908d12e1dc18d373dbde83fa4d0ea27c900807 Mon Sep 17 00:00:00 2001 From: Max Peterson Date: Mon, 26 Oct 2026 12:10:51 -0700 Subject: [PATCH 1/2] migrate the context gatekeeper to kernel-owned observer tracking --- docs/observers.md | 8 +- .../__tests__/collection-sharing.test.ts | 12 +- .../__tests__/context-observers.test.ts | 139 ++++++++++++------ .../__tests__/library-read.test.ts | 88 ++++++----- .../src/context-observers.ts | 110 ++++++++++---- .../src/library-gatekeeper.ts | 58 ++++---- .../gatekeeper-context/src/library-read.ts | 28 ++-- plans/context-collection-sharing.md | 20 +-- 8 files changed, 283 insertions(+), 180 deletions(-) diff --git a/docs/observers.md b/docs/observers.md index 294a96ebfd..2c26278dd2 100644 --- a/docs/observers.md +++ b/docs/observers.md @@ -893,7 +893,7 @@ its resource types. | **confluence** | Space | **C** | Verify space access; track observed pages and blog posts because content restrictions may be narrower. | | **confluence** | Page / Blog Post | **C** | Verify bound-content access; track observed child pages because they may have stricter restrictions than their parent. | | **zoominfo** | Account | **A** | Always throw. The whole-account binding exposes licensed, entitlement-dependent and account-specific intelligence, and ZoomInfo provides no ACL oracle proving another account can read every historical result. | -| **context** | Context Library singleton | **C** | Track collections whose contents a read revealed (search results, a collection's listing, a document); verify each is in the observer's sharing domain and is public, is everyone-viewable, or grants the observer's Context account a viewer, editor, or owner role. Uses the kit's default observer cap to bound per-read verifier fanout. The top-level collection listing tracks nothing: its titles, descriptions and counts are discovery metadata the agent catalog already advertises unobserved. The denial never names the collection, which the refused collaborator cannot read. | +| **context** | Context Library singleton | **C**, kernel-tracked (§10) | Never called: the connection advertises `supportsObserverTracking`. Each read reports the collections whose contents it revealed (search results, a collection's listing, a document) as `["collection", sharingDomain, collectionId]` subresources, and `verifyObserverAccess` denies each one the observer's Context account cannot read now: outside its sharing domain, or neither public, everyone-viewable, nor granted to it as viewer, editor, or owner. The top-level collection listing reports `[]`: its titles, descriptions and counts are discovery metadata the agent catalog already advertises unobserved. `getHistoricalSubresources` exports every `observedCollection:` marker the binding's own collection tracker left before the move, pending ones included, so those markers are never deleted. The Workshop's denial never names the collection. | | **x** | Account | **C** | Admit any observer with a working X connection: public posts, profiles, public Lists and follow graphs need only an X account. Reads private to the account -- bookmarks, likes, mute state, private Lists, and any page holding a protected account's post -- authorize against one synthetic `owner` collection, which admits only an observer connected as the same X user. | | **x** | Post / List | **C** | As Account, plus a check on every open that the observer's own X account can see the bound post or List (`canViewPost` / `canViewList`). Each check is a read billed to the observer's connection, so a passed check is trusted for an hour; a failed one never is. | | **x** | Profile | **C** | As Account: a public profile needs only an X account, and the `owner` collection covers a protected user's posts. Accepted over-permissiveness for every X binding: an author who has blocked the observer is still readable through the gadget, since checking each author per observer per read would cost a read each and what leaks is public. | @@ -937,7 +937,7 @@ This is why the broad bindings split the way they do: A connection whose `describe()` returns `supportsObserverTracking: true` is tracked by the Workshop itself instead of the protocol above (`src/observer-ledger.ts`). Connections without the flag keep the legacy `addObserver`/`removeObserver`/`excludeObservers` protocol unchanged, and one -workspace can mix the two. No shipping gatekeeper opts in yet. +workspace can mix the two. The Context Library is the first shipping gatekeeper to opt in (§9.2). - **The audit log is the requirement store.** Every tracked observation reports `subresources`: opaque, stable, non-secret IDs, fully qualified within their vendor (`[]` is valid; omitting it, @@ -1038,7 +1038,9 @@ workspace can mix the two. No shipping gatekeeper opts in yet. "use" collaborators only for gadgets that still reach the connection. The integration fixture's tracked mode (`/control/observer-tracking`) exercises this end to end in -`packages/integration-tests/__tests__/observer-tracking.test.ts`. +`packages/integration-tests/__tests__/observer-tracking.test.ts`, and +`context-observer-upgrade.test.ts` upgrades the real Context Library from the release that tracked +its own observers, importing what that release recorded. --- diff --git a/packages/gatekeeper-context/__tests__/collection-sharing.test.ts b/packages/gatekeeper-context/__tests__/collection-sharing.test.ts index cdb51a5f47..35f61325da 100644 --- a/packages/gatekeeper-context/__tests__/collection-sharing.test.ts +++ b/packages/gatekeeper-context/__tests__/collection-sharing.test.ts @@ -1,7 +1,9 @@ import { createExecutionContext, env, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; +import type { RpcStub } from "cloudflare:workers"; import { beforeEach, describe, expect, it } from "vitest"; -import { ObservationGate, privateObservers } from "@gadgets/gatekeeper-kit/observers"; -import type { GatekeeperUserSelectionExchange } from "@gadgets/workshop-shared/gatekeeper"; +import type { + GatekeeperUserSelectionExchange, ObservationAuthorizer, +} from "@gadgets/workshop-shared/gatekeeper"; import { ContextApiImpl } from "../src/context-api.js"; import { canReadCollection, collectionPermissions } from "../src/collection-access.js"; import { ContextCollectionDurableObject } from "../src/context-collection.js"; @@ -70,13 +72,13 @@ async function permissions(id: string, accountId: string) { } function readSession(accountId: string): LibraryReadSession { + // Only authorizeObservation() and disposal are reached by the read session. let authorizer = { async authorizeObservation() {}, [Symbol.dispose]() {}, - } as unknown as ConstructorParameters[0]; + } as unknown as RpcStub; return new LibraryReadSession( - testEnv, testEnv.COLLECTIONS, testEnv.USER_LIBRARIES, domain, accountId, - new ObservationGate(authorizer, privateObservers("No observers in this access-check fixture."))); + testEnv, testEnv.COLLECTIONS, testEnv.USER_LIBRARIES, domain, accountId, authorizer); } /** diff --git a/packages/gatekeeper-context/__tests__/context-observers.test.ts b/packages/gatekeeper-context/__tests__/context-observers.test.ts index 20407e4e40..b99da925b1 100644 --- a/packages/gatekeeper-context/__tests__/context-observers.test.ts +++ b/packages/gatekeeper-context/__tests__/context-observers.test.ts @@ -1,58 +1,103 @@ -import { describe, expect, it } from "vitest"; -import type { ObserverKv } from "@gadgets/gatekeeper-kit/observers"; -import type { GatekeeperUserVerifier } from "@gadgets/workshop-shared/gatekeeper"; +import { env } from "cloudflare:test"; +import { beforeEach, describe, expect, it } from "vitest"; +import type { ContextCollectionDurableObject } from "../src/context-collection.js"; import { - CONTEXT_OBSERVER_DENIED, contextObserverStrategy, + collectionSubresource, deniedCollectionSubresources, forgetLegacyObserver, + legacyCollectionSubresources, } from "../src/context-observers.js"; +import { domainName } from "../src/domain.js"; -function makeKv(): ObserverKv & { map: Map } { - let map = new Map(); - return { - map, - get: (key: string) => map.get(key) as T | undefined, - put: (key: string, value: T) => void map.set(key, value), - delete: (key: string) => map.delete(key), - list: ({ prefix }: { prefix?: string } = {}) => - [...map.entries()].filter(([key]) => key.startsWith(prefix ?? "")) as Iterable<[string, T]>, - } as ObserverKv & { map: Map }; -} +// The miniflare bindings vitest.config.ts declares, which Cloudflare.Env does not. +const testEnv = env as unknown as { COLLECTIONS: DurableObjectNamespace }; +const collections = testEnv.COLLECTIONS; + +// Storage persists across the file's tests, so each gets its own sharing domain. +let domain = ""; +let domains = 0; +beforeEach(() => { + domain = `observers-${domains++}.example`; +}); -function verifier(allowed: string[]) { - let calls: Array<{ sharingDomain: string; collectionIds: string[] }> = []; - let user = { - async hasCollectionsAccess(sharingDomain: string, collectionIds: string[]) { - calls.push({ sharingDomain, collectionIds }); - return collectionIds.map(id => allowed.includes(id)); - }, - } as unknown as Fetcher; - return { user, calls }; +const collection = (id: string) => collections.get(collections.idFromName(domainName(domain, id))); + +async function createCollection(id: string, visibility: "private" | "public", owner = "alice") { + let now = new Date(); + await collection(id).initialize({ + id, title: id, description: "", visibility, + created: now, lastUpdated: now, documentCount: 0, content: { source: "web" }, + }, domain, visibility === "private" ? owner : ""); } -describe("contextObserverStrategy", () => { - it("keeps enforcing collections recorded before the move to the kit tracker", async () => { - for (let legacy of [true, "observed", "pending"]) { - let kv = makeKv(); - kv.put("observedCollection:private", legacy); - let strategy = contextObserverStrategy(kv, "workshop.example"); - - await expect(strategy.addObserver("denied", verifier([]).user)) - .rejects.toThrow(CONTEXT_OBSERVER_DENIED); - let allowed = verifier(["private"]); - await strategy.addObserver("allowed", allowed.user); - expect(allowed.calls).toEqual([ - { sharingDomain: "workshop.example", collectionIds: ["private"] }, - ]); - } +const denied = (accountId: string, subresources: string[]) => + deniedCollectionSubresources(collections, domain, accountId, subresources); + +describe("deniedCollectionSubresources", () => { + it("grants exactly the collections the account can read now", async () => { + await createCollection("shared", "private"); + await collection("shared").share("alice", [{ accountId: "bob", displayName: "bob" }], "viewer"); + await createCollection("everyone", "private"); + await collection("everyone").setEveryoneCanView("alice", true); + await createCollection("public", "public"); + await createCollection("unshared", "private"); + await createCollection("deleted", "private"); + await collection("deleted").removeAccount("alice"); + + let readable = ["shared", "everyone", "public"].map(id => collectionSubresource(domain, id)); + let unreadable = [ + ...["unshared", "deleted", "never-created"].map(id => collectionSubresource(domain, id)), + // Another sharing domain's collection, even one with this ID. + collectionSubresource("elsewhere.example", "public"), + JSON.stringify(["document", domain, "public"]), + "not a subresource", + ]; + expect((await denied("bob", [...readable, ...unreadable])).toSorted()).toEqual(unreadable.toSorted()); + expect(await denied("alice", [collectionSubresource(domain, "unshared")])).toEqual([]); }); - it("refuses without disclosing which collection the collaborator cannot read", async () => { - let kv = makeKv(); - let strategy = contextObserverStrategy(kv, "workshop.example"); - let check = await strategy.prepare!(["private-collection-id"]); - check.commit(); + it("denies a collection as soon as the account's access to it is revoked", async () => { + await createCollection("shared", "private"); + await collection("shared").share("alice", [{ accountId: "bob", displayName: "bob" }], "viewer"); + let shared = collectionSubresource(domain, "shared"); + expect(await denied("bob", [shared])).toEqual([]); + + await collection("shared").removeMember("alice", "bob"); + + expect(await denied("bob", [shared, shared])).toEqual([shared, shared]); + }); +}); + +describe("legacyCollectionSubresources", () => { + it("exports every collection the gatekeeper's own tracker recorded, whatever its state", async () => { + // The layout the tracker left in a binding's storage: one marker per collection a read revealed, + // beside each admitted observer's registration. + let storage = new Map([ + ["observedCollection:unshared", true], + ["observedCollection:shared", "observed"], + ["observedCollection:uncertain", "pending"], + ["observer:collaborator", { verifier: "stub" }], + ["observer-attempt:candidate", { verifier: "stub", at: 0 }], + ["observer-nonce:candidate", "nonce"], + ]); + let kv = { + list: ({ prefix = "" }: SyncKvListOptions = {}) => + [...storage].filter(([key]) => key.startsWith(prefix)).values(), + delete: (key: string) => storage.delete(key), + } as Pick; + await createCollection("unshared", "private"); + await createCollection("shared", "private"); + await collection("shared").share("alice", [{ accountId: "bob", displayName: "bob" }], "viewer"); + await createCollection("uncertain", "private"); + + // Retiring the tracker's registrations leaves the history to export again if the import fails. + forgetLegacyObserver(kv, "collaborator"); + forgetLegacyObserver(kv, "candidate"); + let exported = legacyCollectionSubresources(kv, domain); - let denial = strategy.addObserver("denied", verifier([]).user); - await expect(denial).rejects.toThrow(CONTEXT_OBSERVER_DENIED); - await expect(denial).rejects.not.toThrow(/private-collection-id/); + expect(exported.toSorted()).toEqual( + ["shared", "uncertain", "unshared"].map(id => collectionSubresource(domain, id)).toSorted()); + expect([...storage.keys()].every(key => key.startsWith("observedCollection:"))).toBe(true); + expect((await denied("bob", exported)).toSorted()).toEqual( + ["uncertain", "unshared"].map(id => collectionSubresource(domain, id)).toSorted()); + expect(await denied("alice", exported)).toEqual([]); }); }); diff --git a/packages/gatekeeper-context/__tests__/library-read.test.ts b/packages/gatekeeper-context/__tests__/library-read.test.ts index e0fd4b270c..75dc3f18e0 100644 --- a/packages/gatekeeper-context/__tests__/library-read.test.ts +++ b/packages/gatekeeper-context/__tests__/library-read.test.ts @@ -1,9 +1,7 @@ +import type { RpcStub } from "cloudflare:workers"; import { describe, expect, it } from "vitest"; -import { ObservationGate, type ObserverKv } from "@gadgets/gatekeeper-kit/observers"; -import type { - GatekeeperUserVerifier, ObservationDescription, -} from "@gadgets/workshop-shared/gatekeeper"; -import { CONTEXT_OBSERVER_DENIED, contextObserverStrategy } from "../src/context-observers.js"; +import type { ObservationAuthorizer, ObservationDescription } from "@gadgets/workshop-shared/gatekeeper"; +import { collectionSubresource } from "../src/context-observers.js"; import type { ContextCollectionDurableObject } from "../src/context-collection.js"; import { LibraryReadSession } from "../src/library-read.js"; import type { UserLibraryDurableObject } from "../src/user-library.js"; @@ -11,25 +9,6 @@ import { domainName } from "../src/domain.js"; const DOMAIN = "workshop.example"; -function makeKv(): ObserverKv { - let map = new Map(); - return { - get: (key: string) => map.get(key) as T | undefined, - put: (key: string, value: T) => void map.set(key, value), - delete: (key: string) => void map.delete(key), - list: ({ prefix }: { prefix: string }) => - [...map.entries()].filter(([key]) => key.startsWith(prefix)) as [string, T][], - }; -} - -function verifier(allowed: string[]): Fetcher { - return { - async hasCollectionsAccess(_sharingDomain: string, collectionIds: string[]) { - return collectionIds.map(id => allowed.includes(id)); - }, - } as unknown as Fetcher; -} - const DOCUMENT = { path: "plan.md", name: "plan.md", @@ -39,16 +18,18 @@ const DOCUMENT = { }; // The owner's library: one public collection and one private collection. -function makeSession({ publicDiscoveryFails = false, publicAccessFails = false } = {}) { - let kv = makeKv(); - let strategy = contextObserverStrategy(kv, DOMAIN); +function makeSession({ publicDiscoveryFails = false, publicAccessFails = false, refusal }: { + publicDiscoveryFails?: boolean; publicAccessFails?: boolean; refusal?: Error; +} = {}) { let authorized: ObservationDescription[] = []; + // Only authorizeObservation() and disposal are reached by the read session. let authorizer = { async authorizeObservation(description: ObservationDescription) { + if (refusal) throw refusal; authorized.push(description); }, [Symbol.dispose]() {}, - }; + } as unknown as RpcStub; let collections = { idFromName: (name: string) => name, get: (name: string) => ({ @@ -89,44 +70,57 @@ function makeSession({ publicDiscoveryFails = false, publicAccessFails = false } }, }, } as unknown as Pick; - let session = new LibraryReadSession( - env, collections, userLibraries, DOMAIN, "owner", - new ObservationGate( - authorizer as unknown as ConstructorParameters[0], strategy)); - return { session, strategy, authorized }; + let session = new LibraryReadSession(env, collections, userLibraries, DOMAIN, "owner", authorizer); + return { session, authorized }; } -describe("LibraryReadSession sharing", () => { - it("lists every collection without restricting who may open the workspace", async () => { - let { session, strategy, authorized } = makeSession(); - await strategy.addObserver("existing", verifier(["public"])); +describe("LibraryReadSession observations", () => { + it("lists every collection without depending on any of them", async () => { + let { session, authorized } = makeSession(); let listing = await session.list(); expect(listing.entries.map(entry => entry.type === "collection" && entry.id)) .toEqual(["private", "public"]); - expect(authorized).toHaveLength(1); - expect(authorized[0].excludeObservers).toBeUndefined(); - await strategy.addObserver("later", verifier(["public"])); + expect(authorized.map(description => description.subresources)).toEqual([[]]); }); - it("restricts sharing once a private collection's contents are listed or read", async () => { + it("depends on a private collection once its contents are listed or read", async () => { for (let reveal of [ (session: LibraryReadSession) => session.list({ collectionId: "private" }), (session: LibraryReadSession) => session.read("private/plan.md"), ]) { - let { session, strategy, authorized } = makeSession(); - await strategy.addObserver("existing", verifier(["public"])); + let { session, authorized } = makeSession(); await reveal(session); - expect(authorized[0].excludeObservers).toEqual(["existing"]); - await expect(strategy.addObserver("later", verifier(["public"]))) - .rejects.toThrow(CONTEXT_OBSERVER_DENIED); - await strategy.addObserver("owner-account", verifier(["public", "private"])); + expect(authorized.map(description => description.subresources)) + .toEqual([[collectionSubresource(DOMAIN, "private")]]); } }); + it("depends only on the collections a search returned results from", async () => { + let { session, authorized } = makeSession(); + + expect((await session.search("plan", { limit: 1 })).map(result => result.docId)) + .toEqual(["private/plan.md"]); + expect(await session.search("plan")).toHaveLength(2); + + expect(authorized.map(description => description.subresources)).toEqual([ + [collectionSubresource(DOMAIN, "private")], + [collectionSubresource(DOMAIN, "private"), collectionSubresource(DOMAIN, "public")], + ]); + }); + + it("returns nothing a refused observation covered", async () => { + let refusal = new Error("This workspace may not read that."); + let { session } = makeSession({ refusal }); + + await expect(session.read("private/plan.md")).rejects.toBe(refusal); + await expect(session.list({ collectionId: "private" })).rejects.toBe(refusal); + await expect(session.search("plan")).rejects.toBe(refusal); + }); + it("keeps healthy private and public results when another public collection is unavailable", async () => { using session = makeSession({ publicAccessFails: true }).session; diff --git a/packages/gatekeeper-context/src/context-observers.ts b/packages/gatekeeper-context/src/context-observers.ts index cf2f7bdb46..dfd383504b 100644 --- a/packages/gatekeeper-context/src/context-observers.ts +++ b/packages/gatekeeper-context/src/context-observers.ts @@ -1,16 +1,21 @@ -import { - trackedCollectionObservers, type ObserverKv, type ObserverStrategy, -} from "@gadgets/gatekeeper-kit/observers"; +// Kernel-owned observer tracking for the Context Library (docs/observers.md §10). Collections are the +// unit: every document and skill in one inherits its access, so a read depends on each collection +// whose contents it revealed. The Workshop records those collections and asks each collaborator's +// own verifier whether they can still read them. + import type { GatekeeperUserVerifier } from "@gadgets/workshop-shared/gatekeeper"; -import { VENDOR_ID } from "./context-types.js"; +import { canReadCollection } from "./collection-access.js"; +import type { ContextCollectionDurableObject } from "./context-collection.js"; +import { MAX_COLLECTION_FANOUT, mapWithConcurrency } from "./concurrency.js"; +import { domainName } from "./domain.js"; /** - * The non-standard method Context gatekeepers call on their own verifier. The overseer only passes - * a verifier back to the vendor that minted it, so the gatekeeper may trust this result. + * A Context account's verifier. The overseer only passes a verifier back to the vendor that minted + * it, so the management API may trust `getAccountId()`. */ export interface ContextVerifierApi extends GatekeeperUserVerifier { - /** One verdict per collection ID: whether this account can read that collection. */ - hasCollectionsAccess(sharingDomain: string, collectionIds: string[]): Promise; + /** Denies each collection subresource this account cannot read now. */ + verifyObserverAccess(subresources: string[]): Promise<{ denied: string[] }>; /** * The account's ID, so a collection can be shared with the person a user-picker ticket names. * Null for any other sharing domain, which this account has no collections in. @@ -19,26 +24,81 @@ export interface ContextVerifierApi extends GatekeeperUserVerifier { } /** - * Shown to a collaborator refused admission. It must not name the collection: the collaborator - * cannot read it, and a private collection's title is its members' data. + * The subresource a read revealing `collectionId`'s contents depends on. It names the sharing domain + * too, since collection IDs are only unique within one. A JSON tuple, so no pair of strings can + * spell another pair's ID. */ -export const CONTEXT_OBSERVER_DENIED = - "This workspace has read from a Skills collection that hasn't been shared with your account, so " + - "you can't open it. Only collaborators who can read every collection the workspace has used may " + - "open it."; +export function collectionSubresource(sharingDomain: string, collectionId: string): string { + return JSON.stringify(["collection", sharingDomain, collectionId]); +} + +function parseCollectionSubresource(subresource: string): + { sharingDomain: string; collectionId: string } | null { + let parsed: unknown; + try { + parsed = JSON.parse(subresource); + } catch { + return null; + } + if (!Array.isArray(parsed) || parsed.length !== 3 || parsed[0] !== "collection") return null; + let sharingDomain: unknown = parsed[1], collectionId: unknown = parsed[2]; + if (typeof sharingDomain !== "string" || typeof collectionId !== "string" || !collectionId) { + return null; + } + return { sharingDomain, collectionId }; +} /** - * Strategy C for the broad Context Library singleton. Collections are the data sets: public - * collections are domain-wide; private collections admit their grantees and, when enabled, everyone - * in the sharing domain. The prefix keeps markers recorded before this tracker moved to the kit. + * The subresources `accountId` cannot read now: collections it has no access to under + * `collection-access.ts`, collections in another sharing domain, and anything that is not a + * collection subresource. Throws when a collection cannot be asked, which the Workshop treats as a + * denial. */ -export function contextObserverStrategy(kv: ObserverKv, sharingDomain: string): ObserverStrategy { - return trackedCollectionObservers>({ - kv, - collectionPrefix: "observedCollection:", - hasCollectionAccess: (verifier, collectionIds) => - verifier.hasCollectionsAccess(sharingDomain, [...collectionIds]), - denyMessage: () => CONTEXT_OBSERVER_DENIED, - vendorId: VENDOR_ID, +export async function deniedCollectionSubresources( + collections: DurableObjectNamespace, + sharingDomain: string, accountId: string, subresources: string[]): Promise { + let denied: string[] = []; + let byCollection = new Map(); + for (let subresource of subresources) { + let parsed = parseCollectionSubresource(subresource); + if (!parsed || parsed.sharingDomain !== sharingDomain) { + denied.push(subresource); + continue; + } + let named = byCollection.get(parsed.collectionId); + if (named) named.push(subresource); + else byCollection.set(parsed.collectionId, [subresource]); + } + let collectionIds = [...byCollection.keys()]; + let readable = await mapWithConcurrency(collectionIds, MAX_COLLECTION_FANOUT, async collectionId => + canReadCollection(await collections.get( + collections.idFromName(domainName(sharingDomain, collectionId))).getAccess(accountId))); + collectionIds.forEach((collectionId, index) => { + if (!readable[index]) denied.push(...byCollection.get(collectionId)!); }); + return denied; +} + +// Where the gatekeeper's own tracker recorded each collection a read revealed, before the Workshop +// tracked observations. Never deleted: a binding upgrades whenever its workspace next uses it, and +// its export may be asked for again until the Workshop commits the import. +const LEGACY_COLLECTION_PREFIX = "observedCollection:"; + +// Where that tracker kept each admitted observer's verifier, and each admission in flight. +const LEGACY_OBSERVER_PREFIXES = ["observer:", "observer-attempt:", "observer-nonce:"]; + +/** + * Every collection the binding's reads revealed before the Workshop tracked them, as subresources. + * A marker of any state counts: a "pending" one belongs to a read whose authorization outcome was + * never learned, so the Workshop may have recorded it. + */ +export function legacyCollectionSubresources( + kv: Pick, sharingDomain: string): string[] { + return Array.from(kv.list({ prefix: LEGACY_COLLECTION_PREFIX }), ([key]) => + collectionSubresource(sharingDomain, key.slice(LEGACY_COLLECTION_PREFIX.length))); +} + +/** Drops the verifier and any admission attempt the binding's tracker kept for `observerId`. */ +export function forgetLegacyObserver(kv: Pick, observerId: string): void { + for (let prefix of LEGACY_OBSERVER_PREFIXES) kv.delete(prefix + observerId); } diff --git a/packages/gatekeeper-context/src/library-gatekeeper.ts b/packages/gatekeeper-context/src/library-gatekeeper.ts index 83e9aae9bc..08961706ca 100644 --- a/packages/gatekeeper-context/src/library-gatekeeper.ts +++ b/packages/gatekeeper-context/src/library-gatekeeper.ts @@ -13,12 +13,12 @@ import type { SlashCommandDescriptor, SlashCommandProvider, SlashCommandResult, } from "@gadgets/workshop-shared/gatekeeper"; import { workerVersionTag } from "@gadgets/workshop-shared/gatekeeper"; -import { ObservationGate } from "@gadgets/gatekeeper-kit/observers"; import { LibraryReadSession } from "./library-read.js"; import { ContextApiImpl } from "./context-api.js"; -import { contextObserverStrategy } from "./context-observers.js"; -import type { ContextVerifierApi } from "./context-observers.js"; -import { canReadCollection } from "./collection-access.js"; +import { + deniedCollectionSubresources, forgetLegacyObserver, legacyCollectionSubresources, + type ContextVerifierApi, +} from "./context-observers.js"; import { buildAgentSkillCommands, buildAgentSkillMessage, buildContextCatalog, parseSkillManifest, type CollectionSkills, @@ -198,8 +198,8 @@ export class ContextAccount } /** - * Mint a verifier tied to this account. ContextGatekeeper uses it to check whether a prospective - * observer can independently read each collection the Gadget has observed. + * Mint a verifier tied to this account. The Workshop asks it whether a prospective observer can + * independently read each collection a workspace has observed. */ @skipRpcValidation() async getVerifier(): Promise> { @@ -211,12 +211,11 @@ export class ContextAccount export class ContextVerifier extends WorkerEntrypoint implements ContextVerifierApi { - async hasCollectionsAccess(sharingDomain: string, collectionIds: string[]): Promise { - if (sharingDomain !== this.ctx.props.sharingDomain) return collectionIds.map(() => false); - let collections = this.ctx.exports.ContextCollectionDurableObject; - return Promise.all(collectionIds.map(async collectionId => canReadCollection( - await collections.get(collections.idFromName(domainName(sharingDomain, collectionId))) - .getAccess(this.ctx.props.accountId)))); + async verifyObserverAccess(subresources: string[]): Promise<{ denied: string[] }> { + return { + denied: await deniedCollectionSubresources(this.ctx.exports.ContextCollectionDurableObject, + this.ctx.props.sharingDomain, this.ctx.props.accountId, subresources), + }; } async getAccountId(sharingDomain: string): Promise { @@ -231,9 +230,6 @@ export class ContextGatekeeper implements Gatekeeper { #collections() { return this.ctx.exports.ContextCollectionDurableObject; } #userLibraries() { return this.ctx.exports.UserLibraryDurableObject; } - // One strategy over the same storage object for the DO's lifetime: the kit coordinates in-flight - // reads in memory keyed on it. - readonly #observers = contextObserverStrategy(this.ctx.storage.kv, this.ctx.props.sharingDomain); async #loadSkills( collections: EnabledCollectionInfo[]): @@ -270,25 +266,24 @@ export class ContextGatekeeper suggestedBindingName: "CONTEXT", tsType: "ContextLibrary", hasSlashCommands: true, + supportsObserverTracking: true, }; } + /** The collections this binding's reads revealed while it tracked them itself. */ + async getHistoricalSubresources(): Promise { + return legacyCollectionSubresources(this.ctx.storage.kv, this.ctx.props.sharingDomain); + } + async getTypeScriptTypes(): Promise { return CONTEXT_LIBRARY_TYPES; } #newReadSession(authorizer: NativeRpcStub): LibraryReadSession { - // The read session uses this authorizer after startSession() returns, so its gate owns a - // duplicate. - let gate = new ObservationGate(authorizer.dup(), this.#observers); - try { - return new LibraryReadSession( - this.env, this.#collections(), this.#userLibraries(), - this.ctx.props.sharingDomain, this.ctx.props.accountId, gate); - } catch (err) { - gate[Symbol.dispose](); - throw err; - } + // The read session uses this authorizer after startSession() returns, so it owns a duplicate. + return new LibraryReadSession( + this.env, this.#collections(), this.#userLibraries(), + this.ctx.props.sharingDomain, this.ctx.props.accountId, authorizer.dup()); } async startSession(approvalQueue: NativeRpcStub): Promise { @@ -343,15 +338,16 @@ export class ContextGatekeeper } /** - * The Context singleton is a broad binding over public and shared collections. Track the - * collections actually revealed and verify every observer against each one. + * The Workshop verifies observers of a connection that supports observer tracking itself, so it + * never calls this; an admission reaching it would check nothing. */ - async addObserver(id: string, user: Fetcher): Promise { - await this.#observers.addObserver(id, user); + async addObserver(_id: string, _user: Fetcher): Promise { + throw new Error("Skills observers are verified by the Workshop."); } + /** Retires a registration the binding's own tracker kept before the Workshop took over. */ async removeObserver(id: string): Promise { - await this.#observers.removeObserver(id); + forgetLegacyObserver(this.ctx.storage.kv, id); } /** Read-only gatekeeper: no actions are submitted, so these callbacks should never run. */ diff --git a/packages/gatekeeper-context/src/library-read.ts b/packages/gatekeeper-context/src/library-read.ts index a10d2dbdbe..3b3aad5a80 100644 --- a/packages/gatekeeper-context/src/library-read.ts +++ b/packages/gatekeeper-context/src/library-read.ts @@ -1,9 +1,10 @@ -// Agent read path over the account's library. Every returned result is authorized as an observation; -// results revealing collection contents are attributed to those collections for observer checks. +// Agent read path over the account's library. Every returned result is authorized as an observation +// reporting the collections whose contents it revealed, for the Workshop's observer tracking. +import type { RpcStub } from "cloudflare:workers"; import { RpcTarget } from "capnweb"; import { validateRpc } from "capnweb-validate"; -import type { ObservationGate, ObservationInput } from "@gadgets/gatekeeper-kit/observers"; +import type { ObservationAuthorizer, ObservationDescription } from "@gadgets/workshop-shared/gatekeeper"; import { ContextSearchResult, ContextListing, ContextListingEntry, ContextReadResult, decodeDocId, encodeDocId, isTextContentType, VENDOR_ID, @@ -13,6 +14,7 @@ import type { ContextCollectionDurableObject } from "./context-collection.js"; import type { UserLibraryDurableObject } from "./user-library.js"; import { listPublicCollectionsFromKv } from "./collection-kv.js"; import { MAX_COLLECTION_FANOUT, mapWithConcurrency } from "./concurrency.js"; +import { collectionSubresource } from "./context-observers.js"; import { domainName } from "./domain.js"; import { obsContext } from "./observability.js"; @@ -33,14 +35,15 @@ export class LibraryReadSession extends RpcTarget implements ContextLibrary { private userLibraries: DurableObjectNamespace, private domain: string, private accountId: string, - private gate: ObservationGate, + // Owned: the session uses it after startSession() returns. + private authorizer: RpcStub, ) { super(); } - /** Release the authorizer owned by this read session's gate. */ + /** Release the authorizer this read session owns. */ [Symbol.dispose](): void { - this.gate[Symbol.dispose](); + this.authorizer[Symbol.dispose](); } #collection(id: string): DurableObjectStub { @@ -91,12 +94,13 @@ export class LibraryReadSession extends RpcTarget implements ContextLibrary { || ((await this.#discoveredPublicIds()).includes(collectionId) && await this.#isLivePublic(collectionId)); } - // A read revealing no collection contents is authorized against the admission baseline, so it - // does not constrain who may observe the workspace. - #authorize(collectionIds: string[], description: ObservationInput): Promise { - return this.gate.authorize(description, collectionIds.length > 0 - ? { kind: "collections", ids: collectionIds } - : { kind: "baseline" }); + // A read revealing no collection contents depends on nothing that restricts who may see it. + #authorize(collectionIds: string[], + description: Pick): Promise { + return this.authorizer.authorizeObservation({ + ...description, + subresources: collectionIds.map(id => collectionSubresource(this.domain, id)), + }); } async search(query: string, opts?: { diff --git a/plans/context-collection-sharing.md b/plans/context-collection-sharing.md index 092af2f4a8..8334735331 100644 --- a/plans/context-collection-sharing.md +++ b/plans/context-collection-sharing.md @@ -193,29 +193,29 @@ must survive flag changes, and removing an explicit grant must not erase another ## Observer verification -Retain the existing collection-tracking observer strategy: the Context singleton spans collections -with different ACLs, and the tracker already records collection IDs. - -Update `ContextVerifier.hasCollectionsAccess()` to accept: +The Context singleton spans collections with different ACLs, so a read depends on each collection +whose contents it revealed. The Workshop tracks those collections itself (`docs/observers.md` §10): +each observation reports them as subresources, and `ContextVerifier.verifyObserverAccess()` denies +each one an observer cannot read, accepting: - Ownership. - Explicit viewer or editor grants. - Everyone-viewable permission. - Existing deployment-public access. -Keep it free of side effects: the tracker calls it on every read and on admission attempts that +Keep it free of side effects: the Workshop calls it on every read and on admission attempts that fail, so it is not a successful-join notification. Use the same authoritative policy as management and agent reads. A stale shared-list reference must not satisfy observer verification. Revocation must affect subsequent checks even when a read session or discovery entry already exists. -Update `CONTEXT_OBSERVER_DENIED`, which names "another user's private collection" as the example, -so it still makes sense when the collection is shared with others, and keep it free of collection -names. +The Workshop's denial never names a collection, whose title is its members' data. Relevant files: -- `src/context-observers.ts`: vendor-specific batched access oracle and tracked strategy. -- `src/library-gatekeeper.ts`: Context account, verifier, observer admission, and catalog integration. +- `src/context-observers.ts`: collection subresources, the batched access check, and the legacy + history export. +- `src/library-gatekeeper.ts`: Context account, verifier, observer tracking opt-in, and catalog + integration. - `src/library-read.ts`: collection-scoped observations and enabled-set caching. - `src/user-library.ts`: owned/shared discovery indexes. From 39113d962188a8e941c391ed153216620da50d11 Mon Sep 17 00:00:00 2001 From: Max Peterson Date: Mon, 26 Oct 2026 12:10:51 -0700 Subject: [PATCH 2/2] test the context gatekeeper's observer history import across a real upgrade --- docs/integration-testing.md | 9 +- packages/integration-tests/README.md | 16 +++- .../__tests__/context-fixtures.ts | 93 ++++++++++++++++++ .../context-observer-upgrade.test.ts | 84 ++++++++++++++++ .../__tests__/context-sharing.test.ts | 77 +++------------ .../__tests__/google-fake.ts | 27 +----- .../scripts/build-upgrade-base.ts | 95 +++++++++++++------ packages/integration-tests/src/harness.ts | 47 +++++++-- packages/integration-tests/vite.config.ts | 8 +- 9 files changed, 319 insertions(+), 137 deletions(-) create mode 100644 packages/integration-tests/__tests__/context-fixtures.ts create mode 100644 packages/integration-tests/__tests__/context-observer-upgrade.test.ts diff --git a/docs/integration-testing.md b/docs/integration-testing.md index f3b1c90d13..e3705d67e9 100644 --- a/docs/integration-testing.md +++ b/docs/integration-testing.md @@ -123,14 +123,15 @@ module at `fixtures/bundled-blueprints.ts`, and hands that fixture its list thro installer, and the check that decides whether to run it, are the Workshop's own. A gatekeeper's `main` is such an input too, so a gatekeeper can be upgraded from an older release: -`build:upgrade-base` bundles gatekeeper-google as an earlier revision of the repo had it, the -Google hooks suite boots that bundle, and `upgradeGoogle()` (`__tests__/google-fake.ts`) swaps in -this tree's build with `server.update(current => …)`, replacing only that Worker's config. +`build:upgrade-base` bundles gatekeeper-google and gatekeeper-context as earlier revisions of the +repo had them, the Google hooks and Context observer upgrade suites boot those bundles, and +`redeployWorker()` (`src/harness.ts`) swaps in this tree's build with `server.update(current => …)`, +replacing only that Worker's config. `update()` resolves at the first reload, not the last. It hands each Worker its new config in turn, and each rebuilds and reloads on its own, so the reload it waited for can be running one Worker's new build beside the other's old one, with a second reload still to come. That second -restart breaks the sessions and the requests the test has opened since. `upgradeGoogle()` gives +restart breaks the sessions and the requests the test has opened since. `redeployWorker()` gives every Worker's config a mark unique to the update, and waits until each reports it through `getWorker(name).getEnv()`, the one state no reload can leave half-applied. `Harness.redeployWorkshop()` doesn't wait this way yet. diff --git a/packages/integration-tests/README.md b/packages/integration-tests/README.md index beeb64ceaf..43b88f8c86 100644 --- a/packages/integration-tests/README.md +++ b/packages/integration-tests/README.md @@ -68,17 +68,23 @@ without `WEBHOOK_ORIGIN` gets. GitHub is `__tests__/github-fake.ts`, a handler m interceptor. The gatekeeper boots from the tree `vp run -F @gadgets/github-gatekeeper build:integration-worker` validates, which the `test` task and `test:prebuild` build first. -## An upgrade: Google's hooks +## Upgrades: Google's hooks and Context's observer tracking `__tests__/google-hooks-upgrade.test.ts` enables Gmail and Chat hooks under the release of `gatekeeper-google` that came before its hook delivery queue moved into gatekeeper-kit, then upgrades that Worker in place to this tree's build, keeping its storage. It checks that queued deliveries, the hooks' capabilities and alarms, the Gmail watch and cursor and the Chat subscription all carry over, and that disconnecting reaches hooks the old release never told the account about. -Google is `__tests__/google-fake.ts`. `vp run -F @gadgets/integration-tests build:upgrade-base` -(`scripts/build-upgrade-base.ts`) bundles the old release from a temporary worktree of its revision. -The new one comes from `vp run -F @gadgets/google-gatekeeper build:integration-worker`. The `test` -task and `test:prebuild` build both first. +Google is `__tests__/google-fake.ts`. + +`__tests__/context-observer-upgrade.test.ts` reads a private collection under the release of +`gatekeeper-context` that tracked its own observers, upgrades it the same way, and checks that the +Workshop imports what it recorded: a collaborator who cannot read the collection is refused before +any read since, and admitted once they can. + +`vp run -F @gadgets/integration-tests build:upgrade-base` (`scripts/build-upgrade-base.ts`) bundles +each old release from a temporary worktree of its revision. The new ones come from each package's +`build:integration-worker`. The `test` task and `test:prebuild` build them all first. ## Further reading diff --git a/packages/integration-tests/__tests__/context-fixtures.ts b/packages/integration-tests/__tests__/context-fixtures.ts new file mode 100644 index 0000000000..33ef37685d --- /dev/null +++ b/packages/integration-tests/__tests__/context-fixtures.ts @@ -0,0 +1,93 @@ +// The real Context Library gatekeeper as the integration suites boot it: this tree's build, or the +// release before this one for the suite that upgrades it in place, plus the accounts and workspaces +// those suites read collections through. + +import { join } from "node:path"; +import { fileURLToPath, URL } from "node:url"; +import type { RpcStub } from "capnweb"; +import type { AuthenticatedApi } from "@gadgets/workshop-shared/api"; +import type { ContextApi, ContextLibrary } from "../../gatekeeper-context/src/context-types.js"; +import { upgradeBaseDir } from "../scripts/build-upgrade-base.js"; +import { readWorkerConfig, redeployWorker, type GatekeeperSpec, type Harness } from "../src/harness.js"; +import { + connect, listConnectedAccounts, logIn, nextUsernames, signUp, waitFor, type ConnectedAccount, +} from "../src/rpc-client.js"; + +const CONTEXT_DIR = fileURLToPath(new URL("../../gatekeeper-context/", import.meta.url)); +const UPGRADE_BASE_DIR = upgradeBaseDir("gatekeeper-context"); +export const CONTEXT_VENDOR_ID = "context"; +export const DOCUMENT_PATH = "plan.md"; +export const DOCUMENT = { description: "Owner-only launch plan", body: "Launch code: private-context." }; + +/** This tree's gatekeeper-context, from the validated tree the task graph builds before any suite. */ +export function contextGatekeeper(): GatekeeperSpec { + return { binding: "CONTEXT", dir: CONTEXT_DIR, patch: config => { delete config.build; } }; +} + +/** gatekeeper-context as the release before this one shipped it: the bundle `build:upgrade-base` leaves. */ +export function baseContextGatekeeper(): GatekeeperSpec { + return { + binding: "CONTEXT", + dir: UPGRADE_BASE_DIR, + patch: config => { + delete config.build; + config.main = join(UPGRADE_BASE_DIR, "bundle/index.js"); + }, + }; +} + +/** Deploy this tree's gatekeeper-context over the running one, keeping its storage. */ +export async function upgradeContext(harness: Harness): Promise { + const config = readWorkerConfig(CONTEXT_DIR); + delete config.build; + await redeployWorker(harness, config); +} + +export type ContextUser = Disposable & { + username: string; + api: RpcStub; + account: ConnectedAccount; + context: RpcStub; +}; + +/** + * `username` signed in, with their Context account and its management API. A new user signs up and + * provisions the account; an existing one, after a redeploy broke their sessions, logs in again. + */ +export async function contextUser(harness: Harness, username: string, existing = false) + : Promise { + using stack = new DisposableStack(); + const connection = stack.use(connect(harness.url)); + const api = stack.use(existing ? await logIn(connection, username) : await signUp(connection, username)); + if (!existing) await api.provisionAmbientAccount(CONTEXT_VENDOR_ID); + const account = await waitFor("the Context account to be provisioned", async () => + (await listConnectedAccounts(api)).find(entry => entry.vendorId === CONTEXT_VENDOR_ID) ?? null); + const frame = await api.getGatekeeperApp(CONTEXT_VENDOR_ID); + if (frame === null) throw new Error("The Context management app is unavailable"); + const context = stack.use(frame.ui as unknown as RpcStub); + return Object.assign(stack.move(), { username, api, account, context }); +} + +export function newContextUser(harness: Harness, prefix: string): Promise { + const [username] = nextUsernames(prefix); + if (username === undefined) throw new Error("Missing test username"); + return contextUser(harness, username); +} + +/** A new workspace of `owner`'s, with a private collection holding DOCUMENT and a session to read it. */ +export async function privateWorkspace(owner: ContextUser) { + using stack = new DisposableStack(); + const collection = await owner.context.createContextCollection( + `${owner.username} private collection`, "Private discovery metadata", "private"); + await owner.context.putContextDocument(collection.id, DOCUMENT_PATH, DOCUMENT); + const workspace = stack.use(await owner.api.newGadget()); + const { id: workspaceId } = await workspace.getMetadata(); + const requirement = await waitFor("the workspace's ambient Context binding", async () => + (await workspace.listObserverRequirements("build")) + .find(entry => entry.vendorId === CONTEXT_VENDOR_ID) ?? null); + const binding = stack.use(await workspace.getGatekeeperById(requirement.gatekeeperId)); + const session = stack.use(await binding.openSession() as RpcStub); + return Object.assign(stack.move(), { + collection, workspace, workspaceId, session, docId: `${collection.id}/${DOCUMENT_PATH}`, + }); +} diff --git a/packages/integration-tests/__tests__/context-observer-upgrade.test.ts b/packages/integration-tests/__tests__/context-observer-upgrade.test.ts new file mode 100644 index 0000000000..7930fad678 --- /dev/null +++ b/packages/integration-tests/__tests__/context-observer-upgrade.test.ts @@ -0,0 +1,84 @@ +// The Context Library's observer tracking carried across the upgrade that moves it into the +// Workshop: a workspace reads a private collection under the release before this one, whose +// gatekeeper recorded the collection itself, and after the upgrade the Workshop imports that record +// (`getHistoricalSubresources()`) and verifies collaborators against it. The release before is main +// as it was before the move, built by `build:upgrade-base`, and the upgrade is wrangler's +// storage-preserving `update()`. + +import type { RpcStub } from "capnweb"; +import { afterAll, beforeAll, expect, it } from "vitest"; +import type { ContextLibrary } from "../../gatekeeper-context/src/context-types.js"; +import { settleRestart, startHarness, type Harness } from "../src/harness.js"; +import { NetworkInterceptor } from "../src/network-interceptor.js"; +import { MAX_OBSERVER_PROMPTS, ObserverConfigRecorder, stubFor } from "../src/rpc-client.js"; +import { + baseContextGatekeeper, CONTEXT_VENDOR_ID, contextUser, DOCUMENT, newContextUser, privateWorkspace, + upgradeContext, +} from "./context-fixtures.js"; + +/** The Workshop's refusal of an account that cannot read what a tracked connection recorded. */ +const DENIED = "This account can't access some of the data this workspace read through this service."; + +let harness: Harness; +const network = new NetworkInterceptor({ handlers: [] }); + +beforeAll(async () => { + network.install(); + harness = await startHarness({ gatekeepers: [baseContextGatekeeper()] }); +}); + +afterAll(async () => { + try { + await harness?.server.close(); + expect(network.getUnmockedCalls()).toEqual([]); + } finally { + network.uninstall(); + } +}); + +it("verifies collaborators against collections the workspace read before the upgrade", async () => { + using stack = new DisposableStack(); + + // ── Under the release before this one ────────────────────────────────────────────────────── + const ownerBefore = stack.use(await newContextUser(harness, "contextupgradeowner")); + const observerBefore = stack.use(await newContextUser(harness, "contextupgradeobserver")); + const state = stack.use(await privateWorkspace(ownerBefore)); + const { collection, workspaceId, docId } = state; + expect(await state.session.read(docId)).toMatchObject({ content: DOCUMENT.body }); + await state.workspace.addCollaborator(observerBefore.username, "build"); + + // ── The upgrade ───────────────────────────────────────────────────────────────────────────── + await upgradeContext(harness); + + // ── Under this tree's release ─────────────────────────────────────────────────────────────── + const owner = stack.use(await contextUser(harness, ownerBefore.username, true)); + const observer = stack.use(await contextUser(harness, observerBefore.username, true)); + const recorder = new ObserverConfigRecorder() + .alwaysChoose(observer.account.id, MAX_OBSERVER_PROMPTS); + const callback = stack.use(stubFor(recorder)); + + // No read since the upgrade, so only the imported record can refuse the collaborator. + await expect(observer.api.openGadget(workspaceId, undefined, callback)).rejects.toThrow(DENIED); + const failure = recorder.needAt(0).failure; + expect(failure).toMatchObject({ accountId: observer.account.id, reason: expect.stringContaining(DENIED) }); + expect(failure?.reason).not.toContain(collection.id); + expect(failure?.reason).not.toContain(collection.title); + // The import restarts the workspace's sessions: let that land before anyone opens it again. + await settleRestart(); + const workspace = stack.use(await owner.api.openGadget(workspaceId)); + const { entries } = await workspace.listActions({ filter: "observation" }); + expect(entries.map(entry => entry.description.title)).toContain("Imported earlier reads"); + + // The imported collection is checked live, like one read since. + await owner.context.setContextCollectionEveryoneCanView(collection.id, true); + const opened = stack.use(await observer.api.openGadget(workspaceId, undefined, callback)); + expect(await opened.getMetadata()).toMatchObject({ id: workspaceId, role: "build" }); + + // The upgraded connection reads on, now tracked by the Workshop. + const requirement = (await workspace.listObserverRequirements("build")) + .find(entry => entry.vendorId === CONTEXT_VENDOR_ID); + if (!requirement) throw new Error("The workspace lost its ambient Context binding"); + const binding = stack.use(await workspace.getGatekeeperById(requirement.gatekeeperId)); + const session = stack.use(await binding.openSession() as RpcStub); + expect(await session.read(docId)).toMatchObject({ content: DOCUMENT.body }); +}); diff --git a/packages/integration-tests/__tests__/context-sharing.test.ts b/packages/integration-tests/__tests__/context-sharing.test.ts index 8b98f871a8..4b39322c1f 100644 --- a/packages/integration-tests/__tests__/context-sharing.test.ts +++ b/packages/integration-tests/__tests__/context-sharing.test.ts @@ -1,39 +1,15 @@ -import { fileURLToPath, URL } from "node:url"; -import type { RpcStub } from "capnweb"; import { afterAll, beforeAll, expect, it } from "vitest"; -import type { AuthenticatedApi } from "@gadgets/workshop-shared/api"; -import type { ContextApi, ContextLibrary } from "../../gatekeeper-context/src/context-types.js"; import { startHarness, type Harness } from "../src/harness.js"; import { NetworkInterceptor } from "../src/network-interceptor.js"; -import { - connect, listConnectedAccounts, MAX_OBSERVER_PROMPTS, nextUsernames, - ObserverConfigRecorder, signUp, stubFor, waitFor, type ConnectedAccount, -} from "../src/rpc-client.js"; +import { MAX_OBSERVER_PROMPTS, ObserverConfigRecorder, stubFor } from "../src/rpc-client.js"; +import { contextGatekeeper, DOCUMENT, DOCUMENT_PATH, newContextUser, privateWorkspace } from "./context-fixtures.js"; -const CONTEXT_DIR = fileURLToPath(new URL("../../gatekeeper-context/", import.meta.url)); -const VENDOR_ID = "context"; -const DOCUMENT_PATH = "plan.md"; -const DOCUMENT = { description: "Owner-only launch plan", body: "Launch code: private-context." }; let harness: Harness; - -type ContextUser = Disposable & { - username: string; - api: RpcStub; - account: ConnectedAccount; - context: RpcStub; -}; const network = new NetworkInterceptor({ handlers: [] }); beforeAll(async () => { network.install(); - harness = await startHarness({ - gatekeepers: [{ - binding: "CONTEXT", - dir: CONTEXT_DIR, - // The task graph builds this tree before any test file boots a Worker. - patch: config => { delete config.build; }, - }], - }); + harness = await startHarness({ gatekeepers: [contextGatekeeper()] }); }); afterAll(async () => { @@ -45,39 +21,10 @@ afterAll(async () => { } }); -async function newAccount(prefix: string): Promise { - using stack = new DisposableStack(); - const [username] = nextUsernames(prefix); - if (username === undefined) throw new Error("Missing test username"); - const api = stack.use(await signUp(stack.use(connect(harness.url)), username)); - await api.provisionAmbientAccount(VENDOR_ID); - const account = await waitFor("the Context account to be provisioned", async () => - (await listConnectedAccounts(api)).find(entry => entry.vendorId === VENDOR_ID) ?? null); - const frame = await api.getGatekeeperApp(VENDOR_ID); - if (frame === null) throw new Error("The Context management app is unavailable"); - const context = stack.use(frame.ui as unknown as RpcStub); - return Object.assign(stack.move(), { username, api, account, context }); -} - -async function privateWorkspace(owner: ContextUser) { - using stack = new DisposableStack(); - const collection = await owner.context.createContextCollection( - `${owner.username} private collection`, "Private discovery metadata", "private"); - await owner.context.putContextDocument(collection.id, DOCUMENT_PATH, DOCUMENT); - const workspace = stack.use(await owner.api.newGadget()); - const { id: workspaceId } = await workspace.getMetadata(); - const requirement = await waitFor("the workspace's ambient Context binding", async () => - (await workspace.listObserverRequirements("build")) - .find(entry => entry.vendorId === VENDOR_ID) ?? null); - const binding = stack.use(await workspace.getGatekeeperById(requirement.gatekeeperId)); - const session = stack.use(await binding.openSession() as RpcStub); - return Object.assign(stack.move(), { collection, workspace, workspaceId, session }); -} - it.concurrent("top-level private collection metadata does not prevent a collaborator opening", async () => { - using owner = await newAccount("contextlistingowner"); - using observer = await newAccount("contextlistingobserver"); + using owner = await newContextUser(harness, "contextlistingowner"); + using observer = await newContextUser(harness, "contextlistingobserver"); using state = await privateWorkspace(owner); const { collection, workspace, workspaceId, session } = state; @@ -94,13 +41,12 @@ it.concurrent("top-level private collection metadata does not prevent a collabor }); it.concurrent.each(["list", "read"] as const)( - "a private collection's %s prevents a collaborator without collection access opening", + "a private collection's %s admits only collaborators who can read the collection", async reveal => { - using owner = await newAccount("contextcontentowner"); - using observer = await newAccount("contextcontentobserver"); + using owner = await newContextUser(harness, "contextcontentowner"); + using observer = await newContextUser(harness, "contextcontentobserver"); using state = await privateWorkspace(owner); - const { collection, workspace, workspaceId, session } = state; - const docId = `${collection.id}/${DOCUMENT_PATH}`; + const { collection, workspace, workspaceId, session, docId } = state; expect(await observer.context.getContextCollectionMetadata(collection.id)).toBeNull(); if (reveal === "list") { @@ -122,5 +68,10 @@ it.concurrent.each(["list", "read"] as const)( expect(failure).toMatchObject({ accountId: observer.account.id, reason: expect.any(String) }); expect(failure?.reason).not.toContain(collection.id); expect(failure?.reason).not.toContain(collection.title); + + // Access to the collection is checked live, so once it is theirs to read they may open. + await owner.context.setContextCollectionEveryoneCanView(collection.id, true); + using opened = await observer.api.openGadget(workspaceId, undefined, callback); + expect(await opened.getMetadata()).toMatchObject({ id: workspaceId, role: "build" }); }, ); diff --git a/packages/integration-tests/__tests__/google-fake.ts b/packages/integration-tests/__tests__/google-fake.ts index 93aa428a2d..0b5216dd6d 100644 --- a/packages/integration-tests/__tests__/google-fake.ts +++ b/packages/integration-tests/__tests__/google-fake.ts @@ -13,12 +13,12 @@ import { fileURLToPath } from "node:url"; import type { RpcStub } from "capnweb"; import { parse } from "jsonc-parser"; import type { AuthenticatedApi } from "@gadgets/workshop-shared/api"; -import { UPGRADE_BASE_DIR } from "../scripts/build-upgrade-base.js"; -import type { GatekeeperSpec, Harness, WorkerConfig } from "../src/harness.js"; +import { upgradeBaseDir } from "../scripts/build-upgrade-base.js"; +import { redeployWorker, type GatekeeperSpec, type Harness, type WorkerConfig } from "../src/harness.js"; import type { Handler } from "../src/network-interceptor.js"; import { listConnectedAccounts, waitFor } from "../src/rpc-client.js"; - const GOOGLE_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "../../gatekeeper-google"); +const UPGRADE_BASE_DIR = upgradeBaseDir("gatekeeper-google"); export const GOOGLE_WORKER = "gatekeeper-google"; export const GOOGLE_VENDOR_ID = "google"; /** Where browsers and Pub/Sub reach the gatekeeper, through the router, as `/gatekeeper/google`. */ @@ -78,26 +78,7 @@ export async function upgradeGoogle(harness: Harness): Promise { const google = parse(readFileSync(join(GOOGLE_DIR, "wrangler.jsonc"), "utf8")) as WorkerConfig; configure(google); google.main = main; - // update() resolves at the first reload, but each Worker rebuilds and reloads on its own, so - // that reload can still be running either one's earlier build, with another to come. Every - // Worker carries this upgrade's mark, and it is done once each reports it. - const upgrade = randomUUID(); - const workers: string[] = []; - await harness.server.update(current => ({ - ...current, - workers: current.workers.map(worker => { - if (!("config" in worker)) throw new Error("The harness boots inline configs only"); - const config = worker.config.name === GOOGLE_WORKER ? google : worker.config; - workers.push(config.name!); - return { config: { ...config, vars: { ...config.vars, TEST_UPGRADE: upgrade } } }; - }), - })); - await waitFor("every Worker to run the upgrade", async () => { - for (const name of workers) { - if ((await harness.server.getWorker(name).getEnv()).TEST_UPGRADE !== upgrade) return null; - } - return true; - }); + await redeployWorker(harness, google); } type Row = Record; diff --git a/packages/integration-tests/scripts/build-upgrade-base.ts b/packages/integration-tests/scripts/build-upgrade-base.ts index 36df7a5eeb..26716f9aac 100644 --- a/packages/integration-tests/scripts/build-upgrade-base.ts +++ b/packages/integration-tests/scripts/build-upgrade-base.ts @@ -1,34 +1,64 @@ -// Builds gatekeeper-google as it was at UPGRADE_BASE, for the suite that upgrades it in place to -// this tree's build (`__tests__/google-hooks-upgrade.test.ts`): the release a deployment runs -// before this one, whose Durable Object storage the new code has to take over. +// Builds gatekeepers as an earlier revision of the repo had them, for the suites that upgrade them in +// place to this tree's build: the release a deployment runs before this one, whose Durable Object +// storage the new code has to take over. gatekeeper-google for `__tests__/google-hooks-upgrade.test.ts`, +// gatekeeper-context for `__tests__/context-observer-upgrade.test.ts`. // -// It checks the revision out into a temporary git worktree, installs that tree's dependencies -// from the store, and bundles the worker with `wrangler deploy --dry-run`, so the suite boots a -// self-contained bundle with nothing of the old tree left to resolve. The worktree goes once the -// bundle is out. The bundle depends on this file alone, which names the revision and every step, +// Each is checked out into a temporary git worktree of its revision, which installs that tree's +// dependencies from the store and bundles the worker with `wrangler deploy --dry-run`, so the suite +// boots a self-contained bundle with nothing of the old tree left to resolve. The worktree goes once +// the bundle is out. A bundle depends on this file alone, which names each revision and every step, // so a rerun with this file unchanged returns at once. import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { pnpmCommand } from "@gadgets/scripts/pnpm-command"; -/** - * main as it was before the hook delivery queue moved into gatekeeper-kit. Move it forward when the - * next change to what the hook drivers store wants its upgrade tested, or once the Workshop no - * longer serves a gatekeeper this old. - */ -export const UPGRADE_BASE = "d958d4786fefa7284fba582043dc7566ec2d2ece"; - const SCRIPT = fileURLToPath(import.meta.url); const PACKAGE_DIR = resolve(dirname(SCRIPT), ".."); const WORKSPACE_DIR = resolve(PACKAGE_DIR, "../.."); -/** Where the bundle lands: `bundle/` holds the worker, beside the `wrangler.jsonc` it shipped with. */ -export const UPGRADE_BASE_DIR = join(PACKAGE_DIR, ".wrangler/upgrade-base/gatekeeper-google"); +type UpgradeBase = { + revision: string; + /** The gatekeeper's directory under `packages/`. */ + packageDir: string; + packageName: string; + /** Generates what bundling the gatekeeper reads but its own build step does not write. */ + prepare(gatekeeper: string): void; +}; + +const UPGRADE_BASES: UpgradeBase[] = [ + { + // main as it was before the hook delivery queue moved into gatekeeper-kit. Move it forward when + // the next change to what the hook drivers store wants its upgrade tested, or once the Workshop + // no longer serves a gatekeeper this old. + revision: "d958d4786fefa7284fba582043dc7566ec2d2ece", + packageDir: "gatekeeper-google", + packageName: "@gadgets/google-gatekeeper", + prepare: gatekeeper => pnpm(["exec", "gadgets-build-configurator", "."], gatekeeper), + }, + { + // main as it was before the Context Library moved its observer tracking into the Workshop. + revision: "835de9e8776429fc16781edbe66367d879a64b87", + packageDir: "gatekeeper-context", + packageName: "@gadgets/gatekeeper-context", + prepare: gatekeeper => { + // The worker bundles typed-storage's build output. + pnpm(["exec", "tsc", "-p", "../typed-storage"], gatekeeper); + // The management app is only embedded, and no suite renders the old release's. + mkdirSync(join(gatekeeper, "src/generated"), { recursive: true }); + writeFileSync(join(gatekeeper, "src/generated/app.txt"), "Skills"); + }, + }, +]; + +/** Where a gatekeeper's base lands: `bundle/` holds the worker, beside the `wrangler.jsonc` it shipped with. */ +export function upgradeBaseDir(packageDir: string): string { + return join(PACKAGE_DIR, ".wrangler/upgrade-base", packageDir); +} function run(command: string, args: string[], cwd: string): void { execFileSync(command, args, { cwd, stdio: "inherit" }); @@ -39,29 +69,34 @@ function pnpm(args: string[], cwd: string): void { run(command, argv, cwd); } -export function buildUpgradeBase(): void { - const marker = join(UPGRADE_BASE_DIR, "built-by"); - const builtBy = createHash("sha256").update(readFileSync(SCRIPT)).digest("hex"); +function buildBase({ revision, packageDir, packageName, prepare }: UpgradeBase, builtBy: string): void { + const out = upgradeBaseDir(packageDir); + const marker = join(out, "built-by"); if (existsSync(marker) && readFileSync(marker, "utf8") === builtBy) return; // A shallow clone, as CI checks out, lacks the revision. try { - execFileSync("git", ["cat-file", "-e", `${UPGRADE_BASE}^{commit}`], { cwd: WORKSPACE_DIR, stdio: "ignore" }); + execFileSync("git", ["cat-file", "-e", `${revision}^{commit}`], { cwd: WORKSPACE_DIR, stdio: "ignore" }); } catch { - run("git", ["fetch", "--depth=1", "origin", UPGRADE_BASE], WORKSPACE_DIR); + run("git", ["fetch", "--depth=1", "origin", revision], WORKSPACE_DIR); } const tree = mkdtempSync(join(tmpdir(), "gadgets-upgrade-base-")); - run("git", ["worktree", "add", "--detach", tree, UPGRADE_BASE], WORKSPACE_DIR); + run("git", ["worktree", "add", "--detach", tree, revision], WORKSPACE_DIR); try { - pnpm(["install", "--frozen-lockfile", "--prefer-offline", "--filter", "@gadgets/google-gatekeeper..."], tree); - const gatekeeper = join(tree, "packages/gatekeeper-google"); - pnpm(["exec", "gadgets-build-configurator", "."], gatekeeper); - rmSync(UPGRADE_BASE_DIR, { recursive: true, force: true }); - pnpm(["exec", "wrangler", "deploy", "--dry-run", "--outdir", join(UPGRADE_BASE_DIR, "bundle")], gatekeeper); - copyFileSync(join(gatekeeper, "wrangler.jsonc"), join(UPGRADE_BASE_DIR, "wrangler.jsonc")); + pnpm(["install", "--frozen-lockfile", "--prefer-offline", "--filter", `${packageName}...`], tree); + const gatekeeper = join(tree, "packages", packageDir); + prepare(gatekeeper); + rmSync(out, { recursive: true, force: true }); + pnpm(["exec", "wrangler", "deploy", "--dry-run", "--outdir", join(out, "bundle")], gatekeeper); + copyFileSync(join(gatekeeper, "wrangler.jsonc"), join(out, "wrangler.jsonc")); writeFileSync(marker, builtBy); } finally { run("git", ["worktree", "remove", "--force", tree], WORKSPACE_DIR); } } -if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) buildUpgradeBase(); +export function buildUpgradeBases(): void { + const builtBy = createHash("sha256").update(readFileSync(SCRIPT)).digest("hex"); + for (const base of UPGRADE_BASES) buildBase(base, builtBy); +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) buildUpgradeBases(); diff --git a/packages/integration-tests/src/harness.ts b/packages/integration-tests/src/harness.ts index ee638b69d5..3f886a8f66 100644 --- a/packages/integration-tests/src/harness.ts +++ b/packages/integration-tests/src/harness.ts @@ -5,13 +5,14 @@ // harness at the package and plug in a handler module", not a forked copy of this file. Per-vendor // suites in consumer repos use this as-is. -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { mkdirSync, readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parse } from "jsonc-parser"; import { createTestHarness, type TestHarness } from "wrangler"; import { z } from "zod"; +import { waitFor } from "./rpc-client.js"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -77,13 +78,15 @@ export type GatekeeperSpec = { patch?: (config: WorkerConfig) => void; }; -// Read a checked-in wrangler.jsonc and make it usable as an *inline* harness config. -// -// A worker whose `main` is generated (capnweb-validate) needs `build.cwd` pinned to its own directory -// or the output lands in the wrong place -- run-dev-server.ts pins it for the same reason. `main` then -// has to be absolute too: an inline config has no file path of its own, so wrangler resolves a -// relative `main` against the harness `root` rather than the worker directory. -function readWorkerConfig(dir: string): WorkerConfig { +/** + * Read a checked-in wrangler.jsonc and make it usable as an *inline* harness config. + * + * A worker whose `main` is generated (capnweb-validate) needs `build.cwd` pinned to its own directory + * or the output lands in the wrong place -- run-dev-server.ts pins it for the same reason. `main` then + * has to be absolute too: an inline config has no file path of its own, so wrangler resolves a + * relative `main` against the harness `root` rather than the worker directory. + */ +export function readWorkerConfig(dir: string): WorkerConfig { const path = join(dir, "wrangler.jsonc"); const parsed = WORKER_CONFIG.safeParse(parse(readFileSync(path, "utf8"))); if (!parsed.success) { @@ -183,6 +186,34 @@ export async function startHarness(opts: { }; } +/** + * Deploy `config` over the running Worker of the same name, as releasing it does: the same Worker, + * so the same Durable Objects and storage, running the new code. Every Worker restarts, which breaks + * the RPC sessions open at the time, so only a suite that started its own harness should call this. + */ +export async function redeployWorker(harness: Harness, config: WorkerConfig): Promise { + // update() resolves at the first reload, but each Worker rebuilds and reloads on its own, so that + // reload can still be running either one's earlier build, with another to come. Every Worker + // carries this redeploy's mark, and it is done once each reports it. + const mark = randomUUID(); + const workers: string[] = []; + await harness.server.update(current => ({ + ...current, + workers: current.workers.map(worker => { + if (!("config" in worker)) throw new Error("The harness boots inline configs only"); + const next = worker.config.name === config.name ? config : worker.config; + workers.push(next.name!); + return { config: { ...next, vars: { ...next.vars, TEST_UPGRADE: mark } } }; + }), + })); + await waitFor("every Worker to run the redeploy", async () => { + for (const name of workers) { + if ((await harness.server.getWorker(name).getEnv()).TEST_UPGRADE !== mark) return null; + } + return true; + }); +} + // Takes the place of the module the Workshop's build generates its bundled blueprints into. const BUNDLED_BLUEPRINTS_MODULE = resolve(HERE, "../fixtures/bundled-blueprints.ts"); diff --git a/packages/integration-tests/vite.config.ts b/packages/integration-tests/vite.config.ts index 7ff966eec9..2a25771ed1 100644 --- a/packages/integration-tests/vite.config.ts +++ b/packages/integration-tests/vite.config.ts @@ -20,9 +20,9 @@ export default { ], }, /** - * gatekeeper-google as the release before the hook delivery queue moved, for the suite that - * upgrades it in place (`scripts/build-upgrade-base.ts`). Uncached: the script reads git and - * the pnpm store, which no fingerprint covers, and returns at once when its output is current. + * gatekeeper-google and gatekeeper-context as earlier releases shipped them, for the suites + * that upgrade them in place (`scripts/build-upgrade-base.ts`). Uncached: the script reads git + * and the pnpm store, which no fingerprint covers, and returns at once when its output is current. */ 'build:upgrade-base': { command: withTestTimeout('node scripts/build-upgrade-base.ts'), @@ -51,7 +51,7 @@ export default { ...WRANGLER_RUNTIME_SCRATCH_EXCLUSIONS, ]), // Real gatekeepers boot from prebuilt validated trees, tracked as inputs like the fixture, - // plus the release the Google upgrade suite starts from. + // plus the releases the upgrade suites start from. dependsOn: [ 'build:test-gatekeeper', 'build:upgrade-base', '@gadgets/github-gatekeeper#build:integration-worker',