diff --git a/packages/agent-bff/package.json b/packages/agent-bff/package.json index 502c25c7c6..061952a66e 100644 --- a/packages/agent-bff/package.json +++ b/packages/agent-bff/package.json @@ -37,6 +37,7 @@ "@forestadmin/datasource-toolkit": "1.55.1", "@forestadmin/forestadmin-client": "1.43.2", "@koa/bodyparser": "^6.1.0", + "inflected": "^1.1.6", "jsonwebtoken": "^9.0.3", "koa": "^3.0.1", "zod": "4.3.6" @@ -46,6 +47,7 @@ "@forestadmin/agent-testing": "1.2.7", "@hey-api/openapi-ts": "0.99.0", "@redocly/cli": "2.35.1", + "@types/inflected": "^1.1.29", "@types/jsonwebtoken": "^9.0.1", "@types/koa": "^2.13.5", "@types/supertest": "^6.0.2", diff --git a/packages/agent-bff/src/openapi/collect-unfolding.ts b/packages/agent-bff/src/openapi/collect-unfolding.ts index 9c2a50dfd2..2a38ddb88c 100644 --- a/packages/agent-bff/src/openapi/collect-unfolding.ts +++ b/packages/agent-bff/src/openapi/collect-unfolding.ts @@ -132,7 +132,7 @@ async function collectFields( } return { - projectable: capabilities.fields.map(field => field.name), + projectable: capabilities.fields.map(({ name, type }) => ({ name, type })), filterable: collectFilterableFields(collection, capabilities, logger), degraded: null, }; diff --git a/packages/agent-bff/src/openapi/names.ts b/packages/agent-bff/src/openapi/names.ts index 80354609f8..b8c337a096 100644 --- a/packages/agent-bff/src/openapi/names.ts +++ b/packages/agent-bff/src/openapi/names.ts @@ -34,3 +34,8 @@ export default function createNamer(): Namer { return candidate; }; } + +/** A customer name inside prose. Quoted rather than bare: a name can carry spaces or punctuation. */ +export function quoted(name: string): string { + return JSON.stringify(name); +} diff --git a/packages/agent-bff/src/openapi/record-schemas.ts b/packages/agent-bff/src/openapi/record-schemas.ts new file mode 100644 index 0000000000..48dc7b497c --- /dev/null +++ b/packages/agent-bff/src/openapi/record-schemas.ts @@ -0,0 +1,153 @@ +import type { ProjectableField, UnfoldedCollection } from './unfolding'; +import type { ReferenceObject, SchemaObject } from 'openapi3-ts/oas31'; + +import Inflector from 'inflected'; + +import toFieldSchema from './field-schemas'; +import { quoted } from './names'; +import { PACKED_ID_SEPARATOR } from '../data/pack-id'; + +/** + * The key a field really carries in a response record. `agent-client` deserializes the agent's + * JSON:API with `keyForAttribute: 'camelCase'` (`http-requester.ts`), which is exactly this pair of + * `inflected` calls (`jsonapi-serializer/lib/inflector.js`), so a `first_name` column is PROJECTED + * under that name and RETURNED as `firstName`. The same library rather than a transcription: the + * transform handles acronyms and non-ASCII, and a mirror would drift from the deserializer without + * anything failing. + */ +export function recordKey(field: string): string { + return Inflector.camelize(Inflector.underscore(field), false); +} + +// The flat id is the JSON:API resource id, which is a string by specification whatever the key +// column holds. `__forest.primaryKey` is the same id unpacked and typed, so the two forms of one +// record disagree by construction — the trap this schema exists to name. +const ID_SCHEMA: SchemaObject = { + type: 'string', + description: + 'The record id, always a string — the agent serializes it as the JSON:API resource id, even ' + + 'when the key column is a Number. `__forest.primaryKey` carries the same id TYPED, so ' + + `comparing the two without coercion fails. A composite key is its values joined by ` + + `${quoted(PACKED_ID_SEPARATOR)}.`, +}; + +function isReference(schema: SchemaObject | ReferenceObject): schema is ReferenceObject { + return '$ref' in schema; +} + +/** + * Every published field is nullable. The capabilities report a column type and never its + * nullability, so a nullable column answers `null` against a type this schema would otherwise + * declare non-null — a generated client validating the response would reject what the runtime + * really sends. An unconstrained schema already accepts null and is left alone. + */ +function nullable(schema: SchemaObject): SchemaObject { + if (typeof schema.type !== 'string') return schema; + + const widened: SchemaObject = { ...schema, type: [schema.type, 'null'] }; + + if (schema.items !== undefined && !isReference(schema.items)) { + widened.items = nullable(schema.items); + } + + if (schema.properties !== undefined) { + widened.properties = Object.fromEntries( + Object.entries(schema.properties).map(([key, nested]) => [ + key, + isReference(nested) ? nested : nullable(nested), + ]), + ); + } + + return widened; +} + +/** + * The schema of one field, under the key the response carries it as. Several fields can collapse to + * the same key — the transform is lossy, `first_name` and `firstName` both yield `firstName` — and + * which one wins depends on the order the agent serialized them in, so the type is left open rather + * than picked. + */ +function propertySchema(key: string, fields: ProjectableField[]): SchemaObject { + const names = fields.map(field => quoted(field.name)).join(', '); + + if (fields.length > 1) { + return { + description: + `${names} all reach the response under this single key, so which one it holds is not ` + + 'determined here. Project one of them at a time to know.', + }; + } + + const [field] = fields; + const schema = nullable(toFieldSchema(field.type)); + + if (field.name === key) return schema; + + const description = [schema.description, `The ${names} field.`].filter(Boolean).join(' '); + + return { ...schema, description }; +} + +function fieldProperties(projectable: ProjectableField[]): Record { + const byKey = new Map(); + + projectable.forEach(field => { + const key = recordKey(field.name); + const collapsed = byKey.get(key); + + if (collapsed) collapsed.push(field); + else byKey.set(key, [field]); + }); + + return Object.fromEntries([...byKey].map(([key, fields]) => [key, propertySchema(key, fields)])); +} + +/** + * The record shape of one collection. The properties are what the capabilities report, and nothing + * more is forbidden: the capabilities route leaves out a OneToOne relation that the default + * projection returns, so a closed schema would reject responses the runtime really sends. `id` wins + * over any field of that name, because the deserializer overwrites the attribute with the resource + * id. + */ +export default function recordSchema( + collection: UnfoldedCollection, + forestMeta: ReferenceObject, +): SchemaObject { + return { + type: 'object', + description: + `A record of ${quoted(collection.name)}. It carries the fields the request projected — ` + + 'omit `projection` and the agent returns them all. The properties below are the ones the ' + + "collection's capabilities report; a record can carry more, such as a to-one relation. " + + 'Every field is nullable: the capabilities report a column type and never its nullability, ' + + 'so a null is always possible whatever the type says.', + properties: { + ...fieldProperties(collection.fields.projectable), + id: ID_SCHEMA, + __forest: forestMeta, + }, + required: ['id', '__forest'], + }; +} + +export function listResponseSchema( + collection: UnfoldedCollection, + record: ReferenceObject, +): SchemaObject { + return { + type: 'object', + description: + `A page of ${quoted(collection.name)} records. The list never carries a total: call the ` + + 'count endpoint for that, which is why `countStatus` is always `not_requested`.', + properties: { + data: { type: 'array', items: record }, + meta: { + type: 'object', + properties: { countStatus: { type: 'string', const: 'not_requested' } }, + required: ['countStatus'], + }, + }, + required: ['data', 'meta'], + }; +} diff --git a/packages/agent-bff/src/openapi/schemas.ts b/packages/agent-bff/src/openapi/schemas.ts index 23dacfc4c6..95b3670a04 100644 --- a/packages/agent-bff/src/openapi/schemas.ts +++ b/packages/agent-bff/src/openapi/schemas.ts @@ -1,6 +1,7 @@ import { allOperators } from '@forestadmin/datasource-toolkit'; import { z } from './zod-openapi'; +import { PACKED_ID_SEPARATOR } from '../data/pack-id'; import { PageInput, ParentIdInput, @@ -140,7 +141,7 @@ export const ActionRequestSchema = z 'targets no record. Every id is coerced to a string before reaching the agent.', }); -const ForestRecordMetaSchema = z +export const ForestRecordMetaSchema = z .object({ collection: z.string(), primaryKey: z.record(z.string(), z.union([z.string(), z.number()])), @@ -148,7 +149,9 @@ const ForestRecordMetaSchema = z .openapi('ForestRecordMeta', { description: 'The record identity, unpacked from the agent id. A composite primary key carries one ' + - 'entry per column.', + 'entry per column. The values are TYPED here — a Number key column is a number — whereas ' + + 'the record carries the same id as a string under `id`, so comparing the two forms ' + + 'without coercion fails.', }); export const ListResponseSchema = z @@ -160,8 +163,11 @@ export const ListResponseSchema = z }) .openapi('ListResponse', { description: - 'Records are flat, each carrying a `__forest` envelope. The list never carries a total: ' + - 'call the count endpoint for that, which is why `countStatus` is always `not_requested`.', + 'Records are flat, each carrying a `__forest` envelope. A record always holds `id`, the ' + + `agent id as a string — a composite key is its values joined by \`${PACKED_ID_SEPARATOR}\` — ` + + 'while `__forest.primaryKey` holds that same id typed and split per column. The list never ' + + 'carries a total: call the count endpoint for that, which is why `countStatus` is always ' + + '`not_requested`.', }); export const CountResponseSchema = z diff --git a/packages/agent-bff/src/openapi/unfolded-paths.ts b/packages/agent-bff/src/openapi/unfolded-paths.ts index 5e1a1a7f0d..623c255036 100644 --- a/packages/agent-bff/src/openapi/unfolded-paths.ts +++ b/packages/agent-bff/src/openapi/unfolded-paths.ts @@ -14,9 +14,11 @@ import type { OpenAPIRegistry } from '@asteasolutions/zod-to-openapi'; import type { ReferenceObject, SchemaObject } from 'openapi3-ts/oas31'; import toFieldSchema from './field-schemas'; -import createNamer from './names'; +import createNamer, { quoted } from './names'; +import recordSchema, { listResponseSchema } from './record-schemas'; import { CountResponseSchema, + ForestRecordMetaSchema, ListResponseSchema, OPERATORS, PageSchema, @@ -48,6 +50,8 @@ interface CollectionPlan { collection: UnfoldedCollection; key: string; requests: RequestRefs; + /** Carried on the plan so a relation list answers with the FOREIGN collection's own shape. */ + response: ReferenceObject; } interface Deps { @@ -59,10 +63,6 @@ interface Deps { errorResponses: (executeResults: boolean) => Record; } -function quoted(name: string): string { - return JSON.stringify(name); -} - // A name reaches the runtime through decodeURIComponent, so the document must carry it encoded — // including the separators that would otherwise change the route, such as a slash inside an action // name (`a/b` is reachable only as `a%2Fb`). @@ -228,7 +228,7 @@ function fieldRefs(deps: Deps, plan: Pick) const projectable = fieldsEnum( pool, `Fields_${plan.key}`, - fields.projectable, + fields.projectable.map(field => field.name), `A field of ${quoted(name)}.`, ); @@ -260,7 +260,10 @@ function requestDescription(collection: UnfoldedCollection, subject: string): st return `${subject}${note}`; } -function registerRequests(deps: Deps, plan: Omit): RequestRefs { +function registerRequests( + deps: Deps, + plan: Pick, +): RequestRefs { const { pool } = deps; const { collection, key } = plan; const refs = fieldRefs(deps, plan); @@ -292,6 +295,29 @@ function registerRequests(deps: Deps, plan: Omit): R }; } +/** + * A collection whose field set could not be read keeps the shared untyped response: a record schema + * built from an empty projectable set would claim the record holds nothing but `id` and `__forest`. + */ +function registerListResponse( + deps: Deps, + plan: Pick, +): ReferenceObject { + const { pool } = deps; + const { collection, key } = plan; + + if (collection.fields.projectable.length === 0) { + return pool.reuse('ListResponse', ListResponseSchema); + } + + const record = pool.add( + `Record_${key}`, + recordSchema(collection, pool.reuse('ForestRecordMeta', ForestRecordMetaSchema)), + ); + + return pool.add(`ListResponse_${key}`, listResponseSchema(collection, record)); +} + const PARENT_ID_SHAPE = { anyOf: [{ type: 'string' as const, pattern: '\\S' }, { type: 'number' as const }], }; @@ -460,7 +486,6 @@ function registerOperation(deps: Deps, options: OperationOptions): void { function registerCollectionOperations(deps: Deps, plan: CollectionPlan): void { const { pool } = deps; const { name } = plan.collection; - const listResponse = pool.reuse('ListResponse', ListResponseSchema); const countResponse = pool.reuse('CountResponse', CountResponseSchema); registerOperation(deps, { @@ -470,7 +495,7 @@ function registerCollectionOperations(deps: Deps, plan: CollectionPlan): void { summary: `List records of ${name}`, description: `Lists records of the ${quoted(name)} collection.`, request: plan.requests.list, - response: listResponse, + response: plan.response, responseDescription: `A page of ${quoted(name)} records`, bodyRequired: false, }); @@ -513,7 +538,7 @@ function registerRelationOperations( plan.collection.name, )} record through ${parent}.`, request: requests.list, - response: pool.reuse('ListResponse', ListResponseSchema), + response: foreign.response, responseDescription: `A page of related ${quoted(foreign.collection.name)} records`, bodyRequired: true, }); @@ -586,7 +611,12 @@ export default function registerUnfoldedPaths(deps: Deps, unfolding: Unfolding): const plans = unfolding.collections.map(collection => { const key = collections(collection.name); - return { collection, key, requests: registerRequests(deps, { collection, key }) }; + return { + collection, + key, + requests: registerRequests(deps, { collection, key }), + response: registerListResponse(deps, { collection, key }), + }; }); const plansByName = new Map(plans.map(plan => [plan.collection.name, plan])); diff --git a/packages/agent-bff/src/openapi/unfolding.ts b/packages/agent-bff/src/openapi/unfolding.ts index 280149e95d..d179fa05f4 100644 --- a/packages/agent-bff/src/openapi/unfolding.ts +++ b/packages/agent-bff/src/openapi/unfolding.ts @@ -34,11 +34,20 @@ export interface UnfoldedRelation { * their field schemas stay free-form strings — an empty enum would forbid every valid call. */ export interface CollectionFields { - projectable: string[]; + projectable: ProjectableField[]; filterable: FilterableField[]; degraded: DegradedReason | null; } +/** + * A field the agent reports, with the column type it declares. The type is what turns the response + * records into a typed schema; the name is what the request enums carry. + */ +export interface ProjectableField { + name: string; + type: FieldType; +} + /** * A field the agent reports at least one operator for, with that operator set — the very set the * runtime validates a filter leaf against. Normalized to canonical PascalCase and to the diff --git a/packages/agent-bff/test/data/fixtures/live-agent-harness.ts b/packages/agent-bff/test/data/fixtures/live-agent-harness.ts new file mode 100644 index 0000000000..509b3eac3b --- /dev/null +++ b/packages/agent-bff/test/data/fixtures/live-agent-harness.ts @@ -0,0 +1,86 @@ +import type { Logger } from '../../../src/ports/logger-port'; +import type { SchemaFetcher } from '../../../src/read-model/forest-schema-client'; +import type { ForestSchemaCollection } from '@forestadmin/forestadmin-client'; + +import { bodyParser } from '@koa/bodyparser'; +import fs from 'fs/promises'; +import jsonwebtoken from 'jsonwebtoken'; +import Koa from 'koa'; +import net from 'net'; + +import createDataRoutesMiddleware from '../../../src/data/data-routes-middleware'; +import createErrorMiddleware from '../../../src/http/error-middleware'; +import CapabilitiesCache from '../../../src/read-model/capabilities-cache'; +import ReadModelStore from '../../../src/read-model/read-model-store'; +import SchemaCache from '../../../src/read-model/schema-cache'; + +const TIMEZONE = 'Europe/Paris'; +export const AUTH_SECRET = 'b0bdf0a639c16bae8851dd24ee3d79ef0a352e957c5b86cb'; +export const ENV_SECRET = 'ceba742f5bc73946b34da192816a4d7177b3233fee7769955c29c0e90fd584f2'; +export const BOOT_TIMEOUT_MS = 60_000; + +// agent-testing only deletes a schema file whose name carries this prefix, so reusing it keeps the +// temporary schema cleaned up by `agent.stop()` even though the path is chosen here. +export const RESERVED_SCHEMA_PREFIX = 'reserved-forestadmin-schema-test-'; + +const noopLogger: Logger = () => {}; + +export async function findFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + + server.on('error', reject); + server.listen(0, () => { + const { port } = server.address() as net.AddressInfo; + + server.close(() => resolve(port)); + }); + }); +} + +function agentToken(): string { + return jsonwebtoken.sign( + { id: 1, email: 'forest@forest.com', renderingId: 1, team: 'admin' }, + AUTH_SECRET, + { expiresIn: '1 hour' }, + ); +} + +function schemaFetcherFromFile(schemaPath: string): SchemaFetcher { + return { + fetchSchema: async () => { + const { collections } = JSON.parse(await fs.readFile(schemaPath, 'utf8')) as { + collections: ForestSchemaCollection[]; + }; + + return collections; + }, + }; +} + +/** + * The BFF in front of a real agent: the middleware production mounts, the real data client, and a + * read-model built from the schema the agent just wrote. Only the schema transport is swapped — + * production fetches it from the Forest server, which plays no part in what these suites assert. + */ +export function buildApp(agentUrl: string, schemaPath: string): Koa { + const token = agentToken(); + const schemaCache = new SchemaCache({ + fetcher: schemaFetcherFromFile(schemaPath), + metrics: { increment: () => {}, gauge: () => {} }, + }); + const store = new ReadModelStore(schemaCache, new CapabilitiesCache()); + const app = new Koa(); + + app.silent = true; + app.use(createErrorMiddleware({ logger: noopLogger })); + app.use(bodyParser()); + app.use(async (ctx, next) => { + ctx.state.timezone = TIMEZONE; + ctx.state.agentToken = token; + await next(); + }); + app.use(createDataRoutesMiddleware({ store, agentUrl, logger: noopLogger })); + + return app; +} diff --git a/packages/agent-bff/test/data/fixtures/record-contract-datasource.ts b/packages/agent-bff/test/data/fixtures/record-contract-datasource.ts new file mode 100644 index 0000000000..5962050e69 --- /dev/null +++ b/packages/agent-bff/test/data/fixtures/record-contract-datasource.ts @@ -0,0 +1,27 @@ +import type { FieldSchema, RecordData } from '@forestadmin/datasource-toolkit'; + +import { BaseDataSource } from '@forestadmin/datasource-toolkit'; + +import InMemoryCollection from './in-memory-collection'; + +const PEOPLE: RecordData[] = [{ id: 8, first_name: 'Ada' }]; + +const NUMBER_PK: FieldSchema = { type: 'Column', columnType: 'Number', isPrimaryKey: true }; + +export default class RecordContractDataSource extends BaseDataSource { + constructor() { + super(); + + this.addCollection( + new InMemoryCollection( + this, + 'people', + { + id: NUMBER_PK, + first_name: { type: 'Column', columnType: 'String' }, + }, + PEOPLE, + ), + ); + } +} diff --git a/packages/agent-bff/test/data/record-contract.integration.test.ts b/packages/agent-bff/test/data/record-contract.integration.test.ts new file mode 100644 index 0000000000..69c92faaf9 --- /dev/null +++ b/packages/agent-bff/test/data/record-contract.integration.test.ts @@ -0,0 +1,74 @@ +import type { TestableAgent } from '@forestadmin/agent-testing'; +import type Koa from 'koa'; + +import { createTestableAgent } from '@forestadmin/agent-testing'; +import os from 'os'; +import path from 'path'; +import request from 'supertest'; + +import { + AUTH_SECRET, + BOOT_TIMEOUT_MS, + ENV_SECRET, + RESERVED_SCHEMA_PREFIX, + buildApp, + findFreePort, +} from './fixtures/live-agent-harness'; +import RecordContractDataSource from './fixtures/record-contract-datasource'; + +describe('the record contract against a real agent', () => { + let agent: TestableAgent; + let app: Koa; + + beforeAll(async () => { + const port = await findFreePort(); + const schemaPath = path.join( + os.tmpdir(), + `${RESERVED_SCHEMA_PREFIX}-bff-record-contract-${port}.json`, + ); + + agent = await createTestableAgent( + forestAgent => { + forestAgent.addDataSource(async () => new RecordContractDataSource()); + }, + { authSecret: AUTH_SECRET, envSecret: ENV_SECRET, isProduction: false, port, schemaPath }, + ); + + await agent.start(); + + app = buildApp(`http://localhost:${port}`, schemaPath); + }, BOOT_TIMEOUT_MS); + + afterAll(async () => { + await agent?.stop(); + }); + + it('should carry the flat id as a string while __forest.primaryKey holds it typed', async () => { + const response = await request(app.callback()) + .post('/agent/v1/people/list') + .send({ projection: ['id'] }); + + expect(response.status).toBe(200); + expect(response.body.data).toEqual([ + { + id: '8', + __forest: { collection: 'people', primaryKey: { id: 8 } }, + }, + ]); + }); + + it('should return a snake_case column under its camelCase key, while projecting its schema name', async () => { + const response = await request(app.callback()) + .post('/agent/v1/people/list') + .send({ projection: ['id', 'first_name'] }); + + expect(response.status).toBe(200); + expect(response.body.data).toEqual([ + { + id: '8', + firstName: 'Ada', + __forest: { collection: 'people', primaryKey: { id: 8 } }, + }, + ]); + }); +}); diff --git a/packages/agent-bff/test/data/search-agent.integration.test.ts b/packages/agent-bff/test/data/search-agent.integration.test.ts index d54caf5fe1..aebd268dae 100644 --- a/packages/agent-bff/test/data/search-agent.integration.test.ts +++ b/packages/agent-bff/test/data/search-agent.integration.test.ts @@ -1,95 +1,20 @@ -import type { Logger } from '../../src/ports/logger-port'; -import type { SchemaFetcher } from '../../src/read-model/forest-schema-client'; import type { TestableAgent } from '@forestadmin/agent-testing'; -import type { ForestSchemaCollection } from '@forestadmin/forestadmin-client'; +import type Koa from 'koa'; import { createTestableAgent } from '@forestadmin/agent-testing'; -import { bodyParser } from '@koa/bodyparser'; -import fs from 'fs/promises'; -import jsonwebtoken from 'jsonwebtoken'; -import Koa from 'koa'; -import net from 'net'; import os from 'os'; import path from 'path'; import request from 'supertest'; +import { + AUTH_SECRET, + BOOT_TIMEOUT_MS, + ENV_SECRET, + RESERVED_SCHEMA_PREFIX, + buildApp, + findFreePort, +} from './fixtures/live-agent-harness'; import SearchDataSource from './fixtures/search-datasource'; -import createDataRoutesMiddleware from '../../src/data/data-routes-middleware'; -import createErrorMiddleware from '../../src/http/error-middleware'; -import CapabilitiesCache from '../../src/read-model/capabilities-cache'; -import ReadModelStore from '../../src/read-model/read-model-store'; -import SchemaCache from '../../src/read-model/schema-cache'; - -const TIMEZONE = 'Europe/Paris'; -const AUTH_SECRET = 'b0bdf0a639c16bae8851dd24ee3d79ef0a352e957c5b86cb'; -const ENV_SECRET = 'ceba742f5bc73946b34da192816a4d7177b3233fee7769955c29c0e90fd584f2'; -const BOOT_TIMEOUT_MS = 60_000; - -// agent-testing only deletes a schema file whose name carries this prefix, so reusing it keeps the -// temporary schema cleaned up by `agent.stop()` even though the path is chosen here. -const RESERVED_SCHEMA_PREFIX = 'reserved-forestadmin-schema-test-'; - -const noopLogger: Logger = () => {}; - -async function findFreePort(): Promise { - return new Promise((resolve, reject) => { - const server = net.createServer(); - - server.on('error', reject); - server.listen(0, () => { - const { port } = server.address() as net.AddressInfo; - - server.close(() => resolve(port)); - }); - }); -} - -function agentToken(): string { - return jsonwebtoken.sign( - { id: 1, email: 'forest@forest.com', renderingId: 1, team: 'admin' }, - AUTH_SECRET, - { expiresIn: '1 hour' }, - ); -} - -function schemaFetcherFromFile(schemaPath: string): SchemaFetcher { - return { - fetchSchema: async () => { - const { collections } = JSON.parse(await fs.readFile(schemaPath, 'utf8')) as { - collections: ForestSchemaCollection[]; - }; - - return collections; - }, - }; -} - -/** - * The BFF in front of a real agent: the middleware production mounts, the real data client, and a - * read-model built from the schema the agent just wrote. Only the schema transport is swapped — - * production fetches it from the Forest server, which plays no part in search. - */ -function buildApp(agentUrl: string, schemaPath: string): Koa { - const token = agentToken(); - const schemaCache = new SchemaCache({ - fetcher: schemaFetcherFromFile(schemaPath), - metrics: { increment: () => {}, gauge: () => {} }, - }); - const store = new ReadModelStore(schemaCache, new CapabilitiesCache()); - const app = new Koa(); - - app.silent = true; - app.use(createErrorMiddleware({ logger: noopLogger })); - app.use(bodyParser()); - app.use(async (ctx, next) => { - ctx.state.timezone = TIMEZONE; - ctx.state.agentToken = token; - await next(); - }); - app.use(createDataRoutesMiddleware({ store, agentUrl, logger: noopLogger })); - - return app; -} describe('search against a real agent', () => { let agent: TestableAgent; diff --git a/packages/agent-bff/test/openapi/collect-unfolding.test.ts b/packages/agent-bff/test/openapi/collect-unfolding.test.ts index 1327f63752..42375f3415 100644 --- a/packages/agent-bff/test/openapi/collect-unfolding.test.ts +++ b/packages/agent-bff/test/openapi/collect-unfolding.test.ts @@ -92,7 +92,10 @@ describe('collectUnfolding', () => { const { collections } = await collect(readModel); expect(collections[0].fields).toEqual({ - projectable: ['id', 'author'], + projectable: [ + { name: 'id', type: 'Number' }, + { name: 'author', type: 'ManyToOne' }, + ], filterable: [{ name: 'id', operators: ['Equal'] }], degraded: null, }); @@ -159,7 +162,7 @@ describe('collectUnfolding', () => { }), }); - expect(collections[0].fields.projectable).toEqual(['id']); + expect(collections[0].fields.projectable).toEqual([{ name: 'id', type: 'String' }]); expect(collections[0].fields.filterable).toEqual([]); }); diff --git a/packages/agent-bff/test/openapi/fixtures.ts b/packages/agent-bff/test/openapi/fixtures.ts index 5151a174c0..2f76d0f887 100644 --- a/packages/agent-bff/test/openapi/fixtures.ts +++ b/packages/agent-bff/test/openapi/fixtures.ts @@ -4,8 +4,8 @@ import type { Unfolding } from '../../src/openapi/unfolding'; * One snapshot exercising every shape the unfolding has to survive: a name carrying a space, a * dotted name, an action name carrying a slash, two action names that collapse to the same * identifier, a ManyToOne that is projectable but not filterable, two fields of DIFFERENT types - * sharing one operator set, a composite key, a parent with no key metadata, and a collection whose - * capabilities could not be read. + * sharing one operator set, a snake_case field the response returns camelCased, a composite key, a + * parent with no key metadata, and a collection whose capabilities could not be read. */ export default function unfoldingFixture(): Unfolding { return { @@ -13,7 +13,13 @@ export default function unfoldingFixture(): Unfolding { { name: 'My Coll', fields: { - projectable: ['id', 'email', 'tags', 'author'], + projectable: [ + { name: 'id', type: 'Number' }, + { name: 'email', type: 'String' }, + { name: 'tags', type: ['String'] }, + { name: 'author', type: 'ManyToOne' }, + { name: 'created_at', type: 'Date' }, + ], filterable: [ { name: 'id', operators: ['Equal', 'NotEqual', 'In'] }, { name: 'email', operators: ['Equal', 'NotEqual', 'In', 'Contains'] }, @@ -49,7 +55,7 @@ export default function unfoldingFixture(): Unfolding { { name: 'users.address', fields: { - projectable: ['street'], + projectable: [{ name: 'street', type: 'String' }], filterable: [{ name: 'street', operators: ['Equal'] }], degraded: null, }, diff --git a/packages/agent-bff/test/openapi/openapi-unfolded.test.ts b/packages/agent-bff/test/openapi/openapi-unfolded.test.ts index 1c9de502d1..820fdda5d4 100644 --- a/packages/agent-bff/test/openapi/openapi-unfolded.test.ts +++ b/packages/agent-bff/test/openapi/openapi-unfolded.test.ts @@ -38,6 +38,14 @@ function requestSchema(path: string): Record { return schemas[requestSchemaName(path)] as Record; } +function responseRef(path: string): string { + const { responses } = operation(path) as { + responses: Record }>; + }; + + return responses['200'].content['application/json'].schema.$ref; +} + function dereference(ref: string): Record { return schemas[ref.replace('#/components/schemas/', '')]; } @@ -74,7 +82,11 @@ function collectionOf( ): UnfoldedCollection { return { name, - fields: { projectable: filterable.map(field => field.name), filterable, degraded }, + fields: { + projectable: filterable.map(field => ({ name: field.name, type: 'String' as const })), + filterable, + degraded, + }, primaryKeys: [{ name: 'id', type: 'Number' }], relations: [], actions: [], @@ -151,7 +163,10 @@ describe('the unfolded document', () => { it('should enumerate the projectable fields of the collection', () => { expect(schemas.Fields_My_Coll).toEqual( - expect.objectContaining({ type: 'string', enum: ['id', 'email', 'tags', 'author'] }), + expect.objectContaining({ + type: 'string', + enum: ['id', 'email', 'tags', 'author', 'created_at'], + }), ); }); @@ -371,6 +386,124 @@ describe('the unfolded document', () => { }); }); +describe('the per-collection record schema', () => { + const recordOf = (name: string) => + schemas[name] as unknown as { + properties: Record; + required: string[]; + additionalProperties?: unknown; + }; + + it('should type every property from the column type the capabilities report', () => { + const { properties } = recordOf('Record_My_Coll'); + + expect(properties.email).toEqual({ type: ['string', 'null'] }); + expect(properties.tags).toEqual({ + type: ['array', 'null'], + items: { type: ['string', 'null'] }, + }); + expect(properties.author).toEqual({}); + }); + + it('should publish a field under the key the response carries, not its schema name', () => { + const { properties } = recordOf('Record_My_Coll'); + + expect(properties.createdAt).toEqual({ + type: ['string', 'null'], + format: 'date-time', + description: 'The "created_at" field.', + }); + expect(properties.created_at).toBeUndefined(); + }); + + it('should force id to a string even when the key column is a Number', () => { + const { properties } = recordOf('Record_My_Coll'); + + expect((properties.id as { type: string }).type).toBe('string'); + expect((properties.id as { description: string }).description).toContain('always a string'); + }); + + it('should require only id and __forest, the keys a record carries whatever the projection', () => { + expect(recordOf('Record_My_Coll').required).toEqual(['id', '__forest']); + }); + + it('should reference ForestRecordMeta from __forest', () => { + expect(recordOf('Record_My_Coll').properties).toEqual( + expect.objectContaining({ __forest: { $ref: '#/components/schemas/ForestRecordMeta' } }), + ); + }); + + it('should keep the record open, since the capabilities do not report every projected key', () => { + expect(recordOf('Record_My_Coll').additionalProperties).toBeUndefined(); + }); + + it('should sanitize a dotted collection name into the record component name', () => { + expect(recordOf('Record_users_address').properties.street).toEqual({ + type: ['string', 'null'], + }); + }); + + it('should answer a collection list with its own response schema over its own record', () => { + expect(responseRef('My%20Coll/list')).toBe('#/components/schemas/ListResponse_My_Coll'); + expect( + (schemas.ListResponse_My_Coll as { properties: { data: unknown } }).properties.data, + ).toEqual({ type: 'array', items: { $ref: '#/components/schemas/Record_My_Coll' } }); + }); + + it('should answer a relation list with the FOREIGN collection response schema', () => { + expect(responseRef('orders/relations/buyers/list')).toBe( + '#/components/schemas/ListResponse_My_Coll', + ); + }); + + it('should keep the shared untyped ListResponse for a degraded collection own list', () => { + expect(responseRef('orders/list')).toBe('#/components/schemas/ListResponse'); + expect(schemas.Record_orders).toBeUndefined(); + expect(schemas.ListResponse_orders).toBeUndefined(); + }); + + it('should keep the shared untyped ListResponse for a relation pointing at a degraded foreign', () => { + expect(responseRef('My%20Coll/relations/orders/list')).toBe( + '#/components/schemas/ListResponse', + ); + }); +}); + +describe('two fields whose response key collides', () => { + const colliding = unfoldedDocument({ + collections: [ + { + name: 'twins', + fields: { + projectable: [ + { name: 'first_name', type: 'String' }, + { name: 'firstName', type: 'Number' }, + ], + filterable: [], + degraded: null, + }, + primaryKeys: [{ name: 'first_name', type: 'String' }], + relations: [], + actions: [], + }, + ], + }); + const record = (colliding.components?.schemas as Record) + .Record_twins as unknown as { properties: Record }; + + it('should publish one unconstrained property naming the fields that collapse onto it', () => { + expect(record.properties.firstName).toEqual({ + description: + '"first_name", "firstName" all reach the response under this single key, so which one it ' + + 'holds is not determined here. Project one of them at a time to know.', + }); + }); + + it('should emit the collided key once, plus id and __forest, and nothing else', () => { + expect(Object.keys(record.properties)).toEqual(['firstName', 'id', '__forest']); + }); +}); + describe('a collection whose key ends in an index-like suffix', () => { // `sanitizeIdentifier` turns anything outside [A-Za-z0-9_] into `_`, and `createNamer` appends // `_` to a collapsed name, so a collection key can end in `_` on its own. A leaf namespace @@ -431,7 +564,7 @@ describe('an unfolding naming a collection it does not carry', () => { { name: 'users', fields: { - projectable: ['id'], + projectable: [{ name: 'id', type: 'Number' }], filterable: [{ name: 'id', operators: ['Equal'] }], degraded: null, }, @@ -462,7 +595,7 @@ describe('names that collide once sanitized', () => { ): UnfoldedCollection => ({ name, fields: { - projectable: ['id'], + projectable: [{ name: 'id', type: 'Number' }], filterable: [{ name: 'id', operators: ['Equal'] }], degraded: null, }, @@ -511,7 +644,7 @@ describe('an unfolded document with no action', () => { { name: 'users', fields: { - projectable: ['id'], + projectable: [{ name: 'id', type: 'Number' }], filterable: [{ name: 'id', operators: ['Equal'] }], degraded: null, }, diff --git a/packages/agent-bff/test/openapi/record-schemas.test.ts b/packages/agent-bff/test/openapi/record-schemas.test.ts new file mode 100644 index 0000000000..2361a369c9 --- /dev/null +++ b/packages/agent-bff/test/openapi/record-schemas.test.ts @@ -0,0 +1,97 @@ +import type { UnfoldedCollection } from '../../src/openapi/unfolding'; + +import recordSchema, { recordKey } from '../../src/openapi/record-schemas'; + +const PEOPLE: UnfoldedCollection = { + name: 'people', + fields: { + projectable: [{ name: 'profile_file', type: 'File' }], + filterable: [], + degraded: null, + }, + primaryKeys: [{ name: 'id', type: 'Number' }], + relations: [], + actions: [], +}; + +describe('recordSchema', () => { + it('should keep the File data-URI contract when the wire key differs from the schema name', () => { + const schema = recordSchema(PEOPLE, { + $ref: '#/components/schemas/ForestRecordMeta', + }) as { properties: Record }; + + expect(schema.properties.profileFile).toEqual({ + type: ['string', 'null'], + description: 'A data URI. The "profile_file" field.', + }); + }); + + it('should publish every field as nullable, since capabilities never report nullability', () => { + const schema = recordSchema( + { + ...PEOPLE, + fields: { + ...PEOPLE.fields, + projectable: [ + { name: 'first_name', type: 'String' }, + { name: 'age', type: 'Number' }, + { name: 'payload', type: 'Json' }, + ], + }, + }, + { $ref: '#/components/schemas/ForestRecordMeta' }, + ) as { properties: Record }; + + expect(schema.properties.firstName.type).toEqual(['string', 'null']); + expect(schema.properties.age.type).toEqual(['number', 'null']); + expect(schema.properties.payload.type).toBeUndefined(); + }); + + it('should publish array items and nested properties as nullable too, not just the container', () => { + const schema = recordSchema( + { + ...PEOPLE, + fields: { + ...PEOPLE.fields, + projectable: [ + { name: 'tags', type: ['String'] }, + { + name: 'address', + type: { fields: [{ field: 'city', type: 'String' }] }, + }, + ], + }, + }, + { $ref: '#/components/schemas/ForestRecordMeta' }, + ) as { properties: Record }; + + expect(schema.properties.tags).toEqual({ + type: ['array', 'null'], + items: { type: ['string', 'null'] }, + }); + expect(schema.properties.address).toEqual({ + type: ['object', 'null'], + properties: { city: { type: ['string', 'null'] } }, + }); + }); +}); + +describe('recordKey', () => { + it('should leave an already-clean name untouched', () => { + expect(recordKey('email')).toBe('email'); + }); + + it('should camelCase a snake_case name, since that is how the response carries it', () => { + expect(recordKey('created_at')).toBe('createdAt'); + }); + + it('should collapse every casing and separator variant onto one key, like the deserializer', () => { + ['first_name', 'firstName', 'FirstName', 'first-name', 'FIRST_NAME'].forEach(name => { + expect(recordKey(name)).toBe('firstName'); + }); + }); + + it('should camelize non-ASCII names too, which a hand-rolled mirror would likely miss', () => { + expect(recordKey('état_civil')).toBe('étatCivil'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 7ce2f62c84..335eaa7cbe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4630,6 +4630,11 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.4.tgz#7eb47726c391b7345a6ec35ad7f4de469cf5ba4f" integrity sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA== +"@types/inflected@^1.1.29": + version "1.1.29" + resolved "https://registry.yarnpkg.com/@types/inflected/-/inflected-1.1.29.tgz#8ef717dcf618d84584f506108ea85cd852f6d3ab" + integrity sha512-csq2i12fylUrVWQ15ZMnVV3IV/KJ6zti/bn/n1FSHgZfIw1OGZV2OaJLdpGb0e1SRwdg92yalOS3Wftuw59rFA== + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7"