From 0f58f69ec8c68ca4596b4fc4e556e9f865777c6d Mon Sep 17 00:00:00 2001 From: ied <11755605+izzetemredemir@users.noreply.github.com> Date: Sat, 17 Oct 2026 04:22:05 +0300 Subject: [PATCH] feat(deployments): generate the subgraph manifest from the deployment record evm/subgraph/subgraph.yaml still carried 0x...dead placeholder addresses for WraithNames, WraithSender and WraithWithdrawer, and both checks let them through: the manifest consumer skipped placeholders for contracts missing from the manifest, and validate-config.js only reads the instant configs. subgraph.yaml becomes subgraph.template.yaml, without network, addresses or start blocks. `npm run subgraph -- --network ` in scripts/deployment-manifest (`npm run manifest` in evm/subgraph) fills them in from deployments/evm/.json and writes subgraph.yaml, which is git-ignored. It writes nothing and names every offending contract if a data source's contract is not recorded, has no start block or start block 0, or has the placeholder address. `--check` fails unless an existing subgraph.yaml is exactly what the generator writes now for its network; `deploy` and `deploy-local` run it before graph deploy. codegen, build and test read the template (matchstick.yaml sets manifestPath), so CI needs no deployment. The manifest check validates the template instead of subgraph.yaml: every data source must name an EVM contract and must not set network, address or startBlock. horizen-testnet records only the Announcer and Registry, so generation for it fails, naming the other three, until they are recorded. --- .prettierignore | 2 + deployments/README.md | 35 ++- evm/README.md | 18 +- evm/scripts/deploy.ts | 7 +- evm/subgraph/.gitignore | 2 + evm/subgraph/README.md | 36 +++ evm/subgraph/matchstick.yaml | 2 + evm/subgraph/package.json | 9 +- .../{subgraph.yaml => subgraph.template.yaml} | 27 +-- scripts/deployment-manifest/package.json | 1 + scripts/deployment-manifest/src/cli.ts | 58 ++++- scripts/deployment-manifest/src/consumers.ts | 54 +---- scripts/deployment-manifest/src/manifest.ts | 44 ++-- scripts/deployment-manifest/src/subgraph.ts | 179 ++++++++++++++ .../test/check.e2e.test.ts | 223 +++++++++++++++++- 15 files changed, 587 insertions(+), 110 deletions(-) create mode 100644 evm/subgraph/matchstick.yaml rename evm/subgraph/{subgraph.yaml => subgraph.template.yaml} (80%) create mode 100644 scripts/deployment-manifest/src/subgraph.ts diff --git a/.prettierignore b/.prettierignore index d83aecc..26f8e7c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -12,3 +12,5 @@ stellar **/subgraph/build **/subgraph/tests/.bin **/subgraph/tests/.latest.json +# Generated from subgraph.template.yaml by `npm run manifest`. +**/subgraph/subgraph.yaml diff --git a/deployments/README.md b/deployments/README.md index 4446260..4d54dc5 100644 --- a/deployments/README.md +++ b/deployments/README.md @@ -79,7 +79,7 @@ must carry a source commit. | File | Checked | Generated by `build` | | ---------------------------------------------------------------- | ----------------------------------------------------------------------------- | -------------------- | -| `evm/subgraph/subgraph.yaml` | address and `startBlock` per data source; undeployed contracts stay `0x…dead` | no | +| `evm/subgraph/subgraph.template.yaml` | each `source.abi` is an EVM contract; no `network`, `address` or `startBlock` | no | | `evm/subgraph//instant-config.json` | address, chain, `startBlock` | yes | | `evm/subgraph/abis/*.json`, `evm/subgraph//abis/*.json` | canonical ABI hash | no | | `evm/hardhat.config.ts` | `chainId` of the matching network | no | @@ -98,7 +98,8 @@ npm ci ``` - **EVM**: `evm/scripts/deploy.ts` records automatically on any non-local network, using - each contract's deployment receipt. Then update `evm/subgraph/subgraph.yaml` to match. + each contract's deployment receipt. Then regenerate the subgraph manifest from the + record (see [Subgraph manifest](#subgraph-manifest)). - **Stellar**: `stellar/deploy.sh` records automatically, before `init`. - **Solana, CKB**, or anything deployed by hand: record from the chain. @@ -121,11 +122,29 @@ record for that network, and runs `build`. `--rpc` is only used for queries, so may contain an API key; only `--public-rpc` is written to the record. Commit the record, `manifest.json` and every file `build` regenerated. +## Subgraph manifest + +`evm/subgraph/subgraph.yaml` is not checked in. It is generated for one network from +`evm/subgraph/subgraph.template.yaml` and `deployments/evm/.json`: + +```bash +npm run subgraph -- --network horizen-testnet +``` + +Each data source gets `network`, `source.address` and `source.startBlock` from the +contract its `source.abi` names. If any contract is not recorded, has no start block +(or start block 0), or has the placeholder address `0x…dead`, nothing is written and +every such contract is listed. `npm run subgraph -- --check` fails unless the +existing `subgraph.yaml` is exactly what `--network` would write now for the network +it names; `npm run deploy` and `npm run deploy-local` in `evm/subgraph` run it before +`graph deploy`. + ## Commands -| Command | What it does | -| ---------------- | ----------------------------------------------------------------------------------------------- | -| `npm run build` | Validate records, write `manifest.json`, regenerate the generated consumers. | -| `npm run check` | Offline validation of records, manifest and consumers. Runs in CI. | -| `npm run verify` | Re-read every deployment from its chain and compare IDs, heights and hashes. Runs weekly in CI. | -| `npm test` | Unit and end-to-end tests for the tool. | +| Command | What it does | +| ------------------ | ----------------------------------------------------------------------------------------------- | +| `npm run build` | Validate records, write `manifest.json`, regenerate the generated consumers. | +| `npm run check` | Offline validation of records, manifest and consumers. Runs in CI. | +| `npm run verify` | Re-read every deployment from its chain and compare IDs, heights and hashes. Runs weekly in CI. | +| `npm run subgraph` | Write `evm/subgraph/subgraph.yaml` for `--network `; `--check` verifies it. | +| `npm test` | Unit and end-to-end tests for the tool. | diff --git a/evm/README.md b/evm/README.md index ae49184..ebfe5f5 100644 --- a/evm/README.md +++ b/evm/README.md @@ -78,20 +78,22 @@ slither . --config-file slither.config.json --fail-medium ## Subgraph `subgraph/` indexes all five contracts: event handlers for Announcer, Registry, -and Names, plus call handlers for Sender and Withdrawer. The manifest targets -Horizen Testnet; fill in the three `0x…dead` placeholder addresses (Names, -Sender, Withdrawer) from `scripts/deploy.ts` output before deploying. +and Names, plus call handlers for Sender and Withdrawer. The checked-in +`subgraph.template.yaml` has no network, addresses or start blocks; +`npm run manifest` generates `subgraph.yaml` from `deployments/evm/.json` +and refuses if a contract or start block is not recorded (see `subgraph/README.md`). ```bash cd subgraph npm ci -npm run codegen # graph codegen -npm run build # graph build +npm run codegen # graph codegen, from the template +npm run build # graph build, from the template ``` Deploy to a local graph-node: ```bash +npm run manifest -- --network npm run create-local npm run deploy-local ``` @@ -103,8 +105,10 @@ npx hardhat run scripts/deploy.ts --network ``` The script deploys Announcer, Registry, Sender (pointing at the Announcer), -Names, and Withdrawer, logging each address. Point the subgraph at the deployed -addresses and redeploy. +Names, and Withdrawer, logging each address. On any non-local network it records +them in `deployments/evm/.json`. Regenerate the subgraph manifest from +that record (`npm run manifest -- --network ` in `subgraph/`) and +redeploy. ## CI diff --git a/evm/scripts/deploy.ts b/evm/scripts/deploy.ts index 926462a..d4082b6 100644 --- a/evm/scripts/deploy.ts +++ b/evm/scripts/deploy.ts @@ -37,6 +37,7 @@ async function main() { // runtime bytecode, then rebuilds deployments/manifest.json and the subgraph // instant config. See deployments/README.md. const network = hre.network.name; + const slug = network.replace(/_/g, '-'); if (network === 'hardhat' || network === 'localhost') { console.log('\nLocal network: not recording to deployments/.'); } else { @@ -51,7 +52,7 @@ async function main() { 'record', 'evm', '--network', - network.replace(/_/g, '-'), + slug, '--rpc', hre.network.config.url, '--recorded-by', @@ -64,7 +65,9 @@ async function main() { ], { cwd: path.join(__dirname, '../../scripts/deployment-manifest'), stdio: 'inherit' }, ); - console.log('\nUpdate evm/subgraph/subgraph.yaml to match, then run the manifest check.'); + console.log( + `\nGenerate the subgraph manifest from the record (from evm/): cd subgraph && npm run manifest -- --network ${slug}`, + ); } console.log('\nDeployment summary:'); diff --git a/evm/subgraph/.gitignore b/evm/subgraph/.gitignore index 64b7e1d..1889203 100644 --- a/evm/subgraph/.gitignore +++ b/evm/subgraph/.gitignore @@ -3,3 +3,5 @@ generated/ build/ tests/.bin/ tests/.latest.json +# Generated per network from subgraph.template.yaml (npm run manifest). +/subgraph.yaml diff --git a/evm/subgraph/README.md b/evm/subgraph/README.md index 4f7bc40..27cefdc 100644 --- a/evm/subgraph/README.md +++ b/evm/subgraph/README.md @@ -1,11 +1,15 @@ # Subgraph Deployments Each subdirectory contains the Goldsky instant subgraph config for one EVM network. +The `graph-cli` manifest is generated per network from `subgraph.template.yaml`; see +[graph-cli manifest](#graph-cli-manifest). ## Structure ``` subgraph/ + subgraph.template.yaml ← graph-cli manifest without network, addresses or start blocks + subgraph.yaml ← generated from the template for one network (git-ignored) horizen-testnet/ ← Horizen Testnet instant-config.json abis/ @@ -66,6 +70,9 @@ cd evm/subgraph/ goldsky subgraph deploy wraith-protocol-/1.0.0 --from-abi instant-config.json ``` +To deploy with `graph deploy` instead, generate the manifest first (see +[graph-cli manifest](#graph-cli-manifest)). + ### 4. Adding a New Network To add support for a new EVM network: @@ -100,6 +107,32 @@ To add support for a new EVM network: 6. Deploy contracts using the new script 7. Validate and deploy the subgraph +## graph-cli manifest + +`subgraph.template.yaml` is the checked-in `graph-cli` manifest. It has no network, +contract addresses or start blocks. `npm run manifest` fills them in from +`deployments/evm/.json` and writes `subgraph.yaml`, which is git-ignored: + +```bash +(cd scripts/deployment-manifest && npm ci) # once, from the repository root; the generator lives here +cd evm/subgraph +npm ci +npm run manifest -- --network +npm run codegen +npm run deploy -- --node # any graph deploy arguments +``` + +`npm run manifest` fails and writes nothing if a contract the template indexes is +not in the record, has no start block (or start block 0), or has the placeholder +address `0x…dead`. It lists every such contract. `npm run deploy` and +`npm run deploy-local` first run `npm run manifest -- --check`, which fails unless +`subgraph.yaml` exists and is exactly what the generator writes now for the network +it names, so these scripts do not deploy a stale or hand-edited manifest. A bare +`graph deploy` skips that check; use the npm scripts. + +`codegen`, `build` and `test` read the template directly (`matchstick.yaml` points +`graph test` at it), so they work without a deployment. + ## Deployed Subgraphs | Network | Subgraph URL | @@ -114,3 +147,6 @@ To add support for a new EVM network: - **Not yet deployed**: WraithNames, WraithSender, WraithWithdrawer The subgraph currently only indexes the deployed contracts. To add support for WraithNames, WraithSender, and WraithWithdrawer, deploy these contracts to Horizen Testnet and update the subgraph configuration following the redeployment process above. + +Until they are recorded in `deployments/evm/horizen-testnet.json`, +`npm run manifest -- --network horizen-testnet` fails and names the three of them. diff --git a/evm/subgraph/matchstick.yaml b/evm/subgraph/matchstick.yaml new file mode 100644 index 0000000..ba1f9a7 --- /dev/null +++ b/evm/subgraph/matchstick.yaml @@ -0,0 +1,2 @@ +# graph test reads the template; the tests do not depend on a network. +manifestPath: ./subgraph.template.yaml diff --git a/evm/subgraph/package.json b/evm/subgraph/package.json index 909f7ca..018e5c9 100644 --- a/evm/subgraph/package.json +++ b/evm/subgraph/package.json @@ -3,13 +3,14 @@ "version": "0.1.0", "private": true, "scripts": { - "codegen": "graph codegen", - "build": "graph build", + "manifest": "npm --prefix ../../scripts/deployment-manifest run --silent subgraph --", + "codegen": "graph codegen subgraph.template.yaml", + "build": "graph build subgraph.template.yaml", "test": "graph test --version 0.6.0", "test:coverage": "graph test --coverage --version 0.6.0", - "deploy": "graph deploy", + "deploy": "npm run manifest -- --check && graph deploy", "create-local": "graph create --node http://localhost:8020 wraith-protocol/wraith", - "deploy-local": "graph deploy --node http://localhost:8020 --ipfs http://localhost:5001 wraith-protocol/wraith" + "deploy-local": "npm run manifest -- --check && graph deploy --node http://localhost:8020 --ipfs http://localhost:5001 wraith-protocol/wraith" }, "devDependencies": { "@graphprotocol/graph-cli": "^0.96.0", diff --git a/evm/subgraph/subgraph.yaml b/evm/subgraph/subgraph.template.yaml similarity index 80% rename from evm/subgraph/subgraph.yaml rename to evm/subgraph/subgraph.template.yaml index a17025e..a83f297 100644 --- a/evm/subgraph/subgraph.yaml +++ b/evm/subgraph/subgraph.template.yaml @@ -1,3 +1,12 @@ +# Template for subgraph.yaml. It has no network, contract addresses or start +# blocks, so it cannot be deployed as is. scripts/deployment-manifest fills them +# in from deployments/evm/.json: +# +# npm run manifest -- --network +# +# and refuses if a contract or start block is not recorded. codegen, build and +# test read this file directly. See README.md. + specVersion: 1.0.0 description: Wraith Protocol stealth address indexer repository: https://github.com/wraith-protocol/contracts @@ -7,11 +16,8 @@ schema: dataSources: - kind: ethereum name: ERC5564Announcer - network: horizen-testnet source: - address: '0x8AE65c05E7eb48B9bA652781Bc0a3DBA09A484F3' abi: ERC5564Announcer - startBlock: 14202900 mapping: kind: ethereum/events apiVersion: 0.0.7 @@ -28,11 +34,8 @@ dataSources: - kind: ethereum name: ERC6538Registry - network: horizen-testnet source: - address: '0x953E6cEdcdfAe321796e7637d33653F6Ce05c527' abi: ERC6538Registry - startBlock: 14202905 mapping: kind: ethereum/events apiVersion: 0.0.7 @@ -51,12 +54,8 @@ dataSources: - kind: ethereum name: WraithNames - network: horizen-testnet source: - # TODO: fill in after deployment — matches scripts/deploy.ts output. - address: '0x000000000000000000000000000000000000dead' abi: WraithNames - startBlock: 14202900 mapping: kind: ethereum/events apiVersion: 0.0.7 @@ -75,12 +74,8 @@ dataSources: - kind: ethereum name: WraithSender - network: horizen-testnet source: - # TODO: fill in after deployment — matches scripts/deploy.ts output. - address: '0x000000000000000000000000000000000000dead' abi: WraithSender - startBlock: 14202900 mapping: kind: ethereum/events apiVersion: 0.0.7 @@ -103,12 +98,8 @@ dataSources: - kind: ethereum name: WraithWithdrawer - network: horizen-testnet source: - # TODO: fill in after deployment — matches scripts/deploy.ts output. - address: '0x000000000000000000000000000000000000dead' abi: WraithWithdrawer - startBlock: 14202900 mapping: kind: ethereum/events apiVersion: 0.0.7 diff --git a/scripts/deployment-manifest/package.json b/scripts/deployment-manifest/package.json index 2fcca21..4e4ddcc 100644 --- a/scripts/deployment-manifest/package.json +++ b/scripts/deployment-manifest/package.json @@ -12,6 +12,7 @@ "check": "tsx src/cli.ts check", "verify": "tsx src/cli.ts verify", "record": "tsx src/cli.ts record", + "subgraph": "tsx src/cli.ts subgraph", "typecheck": "tsc --noEmit", "test": "vitest run" }, diff --git a/scripts/deployment-manifest/src/cli.ts b/scripts/deployment-manifest/src/cli.ts index df7d051..05429b1 100644 --- a/scripts/deployment-manifest/src/cli.ts +++ b/scripts/deployment-manifest/src/cli.ts @@ -14,7 +14,7 @@ import { type Contract, type Deployment, } from './manifest.js'; -import { MANIFEST_PATH, REPO_ROOT, recordPath } from './paths.js'; +import { MANIFEST_PATH, REPO_ROOT, recordPath, repoPath } from './paths.js'; import { ckbNetworkId, recordCkbScript } from './record/ckb.js'; import { evmNetworkId, recordEvmContract } from './record/evm.js'; import { recordSolanaProgram, solanaNetworkId } from './record/solana.js'; @@ -24,6 +24,12 @@ import { stellarNetworkId, wasmHashFromInstanceEntry, } from './record/stellar.js'; +import { + generateSubgraph, + generatedNetwork, + SUBGRAPH_MANIFEST, + SUBGRAPH_TEMPLATE, +} from './subgraph.js'; const USAGE = `Usage: tsx src/cli.ts @@ -34,6 +40,13 @@ const USAGE = `Usage: tsx src/cli.ts consumer disagrees with it. Offline; this is what CI runs. verify Re-read every deployment from its chain and compare IDs, heights and artifact hashes. Needs network access. + subgraph --network + Write evm/subgraph/subgraph.yaml from evm/subgraph/subgraph.template.yaml + and deployments/evm/.json. Fails, naming each one, if a contract + the template indexes or its start block is not recorded. + subgraph --check + Fail unless evm/subgraph/subgraph.yaml is what --network would write now + for the network it names. The evm/subgraph deploy scripts run it first. record --network --rpc [targets...] Read deployments from the chain and write deployments//.json, merging with any existing record, then run build. @@ -109,6 +122,48 @@ function check(): void { ); } +function subgraph(argv: string[]): void { + const { values } = parseArgs({ + args: argv, + options: { network: { type: 'string' }, check: { type: 'boolean', default: false } }, + }); + if (values.check === Boolean(values.network)) { + throw new Error('subgraph takes either --network or --check'); + } + const out = repoPath(SUBGRAPH_MANIFEST); + + if (values.network) { + const { manifest, errors } = generateSubgraph(values.network); + if (manifest === null) { + console.error(`Cannot generate ${SUBGRAPH_MANIFEST} for ${values.network}:`); + fail(errors); + } + fs.writeFileSync(out, manifest); + console.log(`✓ ${SUBGRAPH_MANIFEST} for ${values.network}`); + return; + } + + if (!fs.existsSync(out)) { + fail([ + `${SUBGRAPH_MANIFEST} has not been generated (run \`npm run manifest -- --network \` in evm/subgraph)`, + ]); + } + const current = fs.readFileSync(out, 'utf8'); + const network = generatedNetwork(current); + if (!network) fail([`${SUBGRAPH_MANIFEST} does not name exactly one network; regenerate it`]); + const { manifest, errors } = generateSubgraph(network); + if (manifest === null) { + console.error(`${SUBGRAPH_MANIFEST} can no longer be generated for ${network}:`); + fail(errors); + } + if (manifest !== current) { + fail([ + `${SUBGRAPH_MANIFEST} differs from ${SUBGRAPH_TEMPLATE} and deployments/evm/${network}.json (run \`npm run manifest -- --network ${network}\`)`, + ]); + } + console.log(`✓ ${SUBGRAPH_MANIFEST} is current for ${network}`); +} + type TargetArgs = Partial< Record<'contract' | 'program' | 'script' | 'wasm' | 'abi' | 'deployer', string[]> >; @@ -315,6 +370,7 @@ async function main(): Promise { if (command === 'build') return build(); if (command === 'check') return check(); if (command === 'verify') return verify(); + if (command === 'subgraph') return subgraph(rest); if (command === 'record' && ['evm', 'stellar', 'solana', 'ckb'].includes(rest[0])) { return record(rest[0] as Chain, rest.slice(1)); } diff --git a/scripts/deployment-manifest/src/consumers.ts b/scripts/deployment-manifest/src/consumers.ts index cfec6d1..ff258af 100644 --- a/scripts/deployment-manifest/src/consumers.ts +++ b/scripts/deployment-manifest/src/consumers.ts @@ -1,10 +1,8 @@ import fs from 'node:fs'; import path from 'node:path'; import { parse as parseToml } from 'smol-toml'; -import { parse as parseYaml } from 'yaml'; import { CHAIN_LABEL, - EVM_PLACEHOLDER_ADDRESS, EXPECTED_CONTRACTS, HEIGHT_LABEL, ID_LABEL, @@ -13,6 +11,7 @@ import { import { canonicalAbiSha256 } from './hash.js'; import { CHAINS, findDeployment, serialize, type Deployment, type Manifest } from './manifest.js'; import { repoPath } from './paths.js'; +import { SUBGRAPH_TEMPLATE, templateErrors } from './subgraph.js'; /** * A checked-in file that repeats deployment data. `check` reports every @@ -33,8 +32,6 @@ function writeIfChanged(rel: string, content: string): boolean { return true; } -const sameAddress = (a: string, b: string) => a.toLowerCase() === b.toLowerCase(); - function listDirs(rel: string): string[] { if (!exists(rel)) return []; return fs @@ -46,46 +43,13 @@ function listDirs(rel: string): string[] { // ─── EVM ───────────────────────────────────────────────────────────────────── -const SUBGRAPH_YAML = 'evm/subgraph/subgraph.yaml'; - -const subgraphManifest: Consumer = { - name: SUBGRAPH_YAML, - check(manifest) { - if (!exists(SUBGRAPH_YAML)) return []; - const doc = parseYaml(read(SUBGRAPH_YAML)) as { - dataSources?: Array<{ - name: string; - network: string; - source: { address: string; abi: string; startBlock: number }; - }>; - }; - const errors: string[] = []; - for (const ds of doc.dataSources ?? []) { - const where = `${SUBGRAPH_YAML} dataSource ${ds.name}`; - const deployment = findDeployment(manifest, 'evm', ds.network); - if (!deployment) { - errors.push(`${where}: network ${ds.network} has no deployments/evm/${ds.network}.json`); - continue; - } - const contract = deployment.contracts.find((c) => c.name === ds.source.abi); - if (!contract) { - if (!sameAddress(ds.source.address, EVM_PLACEHOLDER_ADDRESS)) { - errors.push( - `${where}: ${ds.source.abi} is not in the manifest but has address ${ds.source.address}`, - ); - } - continue; - } - if (!sameAddress(ds.source.address, contract.id)) { - errors.push(`${where}: address ${ds.source.address}, manifest has ${contract.id}`); - } - if (ds.source.startBlock !== contract.deployment.height) { - errors.push( - `${where}: startBlock ${ds.source.startBlock}, manifest height is ${contract.deployment.height}`, - ); - } - } - return errors; +// The deployable evm/subgraph/subgraph.yaml is generated per network by +// `subgraph --network` and not checked in; only its template is. +const subgraphTemplate: Consumer = { + name: SUBGRAPH_TEMPLATE, + check() { + if (!exists(SUBGRAPH_TEMPLATE)) return [`${SUBGRAPH_TEMPLATE}: missing`]; + return templateErrors(read(SUBGRAPH_TEMPLATE)).map((e) => `${SUBGRAPH_TEMPLATE}: ${e}`); }, }; @@ -424,7 +388,7 @@ const readme: Consumer = { }; export const CONSUMERS: Consumer[] = [ - subgraphManifest, + subgraphTemplate, instantConfigs, evmAbis, hardhatNetworks, diff --git a/scripts/deployment-manifest/src/manifest.ts b/scripts/deployment-manifest/src/manifest.ts index d0b1760..cfe7fe7 100644 --- a/scripts/deployment-manifest/src/manifest.ts +++ b/scripts/deployment-manifest/src/manifest.ts @@ -100,6 +100,34 @@ export interface LoadedRecord { deployment: Deployment; } +/** + * Parse one record stored under deployments// and list everything wrong + * with it. `deployment` is returned whenever the file holds a JSON object, even + * if `errors` is not empty. + */ +export function readRecord( + chain: Chain, + file: string, +): { deployment?: Deployment; errors: string[] } { + const name = path.basename(file); + let deployment: Deployment; + try { + deployment = JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (err) { + return { errors: [`invalid JSON (${(err as Error).message})`] }; + } + if (typeof deployment !== 'object' || deployment === null || Array.isArray(deployment)) { + return { errors: ['not a JSON object'] }; + } + const errors = validateDeployment(deployment); + if (deployment.chain !== chain) + errors.push(`chain "${deployment.chain}" but stored under ${chain}/`); + if (`${deployment.network}.json` !== name) { + errors.push(`network "${deployment.network}" does not match file name ${name}`); + } + return { deployment, errors }; +} + /** Read deployments//.json, failing on anything malformed or misplaced. */ export function loadRecords(dir = DEPLOYMENTS_DIR): { records: LoadedRecord[]; errors: string[] } { const records: LoadedRecord[] = []; @@ -110,21 +138,9 @@ export function loadRecords(dir = DEPLOYMENTS_DIR): { records: LoadedRecord[]; e for (const name of fs.readdirSync(chainDir).sort()) { if (!name.endsWith('.json')) continue; const file = path.relative(REPO_ROOT, path.join(chainDir, name)); - let deployment: Deployment; - try { - deployment = JSON.parse(fs.readFileSync(path.join(chainDir, name), 'utf8')); - } catch (err) { - errors.push(`${file}: invalid JSON (${(err as Error).message})`); - continue; - } - const problems = validateDeployment(deployment); - if (deployment.chain !== chain) - problems.push(`chain "${deployment.chain}" but stored under ${chain}/`); - if (`${deployment.network}.json` !== name) { - problems.push(`network "${deployment.network}" does not match file name ${name}`); - } + const { deployment, errors: problems } = readRecord(chain, path.join(chainDir, name)); if (problems.length) errors.push(...problems.map((p) => `${file}: ${p}`)); - else records.push({ file, deployment }); + else records.push({ file, deployment: deployment! }); } } return { records, errors }; diff --git a/scripts/deployment-manifest/src/subgraph.ts b/scripts/deployment-manifest/src/subgraph.ts new file mode 100644 index 0000000..d16915c --- /dev/null +++ b/scripts/deployment-manifest/src/subgraph.ts @@ -0,0 +1,179 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { isMap, isSeq, parseDocument, Scalar, type Document, type YAMLMap } from 'yaml'; +import { EVM_PLACEHOLDER_ADDRESS, EXPECTED_CONTRACTS } from './catalogue.js'; +import { readRecord, type Deployment } from './manifest.js'; +import { DEPLOYMENTS_DIR, REPO_ROOT, recordPath, repoPath } from './paths.js'; + +/** Checked in. graph codegen, build and test read it directly. */ +export const SUBGRAPH_TEMPLATE = 'evm/subgraph/subgraph.template.yaml'; +/** Generated for one network and git-ignored. The only manifest `graph deploy` is given. */ +export const SUBGRAPH_MANIFEST = 'evm/subgraph/subgraph.yaml'; + +// Filled in from the deployment record, so the template must not set them. +const RECORD_FIELDS = [['network'], ['source', 'address'], ['source', 'startBlock']]; + +const NETWORK_SLUG = /^[a-z0-9]+(-[a-z0-9]+)*$/; + +export interface Rendered { + manifest: string | null; + errors: string[]; +} + +function dataSources(doc: Document): unknown[] | null { + const seq = doc.get('dataSources'); + return isSeq(seq) ? seq.items : null; +} + +/** + * Why the template cannot be rendered: a data source that does not name an EVM + * contract, or a value that belongs in the deployment record. + */ +export function templateErrors(source: string): string[] { + const doc = parseDocument(source); + if (doc.errors.length) return doc.errors.map((e) => e.message); + const items = dataSources(doc); + if (!items?.length) return ['has no dataSources']; + const errors: string[] = []; + items.forEach((ds, i) => { + if (!isMap(ds)) { + errors.push(`dataSources[${i}] is not a mapping`); + return; + } + const where = `dataSource ${ds.get('name') ?? i}`; + const abi = ds.getIn(['source', 'abi']); + if (typeof abi !== 'string' || !EXPECTED_CONTRACTS.evm.includes(abi)) { + errors.push( + `${where}: source.abi ${abi} is not one of the EVM contracts (${EXPECTED_CONTRACTS.evm.join(', ')})`, + ); + } + for (const field of RECORD_FIELDS) { + if (ds.hasIn(field)) { + errors.push( + `${where}: ${field.join('.')} is set; it comes from deployments/evm/.json`, + ); + } + } + }); + if (doc.has('templates')) { + errors.push( + 'has templates (dynamic data sources); the generator only sets network on dataSources, so extend it before adding any', + ); + } + return errors; +} + +function insertAt(map: YAMLMap, index: number, key: string, value: unknown, doc: Document) { + map.items.splice(index, 0, doc.createPair(key, value)); +} + +const keyIndex = (map: YAMLMap, key: string) => + map.items.findIndex((pair) => (pair.key as Scalar | null)?.value === key); + +/** + * Fill the template's network, contract addresses and start blocks from one EVM + * deployment record. Every data source needs its contract and a start block in + * the record; anything missing is reported and nothing is rendered. There is no + * placeholder fallback. `deployment` may be a record that failed validation (so + * the contract named in each error can still be found), hence the defensive reads. + */ +export function renderSubgraph( + template: string, + deployment: Deployment, + recordFile: string, +): Rendered { + const errors = templateErrors(template).map((e) => `${SUBGRAPH_TEMPLATE}: ${e}`); + if (errors.length) return { manifest: null, errors }; + + const doc = parseDocument(template); + const contracts = Array.isArray(deployment.contracts) ? deployment.contracts : []; + for (const ds of dataSources(doc) as YAMLMap[]) { + const name = ds.getIn(['source', 'abi']) as string; + const contract = contracts.find((c) => c?.name === name); + if (!contract) { + errors.push(`${name}: not recorded in ${recordFile}`); + continue; + } + const problems: string[] = []; + const height = contract.deployment?.height; + if (typeof height !== 'number' || !Number.isSafeInteger(height) || height < 0) { + problems.push(`no start block (deployment.height) in ${recordFile}`); + } else if (height === 0) { + // 0 is the placeholder start block of the instant configs (validate-config.js). + problems.push(`start block 0 in ${recordFile} is the placeholder, not a deployment block`); + } + if (String(contract.id).toLowerCase() === EVM_PLACEHOLDER_ADDRESS) { + problems.push(`${recordFile} has the placeholder address ${contract.id}`); + } + if (problems.length) { + errors.push(...problems.map((p) => `${name}: ${p}`)); + continue; + } + + insertAt(ds, keyIndex(ds, 'name') + 1, 'network', deployment.network, doc); + const source = ds.get('source') as YAMLMap; + const address = new Scalar(contract.id); + address.type = Scalar.QUOTE_SINGLE; + insertAt(source, keyIndex(source, 'abi'), 'address', address, doc); + insertAt(source, keyIndex(source, 'abi') + 1, 'startBlock', height, doc); + } + if (errors.length) return { manifest: null, errors }; + + doc.commentBefore = [ + ` Generated from ${path.basename(SUBGRAPH_TEMPLATE)} and deployments/evm/${deployment.network}.json`, + ` by scripts/deployment-manifest. Do not edit. Regenerate with`, + ` npm run manifest -- --network ${deployment.network}`, + ].join('\n'); + return { manifest: doc.toString({ lineWidth: 0 }), errors: [] }; +} + +/** Render the subgraph manifest for deployments/evm/.json. */ +export function generateSubgraph(network: string): Rendered { + if (!NETWORK_SLUG.test(network)) { + return { manifest: null, errors: [`"${network}" is not a network slug`] }; + } + const file = recordPath('evm', network); + const rel = path.relative(REPO_ROOT, file); + if (!fs.existsSync(file)) { + const dir = path.join(DEPLOYMENTS_DIR, 'evm'); + const known = fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .map((f) => f.slice(0, -'.json'.length)) + : []; + return { + manifest: null, + errors: [ + `no deployment record ${rel} (recorded EVM networks: ${known.join(', ') || 'none'})`, + ], + }; + } + if (!fs.existsSync(repoPath(SUBGRAPH_TEMPLATE))) { + return { manifest: null, errors: [`${SUBGRAPH_TEMPLATE} is missing`] }; + } + + const { deployment, errors } = readRecord('evm', file); + const recordErrors = errors.map((e) => `${rel}: ${e}`); + if (!deployment) return { manifest: null, errors: recordErrors }; + const rendered = renderSubgraph( + fs.readFileSync(repoPath(SUBGRAPH_TEMPLATE), 'utf8'), + deployment, + rel, + ); + if (recordErrors.length) { + return { manifest: null, errors: [...recordErrors, ...rendered.errors] }; + } + return rendered; +} + +/** The network a generated manifest is for, or null unless its data sources name exactly one. */ +export function generatedNetwork(source: string): string | null { + const doc = parseDocument(source); + if (doc.errors.length) return null; + const networks = new Set( + (dataSources(doc) ?? []).map((ds) => (isMap(ds) ? ds.get('network') : null)), + ); + const [network] = networks; + return networks.size === 1 && typeof network === 'string' ? network : null; +} diff --git a/scripts/deployment-manifest/test/check.e2e.test.ts b/scripts/deployment-manifest/test/check.e2e.test.ts index d00455f..48de853 100644 --- a/scripts/deployment-manifest/test/check.e2e.test.ts +++ b/scripts/deployment-manifest/test/check.e2e.test.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { parse as parseYaml } from 'yaml'; const run = promisify(execFile); const pkgDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); @@ -16,7 +17,7 @@ const FIXTURE_PATHS = [ 'deployments', 'README.md', 'evm/hardhat.config.ts', - 'evm/subgraph/subgraph.yaml', + 'evm/subgraph/subgraph.template.yaml', 'evm/subgraph/abis', 'evm/subgraph/horizen-testnet', 'solana/Anchor.toml', @@ -67,18 +68,18 @@ describe('check', () => { it.each([ [ - 'subgraph address', - 'evm/subgraph/subgraph.yaml', - '0x8AE65c05E7eb48B9bA652781Bc0a3DBA09A484F3', - '0x8AE65c05E7eb48B9bA652781Bc0a3DBA09A48400', - /subgraph\.yaml dataSource ERC5564Announcer: address/, + 'subgraph template address', + 'evm/subgraph/subgraph.template.yaml', + ' abi: WraithNames', + " address: '0x000000000000000000000000000000000000dead'\n abi: WraithNames", + /subgraph\.template\.yaml: dataSource WraithNames: source\.address is set/, ], [ - 'subgraph start block', - 'evm/subgraph/subgraph.yaml', - 'startBlock: 14202905', - 'startBlock: 1', - /startBlock 1, manifest height is 14202905/, + 'subgraph template contract', + 'evm/subgraph/subgraph.template.yaml', + ' abi: WraithSender', + ' abi: WraithRouter', + /dataSource WraithSender: source\.abi WraithRouter is not one of the EVM contracts/, ], [ 'instant config', @@ -207,3 +208,203 @@ describe('build', () => { expect(code).toBe(1); }); }); + +describe('subgraph', () => { + const RECORD = 'deployments/evm/horizen-testnet.json'; + const TEMPLATE = 'evm/subgraph/subgraph.template.yaml'; + const GENERATED = 'evm/subgraph/subgraph.yaml'; + + // The three contracts horizen-testnet does not record yet, with made-up + // addresses and blocks. They only ever live in the temporary fixture. + const missing = [ + { name: 'WraithNames', id: `0x${'1'.repeat(40)}`, height: 14300000 }, + { name: 'WraithSender', id: `0x${'2'.repeat(40)}`, height: 14300001 }, + { name: 'WraithWithdrawer', id: `0x${'3'.repeat(40)}`, height: 14300002 }, + ]; + + type EvmRecord = { + network: string; + contracts: Array<{ name: string; id: string; deployment: { height?: number } }>; + }; + const readJson = (rel: string) => JSON.parse(fs.readFileSync(path.join(root, rel), 'utf8')); + const readYaml = (rel: string) => parseYaml(fs.readFileSync(path.join(root, rel), 'utf8')); + const generated = () => fs.existsSync(path.join(root, GENERATED)); + + function recordAll(change?: (record: EvmRecord) => void) { + const record: EvmRecord = readJson(RECORD); + for (const { name, id, height } of missing) { + record.contracts.push({ + name, + id, + version: null, + sourceCommit: null, + artifactHash: { algorithm: 'keccak256', value: `0x${'00'.repeat(32)}` }, + deployment: { height }, + } as EvmRecord['contracts'][number]); + } + change?.(record); + fs.writeFileSync(path.join(root, RECORD), JSON.stringify(record, null, 2)); + return record; + } + + it('refuses the checked-in horizen-testnet record, naming the three contracts it lacks', async () => { + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + for (const name of ['WraithNames', 'WraithSender', 'WraithWithdrawer']) { + expect(output).toContain(`${name}: not recorded in ${RECORD}`); + } + expect(output).not.toMatch(/ERC5564Announcer|ERC6538Registry/); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('fills network, address and start block from a record with every contract', async () => { + const record = recordAll(); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toContain(`✓ ${GENERATED} for horizen-testnet`); + expect(code).toBe(0); + + const template = readYaml(TEMPLATE); + const manifest = readYaml(GENERATED); + expect(manifest.dataSources).toHaveLength(5); + manifest.dataSources.forEach((ds: any, i: number) => { + const contract = record.contracts.find((c) => c.name === ds.source.abi)!; + expect(ds.network).toBe('horizen-testnet'); + expect(ds.source.address).toBe(contract.id); + expect(ds.source.startBlock).toBe(contract.deployment.height); + // Nothing else differs from the template. + const { network: _n, ...rest } = ds; + const { address: _a, startBlock: _s, ...source } = ds.source; + expect({ ...rest, source }).toEqual(template.dataSources[i]); + }); + expect({ ...manifest, dataSources: [] }).toEqual({ ...template, dataSources: [] }); + expect(manifest.dataSources.map((ds: any) => ds.source.startBlock)).toEqual([ + 14202900, 14202905, 14300000, 14300001, 14300002, + ]); + expect(fs.readFileSync(path.join(root, GENERATED), 'utf8')).toMatch( + /^# Generated from subgraph\.template\.yaml and deployments\/evm\/horizen-testnet\.json/, + ); + }); + + it.each([ + ['has no start block', (c: EvmRecord['contracts'][number]) => delete c.deployment.height], + ['has start block 0', (c: EvmRecord['contracts'][number]) => (c.deployment.height = 0)], + ])('fails when a contract %s', async (_label, change) => { + recordAll((record) => change(record.contracts.find((c) => c.name === 'WraithSender')!)); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toMatch(/WraithSender: (no start block|start block 0)/); + expect(output).not.toMatch(/WraithNames|WraithWithdrawer/); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('fails when a contract has the placeholder address', async () => { + recordAll((record) => { + record.contracts.find((c) => c.name === 'WraithNames')!.id = + '0x000000000000000000000000000000000000dEaD'; + }); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toMatch(/WraithNames: .* has the placeholder address/); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('fails for a network without a deployment record', async () => { + const { code, output } = await cli('subgraph', '--network', 'base-sepolia'); + expect(output).toContain( + 'no deployment record deployments/evm/base-sepolia.json (recorded EVM networks: horizen-testnet)', + ); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('--check passes only while the generated manifest matches the template and record', async () => { + const none = await cli('subgraph', '--check'); + expect(none.output).toMatch(/subgraph\.yaml has not been generated/); + expect(none.code).toBe(1); + + recordAll(); + expect((await cli('subgraph', '--network', 'horizen-testnet')).code).toBe(0); + const ok = await cli('subgraph', '--check'); + expect(ok.output).toContain(`✓ ${GENERATED} is current for horizen-testnet`); + expect(ok.code).toBe(0); + + edit(GENERATED, 'startBlock: 14300001', 'startBlock: 1'); + const edited = await cli('subgraph', '--check'); + expect(edited.output).toMatch(/subgraph\.yaml differs from/); + expect(edited.code).toBe(1); + + // A record that loses a contract after generation blocks the deploy too. + expect((await cli('subgraph', '--network', 'horizen-testnet')).code).toBe(0); + fs.cpSync(path.join(repoRoot, RECORD), path.join(root, RECORD)); + const stale = await cli('subgraph', '--check'); + expect(stale.output).toContain(`WraithWithdrawer: not recorded in ${RECORD}`); + expect(stale.code).toBe(1); + }); + + it('--check refuses a manifest whose data sources name more than one network', async () => { + recordAll(); + expect((await cli('subgraph', '--network', 'horizen-testnet')).code).toBe(0); + edit(GENERATED, 'network: horizen-testnet', 'network: base-sepolia'); + const { code, output } = await cli('subgraph', '--check'); + expect(output).toMatch(/subgraph\.yaml does not name exactly one network/); + expect(code).toBe(1); + }); + + it('lists every problem of a contract, not only the first', async () => { + recordAll((record) => { + const names = record.contracts.find((c) => c.name === 'WraithNames')!; + names.deployment.height = 0; + names.id = '0x000000000000000000000000000000000000dEaD'; + }); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toMatch(/WraithNames: start block 0/); + expect(output).toMatch(/WraithNames: .* has the placeholder address/); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('reports a record that fails validation and writes nothing', async () => { + recordAll((record) => { + record.network = 'base-sepolia'; + }); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toContain(`${RECORD}: network "base-sepolia" does not match file name`); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('reports a record that is not a JSON object', async () => { + fs.writeFileSync(path.join(root, RECORD), 'null'); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toContain(`${RECORD}: not a JSON object`); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it('refuses a template with dynamic data source templates', async () => { + recordAll(); + fs.appendFileSync( + path.join(root, TEMPLATE), + '\ntemplates:\n - kind: ethereum\n name: Dynamic\n source:\n abi: WraithNames\n', + ); + const { code, output } = await cli('subgraph', '--network', 'horizen-testnet'); + expect(output).toMatch(/has templates \(dynamic data sources\)/); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); + + it.each([ + ['a network that is not a slug', ['--network', '../horizen-testnet'], /is not a network slug/], + ['neither --network nor --check', [], /subgraph takes either --network or --check/], + [ + 'both --network and --check', + ['--network', 'horizen-testnet', '--check'], + /subgraph takes either --network or --check/, + ], + ])('refuses %s', async (_label, args, message) => { + const { code, output } = await cli('subgraph', ...args); + expect(output).toMatch(message); + expect(code).toBe(1); + expect(generated()).toBe(false); + }); +});