diff --git a/.github/workflows/deploy-tests.yaml b/.github/workflows/deploy-tests.yaml index 9c49d3ad..b18f19c7 100644 --- a/.github/workflows/deploy-tests.yaml +++ b/.github/workflows/deploy-tests.yaml @@ -7,7 +7,8 @@ name: Deploy tests # Apart from CI, for two reasons. On a host it needs credentials, which a pull request from a fork must # never be given. And every target deploys — each suite is an application installed, built and served, a # minute of it at the least. So it is dispatched by hand, and the two targets on the runner, which need -# no credential, also run every night over the whole manifest. Every push to a pull request runs `node` +# no credential, also run every night over the whole manifest, once for each Node.js the applications +# are built and served under (24 and 26). Every push to a pull request runs `node` # over the whole manifest against the supported release; a newer push to the same pull request cancels # the run before it. CI checks the hooks themselves on every pull request (`check:deploy-tests`, # `check:deploy-tests-local`). @@ -44,8 +45,8 @@ on: # Every push to a pull request: `node` over the whole manifest against the supported release. It # needs no credential, so a pull request from a fork runs it too. pull_request: - # Every night, both targets on the runner over the whole manifest (`plan`): the supported - # release first, then the latest published canary in a separate run. + # Every night, both targets on the runner over the whole manifest (`plan`), under Node.js 24 and 26: + # the supported release first, then the latest published canary in a separate run. schedule: - cron: '23 17 * * *' - cron: '23 18 * * *' @@ -89,6 +90,14 @@ on: options: - none - hosted + node: + description: 'On the runner: the Node.js each application is built and served under, or both' + default: '24' + type: choice + options: + - '24' + - '26' + - '24 26' permissions: {} @@ -141,6 +150,7 @@ jobs: ONLY: ${{ inputs.only }} CONCURRENCY: ${{ inputs.concurrency }} CACHE: ${{ inputs.cache }} + NODE: ${{ inputs.node }} # Counted, never printed: the count is not a project anybody can name, the ids are. PROJECT_IDS: ${{ secrets.ADAPTER_TEST_PROJECT_IDS }} PROJECT_ID: ${{ secrets.ADAPTER_TEST_PROJECT_ID }} @@ -151,6 +161,7 @@ jobs: # dispatch is what it was asked for. if [ "$EVENT" = 'pull_request' ]; then targets='node' + nodes='24' TESTS='all' SHARDS="$RUNNER_SHARDS" ONLY='' @@ -158,6 +169,7 @@ jobs: nextjs="$NIGHTLY_NEXTJS_REF" elif [ "$EVENT" = 'schedule' ]; then targets='cloudflare node' + nodes='24 26' TESTS='all' SHARDS="$RUNNER_SHARDS" ONLY='' @@ -190,6 +202,7 @@ jobs: esac else targets="$TARGET" + nodes="${NODE:-24}" nextjs="$NEXTJS_REF" fi # Outputs are line-oriented records. Refuse input that could introduce another record. @@ -200,6 +213,16 @@ jobs: host | cloudflare | node | 'cloudflare node') ;; *) echo "target must be host, cloudflare or node, not '$targets'" >&2; exit 1 ;; esac + # The Node.js each application is built and served under. The harness itself runs on 24 + # (the job's own steps). A host builds and serves on its own runtime, so it takes 24 alone. + case "$nodes" in + 24 | 26 | '24 26') ;; + *) echo "node must be 24, 26 or both, not '$nodes'" >&2; exit 1 ;; + esac + if [ "$targets" = 'host' ] && [ "$nodes" != '24' ]; then + echo 'a host builds and serves on its own runtime: node must be 24 there' >&2 + exit 1 + fi selection="$(printf '%s' "$TESTS" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//')" if [ -z "$selection" ]; then echo 'no tests were selected: name upstream paths, or `all` for the whole manifest' >&2 @@ -299,11 +322,18 @@ jobs: include='' jobs=0 for target in $targets; do - for shard in $(printf '%s' "$shards" | tr -d '[]' | tr ',' ' '); do - include="${include:+$include,}{\"target\":\"$target\",\"shard\":$shard}" - jobs=$((jobs + 1)) + for node in $nodes; do + for shard in $(printf '%s' "$shards" | tr -d '[]' | tr ',' ' '); do + include="${include:+$include,}{\"target\":\"$target\",\"node\":\"$node\",\"shard\":$shard}" + jobs=$((jobs + 1)) + done done done + # A matrix holds 256 jobs: both targets under both Node.js versions in 64 pieces is all of it. + if [ "$jobs" -gt 256 ]; then + echo "the run would be $jobs jobs, past the 256 a matrix holds: cut it into fewer shards" >&2 + exit 1 + fi # On the runner nothing is shared, and every job runs at once; on a host, as many as there # are projects for (below). parallel="$jobs" @@ -358,7 +388,7 @@ jobs: if [ -n "$ONLY" ]; then echo "$targets, $selection: shard $only of $total, on its own" >>"$GITHUB_STEP_SUMMARY" else - echo "$targets, $selection in $total shard(s) each, $parallel at a time, $concurrency suite(s) at once in each" \ + echo "$targets under Node.js $nodes, $selection in $total shard(s) each, $parallel at a time, $concurrency suite(s) at once in each" \ >>"$GITHUB_STEP_SUMMARY" # Said out loud, because one at a time where more was asked for is a list to fix rather than # a run to wonder about. Which ids, and how many, stay out of it. @@ -375,7 +405,7 @@ jobs: deploy-tests: needs: plan - name: ${{ matrix.target }} ${{ matrix.shard }}/${{ needs.plan.outputs.total }} + name: ${{ matrix.target }} ${{ matrix.shard }}/${{ needs.plan.outputs.total }} (Node.js ${{ matrix.node }}) runs-on: ubuntu-latest # Under the six hours a job is allowed, with room for a slice that runs long to be reported rather # than cut off mid-suite. @@ -403,6 +433,18 @@ jobs: path: nextjs fetch-depth: 25 persist-credentials: false + # The Node.js the applications are built and served under, where the lane is not the harness's: + # the deploy hooks put it first on their own `PATH` (`ADAPTER_TEST_NODE_DIR`), and the harness, + # set up after it, keeps 24 for itself. Exactly the version the adapter guards a native Function + # by (`isGuardedNodeVersion`), since a build and the server that loads it must agree on it. + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: matrix.node == '26' + with: + node-version: 26.13.0 + - name: Note where the applications' Node.js is + id: application-node + if: matrix.node == '26' + run: echo "dir=$(dirname "$(command -v node)")" >>"$GITHUB_OUTPUT" # Node.js 24, not the 26 this repository builds with: the suite runs out of Next.js's checkout, and # its harness does not load on 26. `run-tests.js` requires `yargs/yargs`, an extensionless file in # a `"type": "module"` package, which 26 reads as an ES module ("require is not defined in ES @@ -597,6 +639,8 @@ jobs: NEXT_TEST_CLEANUP_SCRIPT_PATH: ${{ github.workspace }}/adapter/tools/deploy-tests/scripts/e2e-cleanup.sh # Where each application is served: a host, or this runner (`tools/deploy-tests/src/target.ts`). ADAPTER_TEST_TARGET: ${{ matrix.target }} + # The Node.js each application is built and served under, where it is not the harness's 24. + ADAPTER_TEST_NODE_DIR: ${{ steps.application-node.outputs.dir }} # The runtime cache a host's builds are given, as the plan read it (`e2e-deploy.sh`). ADAPTER_TEST_CACHE: ${{ needs.plan.outputs.cache }} # The host to deploy to, and nothing on the runner's targets. Secrets rather than anything @@ -681,19 +725,20 @@ jobs: if: always() && matrix.target != 'host' && steps.inventory.outcome == 'success' env: TARGET: ${{ matrix.target }} + NODE: ${{ matrix.node }} SHARD: ${{ matrix.shard }} run: | set -euo pipefail NEXT_COMMIT="$(git -C nextjs rev-parse HEAD)" \ node adapter/tools/deploy-tests/src/judge-results.ts --next-dir nextjs \ --results "$RUNNER_TEMP/deploy-results" - # Named by the shard alone on a host, as a record reads them, and by the target as well on this - # machine, whose two targets share a night's run. + # Named by the shard alone on a host, as a record reads them, and by the target and the Node.js as + # well on this machine, whose two targets under two Node.js versions share a night's run. - name: Save this shard's results if: always() && steps.inventory.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: ${{ matrix.target == 'host' && format('deploy-results-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.shard) || format('deploy-results-{0}-{1}-{2}-{3}', github.run_id, github.run_attempt, matrix.target, matrix.shard) }} + name: ${{ matrix.target == 'host' && format('deploy-results-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.shard) || format('deploy-results-{0}-{1}-{2}-node{3}-{4}', github.run_id, github.run_attempt, matrix.target, matrix.node, matrix.shard) }} path: ${{ runner.temp }}/deploy-results if-no-files-found: error retention-days: 30 diff --git a/tools/deploy-tests/README.md b/tools/deploy-tests/README.md index 3e80b597..89281c3a 100644 --- a/tools/deploy-tests/README.md +++ b/tools/deploy-tests/README.md @@ -203,7 +203,13 @@ many pieces (upstream's own `-g n/total`), and `only` runs one piece on its own, after it was cut short. On the runner's targets nothing is shared, so a run is cut as finely as the workflow cuts — 64 pieces unless `shards` says otherwise, all at once. Every night both targets run the whole manifest in two separate runs: the supported release at 17:23 UTC (02:23 JST), and the latest -published `next@canary` at 18:23 UTC (03:23 JST). Each run has 128 jobs. The canary run reads npm's +published `next@canary` at 18:23 UTC (03:23 JST). Each run builds and serves the applications under +Node.js 24 and under Node.js 26, so it has 256 jobs, the most a matrix holds. A dispatch picks either +or both with `node` (24 unless it says otherwise), and a pull request's run takes 24. The harness itself +runs on Node.js 24.22.1 either way, since Next.js's `run-tests.js` does not load on 26. Under 26, the +deploy and cleanup hooks put Node.js 26.13.0 first on their own `PATH` (`ADAPTER_TEST_NODE_DIR`), so the +application's install, its build and the server that serves it all run on it. The job summary and the +report name each lane: `node, Node.js 26`. The canary run reads npm's dist-tag once and resolves that version's Git tag to a commit SHA, which every shard checks out. Its plan summary names the version and SHA; a failed lookup stops the plan. Results and summaries stay separate for each run, and canary failures are judged on their own version and commit, without the diff --git a/tools/deploy-tests/REVIEWED.md b/tools/deploy-tests/REVIEWED.md index 5fcd2902..916104e2 100644 --- a/tools/deploy-tests/REVIEWED.md +++ b/tools/deploy-tests/REVIEWED.md @@ -23,7 +23,11 @@ A reading is one of four: Read at commit `29bbeab09981148ade186e8599ecdede15abc281`, from the night's run of 2026-10-26 (run 40369598264) and from both targets run again at `d2df8807` (runs 40453829649 and 40453835497). Each -reading holds on both targets unless it says otherwise. +reading holds on both targets unless it says otherwise, and under both Node.js versions a night's run +builds and serves the applications under (24 and 26) unless it names one. A reading for the lane's +own Node.js is tried before one for every Node.js: under Node.js 26 a suite can fail before it gets as +far as the failure its other reading records. The readings for Node.js 26 were read from the whole +manifest run under it on both targets on 2026-10-30 (runs 41023438620 and 41023442374). ### Deploy gates @@ -121,8 +125,32 @@ runs on `next start` 16.4.3 with no adapter, and in 14 of 20 on each target; sta every run. vercel/next.js#100776 starts the test from a page whose only links are the accordions'. The judge reads only `act`'s own timeout ("Timed out waiting for a request to be initiated"). +#### Preview mode on Node.js 26 + +Under Node.js 26, on `node` (12 cases of `prerender-preview`, `preview-fallback` and `prerender`). +`res.setPreviewData` signs its cookie with the `jsonwebtoken` Next.js 16.4.3 compiles in +(`next/dist/compiled/jsonwebtoken`). Its `buffer-equal-constant-time` reads +`require('buffer').SlowBuffer.prototype` as it loads, and Node.js 26 has no `SlowBuffer`. +`require('next/dist/compiled/jsonwebtoken')` throws on Node.js 26.13.0 with no adapter ("Cannot read +properties of undefined (reading 'prototype')"). So the route answers 500 and sets no preview cookie, +and the pages read without one render as no preview. v16.5.0-canary.30 signs with `fast-jwt` +instead. On `cloudflare`, workerd's `buffer` still has `SlowBuffer`, and the cases pass. + ### The environment +#### Native addons on Node.js 26 + +Under Node.js 26, on both targets: `prerender-native-module` (3 of 3) and `turbopack-reports` (1 of +1). Their fixture's addon is built from source at install (`node-gyp rebuild`), and its `addon.cc` +calls `v8::Context::GetIsolate`, which the V8 of Node.js 26 no longer has. The install fails ("has no +member named 'GetIsolate'"), and with it the deploy hook, before anything of the adapter's runs. + +#### `SlowBuffer` on Node.js 26 + +Under Node.js 26, on `node`: `edge-runtime-node-compatibility`'s two `node:buffer` cases expect +`SlowBuffer` among `node:buffer`'s keys. Node.js 26 removed it, and on `node` the edge runtime's +`node:buffer` is the Function's own. On `cloudflare` it is workerd's, which keeps it. + #### Native addons on `cloudflare` `prerender-native-module` (3 of 3) and `turbopack-reports` (1 of 1) load a native Node.js addon from diff --git a/tools/deploy-tests/scripts/e2e-cleanup.sh b/tools/deploy-tests/scripts/e2e-cleanup.sh index 949d2516..cd06a761 100755 --- a/tools/deploy-tests/scripts/e2e-cleanup.sh +++ b/tools/deploy-tests/scripts/e2e-cleanup.sh @@ -10,4 +10,8 @@ set -euo pipefail tool_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# Under the Node.js the deploy hook ran under (`e2e-deploy.sh`). +if [ -n "${ADAPTER_TEST_NODE_DIR:-}" ]; then + export PATH="${ADAPTER_TEST_NODE_DIR}:${PATH}" +fi node "${tool_dir}/src/main.ts" release || true diff --git a/tools/deploy-tests/scripts/e2e-deploy.sh b/tools/deploy-tests/scripts/e2e-deploy.sh index 132c7231..a271c16e 100755 --- a/tools/deploy-tests/scripts/e2e-deploy.sh +++ b/tools/deploy-tests/scripts/e2e-deploy.sh @@ -60,6 +60,20 @@ export ADAPTER_TEST_SUITE_ENV # off (`src/hook.ts`): the build below spends the same time. export ADAPTER_TEST_HOOK_STARTED_MS="${ADAPTER_TEST_HOOK_STARTED_MS:-$(node -p 'Date.now()')}" +# The Node.js the application is built and served under, where the run names one other than the +# harness's (`ADAPTER_TEST_NODE_DIR`, a directory holding `node`): first on the `PATH` of everything +# below, the server this starts included. A build and the server that loads it then agree on it, as +# the adapter requires of a native Function. Only after the suite's variables are read above, which +# would otherwise take this `PATH` for one of the suite's and hand it to the application. +if [ -n "${ADAPTER_TEST_NODE_DIR:-}" ]; then + if [ ! -x "${ADAPTER_TEST_NODE_DIR}/node" ]; then + echo "ADAPTER_TEST_NODE_DIR holds no node: ${ADAPTER_TEST_NODE_DIR}" >&2 + exit 1 + fi + export PATH="${ADAPTER_TEST_NODE_DIR}:${PATH}" +fi +echo "Node.js $(node --version) builds and serves the application" >&2 + repo_root="$(cd "${tool_dir}/../.." && pwd)" # The contract's own variable, and it names the adapter's repository: keep it working, since the # workflow in Next.js's documentation sets it and people copy that workflow. diff --git a/tools/deploy-tests/src/judge-results.ts b/tools/deploy-tests/src/judge-results.ts index b6f72bdd..e2726bbb 100644 --- a/tools/deploy-tests/src/judge-results.ts +++ b/tools/deploy-tests/src/judge-results.ts @@ -4,8 +4,8 @@ import { parseArgs } from 'node:util'; import { fails, judge } from './judge.ts'; import { type JestSuite, jestSuite, listedSuites } from './results.ts'; -import { REVIEWED } from './reviewed.ts'; -import { describe, type ShardVerdict, shardFails } from './verdicts.ts'; +import { isNodeLine, REVIEWED } from './reviewed.ts'; +import { describe, laneOf, type ShardVerdict, shardFails } from './verdicts.ts'; /** * Judges a shard of a run on this machine's targets (`judge.ts`) where it ran: the suites it was to @@ -16,8 +16,8 @@ import { describe, type ShardVerdict, shardFails } from './verdicts.ts'; * left no result, or a runner that did not finish: one that wrote no exit code beside the results * (`runner-exit-code.txt`), stopped at its step's time limit or never started. * - * Where it ran, from the environment: the target (`TARGET`), the shard (`SHARD`) and the commit of - * Next.js (`NEXT_COMMIT`). + * Where it ran, from the environment: the target (`TARGET`), the Node.js of its lane where it has one + * (`NODE`), the shard (`SHARD`) and the commit of Next.js (`NEXT_COMMIT`). */ const { values } = parseArgs({ options: { @@ -45,6 +45,12 @@ if (target !== 'cloudflare' && target !== 'node') { `judge-results: TARGET is one of this machine's, cloudflare or node, not ${target}`, ); } +// The lane's Node.js, where the run names one: a major version, as the plan's matrix has it. +const nodeAsked = process.env['NODE']; +const node = nodeAsked === undefined || nodeAsked === '' ? undefined : nodeAsked; +if (node !== undefined && !isNodeLine(node)) { + throw new Error(`judge-results: NODE is 24 or 26, the lanes' Node.js, not ${node}`); +} const shard = Number(required('SHARD')); if (!Number.isSafeInteger(shard) || shard < 1) { throw new Error('judge-results: SHARD must be a positive whole number'); @@ -83,7 +89,11 @@ function endedByWatchdog(): string[] { const verdict: ShardVerdict = { ...judge( - { target, next: { version: next.version, commit: required('NEXT_COMMIT') } }, + { + target, + ...(node !== undefined && { node }), + next: { version: next.version, commit: required('NEXT_COMMIT') }, + }, listed, results, REVIEWED, @@ -93,7 +103,7 @@ const verdict: ShardVerdict = { ended: endedByWatchdog(), }; writeFileSync(path.join(resultsDir, 'verdict.json'), `${JSON.stringify(verdict, undefined, 2)}\n`); -const text = describe(target, [verdict]); +const text = describe(laneOf(target, node), [verdict]); console.log(text); const summary = process.env['GITHUB_STEP_SUMMARY']; if (summary !== undefined && summary !== '') { diff --git a/tools/deploy-tests/src/judge.ts b/tools/deploy-tests/src/judge.ts index fecc6153..b6c28713 100644 --- a/tools/deploy-tests/src/judge.ts +++ b/tools/deploy-tests/src/judge.ts @@ -1,5 +1,5 @@ import { failureOf, type JestSuite } from './results.ts'; -import type { Reading, ReviewedCase, RunnerTarget, Symptom } from './reviewed.ts'; +import type { NodeLine, Reading, ReviewedCase, RunnerTarget, Symptom } from './reviewed.ts'; /** * A shard's results on this machine's targets, judged against the reviewed cases (`reviewed.ts`): @@ -8,9 +8,14 @@ import type { Reading, ReviewedCase, RunnerTarget, Symptom } from './reviewed.ts * counts and readings, as `keep-results.ts` keeps them: never a message. */ -/** What a run measured: where, and which release of Next.js. */ +/** What a run measured: where, under which Node.js, and which release of Next.js. */ export interface Measured { readonly target: RunnerTarget; + /** + * The major version of the Node.js the applications were built and served under, where the run + * names one (its lane). Absent in a verdict from before there were lanes. + */ + readonly node?: NodeLine; readonly next: { readonly version: string; readonly commit: string }; } @@ -89,12 +94,19 @@ function reviewedFor( entry.next.version === measured.next.version && entry.next.commit === measured.next.commit && entry.suite === suite && - (entry.targets === undefined || entry.targets.includes(measured.target)) + (entry.targets === undefined || entry.targets.includes(measured.target)) && + (entry.nodes === undefined || + (measured.node !== undefined && entry.nodes.includes(measured.node))) ); }); + // A reading for the run's own Node.js before one for every Node.js, so that a lane where a case + // fails for its Node.js is read so; and the case's own before its suite's. + const ranked = holding.toSorted( + (a, b) => Number(b.nodes !== undefined) - Number(a.nodes !== undefined), + ); return ( - holding.find((entry) => entry.fullName === fullName) ?? - holding.find((entry) => entry.fullName === '*') + ranked.find((entry) => entry.fullName === fullName) ?? + ranked.find((entry) => entry.fullName === '*') ); } @@ -254,6 +266,7 @@ export function judge( return { version: 1, target: measured.target, + ...(measured.node !== undefined && { node: measured.node }), next: measured.next, suites: { listed: listed.length, missing }, cases: { ...tally.cases, reviewed: tally.reviewed }, diff --git a/tools/deploy-tests/src/report-results.ts b/tools/deploy-tests/src/report-results.ts index d9306c29..8b128ccc 100644 --- a/tools/deploy-tests/src/report-results.ts +++ b/tools/deploy-tests/src/report-results.ts @@ -2,12 +2,12 @@ import { appendFileSync, existsSync, readdirSync, readFileSync } from 'node:fs'; import path from 'node:path'; import { parseArgs } from 'node:util'; -import { describe, type ShardVerdict } from './verdicts.ts'; +import { describe, laneOf, type ShardVerdict } from './verdicts.ts'; /** * What a run on this machine's targets came to: the verdict of every shard (`verdict.json`, which * `judge-results.ts` wrote and the shard's artifact kept), downloaded into `--dir`, summed for each - * target into the job's summary. The shards the run was cut into, from + * lane — a target, under the Node.js the run names — into the job's summary. The shards the run was cut into, from * the plan's matrix (`MATRIX`), name those that left no verdict. It reports and judges nothing: each * shard failed for itself. */ @@ -18,13 +18,18 @@ if (dir === undefined) { } const matrix = JSON.parse(process.env['MATRIX'] ?? '{"include":[]}') as { - readonly include?: readonly { readonly target?: unknown; readonly shard?: unknown }[]; + readonly include?: readonly { + readonly target?: unknown; + readonly node?: unknown; + readonly shard?: unknown; + }[]; }; const expected = new Map(); const shards = matrix.include ?? []; -for (const { target, shard } of shards) { +for (const { target, node, shard } of shards) { if (typeof target === 'string' && typeof shard === 'number') { - expected.set(target, [...(expected.get(target) ?? []), shard]); + const lane = laneOf(target, typeof node === 'string' ? node : undefined); + expected.set(lane, [...(expected.get(lane) ?? []), shard]); } } @@ -39,17 +44,17 @@ for (const file of files) { } } -const targets = new Set(expected.keys()); -for (const { target } of verdicts) { - targets.add(target); +const lanes = new Set(expected.keys()); +for (const { target, node } of verdicts) { + lanes.add(laneOf(target, node)); } -const text = [...targets] +const text = [...lanes] .toSorted((a, b) => a.localeCompare(b)) - .map((target) => { + .map((lane) => { const own = verdicts - .filter((verdict) => verdict.target === target) + .filter(({ target, node }) => laneOf(target, node) === lane) .toSorted((a, b) => a.shard - b.shard); - return describe(target, own, expected.get(target) ?? []); + return describe(lane, own, expected.get(lane) ?? []); }) .join('\n'); console.log(text); diff --git a/tools/deploy-tests/src/reviewed.ts b/tools/deploy-tests/src/reviewed.ts index c580bda9..581b4643 100644 --- a/tools/deploy-tests/src/reviewed.ts +++ b/tools/deploy-tests/src/reviewed.ts @@ -11,6 +11,14 @@ /** The targets on this machine (`target.ts`), which this record is for. */ export type RunnerTarget = 'cloudflare' | 'node'; +/** The Node.js versions a run builds and serves the applications under (its lanes), by major. */ +export type NodeLine = '24' | '26'; + +/** Whether `value` names one of the lanes' Node.js versions. */ +export function isNodeLine(value: string): value is NodeLine { + return value === '24' || value === '26'; +} + /** * How a reviewed case reads: * - `deploy-gate`: upstream registers the case `test.failing` when deployed, for what Vercel does not @@ -31,6 +39,8 @@ export interface ReviewedCase { readonly fullName: string; /** Where the reading holds; on both targets where absent. */ readonly targets?: readonly RunnerTarget[]; + /** Under which Node.js the reading holds; under every one where absent. */ + readonly nodes?: readonly NodeLine[]; readonly reading: Reading; /** For every reading but `deploy-gate`, whose failure is Jest's own refusal and nothing else. */ readonly symptom?: Symptom; @@ -133,11 +143,24 @@ function inAnotherDirectory(message: string): boolean { } /** A record's cases: each suite, with the full names of the cases read, or `*` for all of them. */ -interface Group extends Pick { +interface Group extends Pick { readonly cases: Readonly>; } const GATE: Reading = 'deploy-gate'; +/** The lane that builds and serves the applications under Node.js 26. */ +const ON_NODE_26: readonly NodeLine[] = ['26']; +/** + * How a preview request fails where the server could not sign its cookie: the API route answers + * 500 and sets no cookie, and the pages read without one render as no preview. + */ +const NO_PREVIEW_COOKIE: readonly string[] = [ + 'Expected: 200\nReceived: 500', + 'Received has value: null', + '- "preview": true,', + '- "isPreview": true,', + 'Unexpected end of JSON input', +]; /** How Next.js's harness says the deploy hook failed, the build or the server it starts. */ const DEPLOY_FAILED = 'Custom deploy script failed'; const EVERY_CASE = ['*']; @@ -318,6 +341,56 @@ const AT_16_4_3: readonly Group[] = [ EVERY_CASE, }, }, + { + reading: 'upstream', + record: 'preview-mode-on-nodejs-26', + targets: ['node'], + nodes: ON_NODE_26, + symptom: (message) => NO_PREVIEW_COOKIE.some((part) => message.includes(part)), + cases: { + 'test/e2e/prerender-preview/prerender-preview.test.ts': [ + 'Prerender Preview Mode should enable preview mode', + 'Prerender Preview Mode should expire cookies with a maxAge', + 'Prerender Preview Mode should set custom path cookies', + 'Prerender Preview Mode should not return fallback page on preview request', + 'Prerender Preview Mode should pass the preview data to API routes', + ], + 'test/e2e/preview-fallback/preview-fallback.test.ts': [ + 'Preview mode with fallback pages should get preview cookie correctly', + 'Preview mode with fallback pages should not write preview index SSG page to cache', + 'Preview mode with fallback pages should not write preview dynamic prerendered SSG page to cache no fallback', + 'Preview mode with fallback pages should not write preview dynamic SSG page to cache no fallback', + 'Preview mode with fallback pages should not write preview dynamic prerendered SSG page to cache with fallback', + 'Preview mode with fallback pages should not write preview dynamic non-prerendered SSG page to cache with fallback', + ], + 'test/e2e/prerender.test.ts': [ + 'Prerender should revalidate on-demand revalidate with preview cookie', + ], + }, + }, + { + reading: 'environment', + record: 'native-addons-on-nodejs-26', + nodes: ON_NODE_26, + symptom: inOrder(DEPLOY_FAILED, 'has no member named', 'GetIsolate'), + cases: { + 'test/e2e/prerender-native-module.test.ts': EVERY_CASE, + 'test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts': EVERY_CASE, + }, + }, + { + reading: 'environment', + record: 'slowbuffer-on-nodejs-26', + targets: ['node'], + nodes: ON_NODE_26, + symptom: inOrder('"exposedKeys": ArrayContaining [', '- "SlowBuffer",'), + cases: { + 'test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts': [ + 'edge runtime node compatibility [app] supports node:buffer', + 'edge runtime node compatibility [pages/api] supports node:buffer', + ], + }, + }, { reading: 'environment', record: 'native-addons-on-cloudflare', diff --git a/tools/deploy-tests/src/verdicts.ts b/tools/deploy-tests/src/verdicts.ts index 54ebaf63..574bf251 100644 --- a/tools/deploy-tests/src/verdicts.ts +++ b/tools/deploy-tests/src/verdicts.ts @@ -15,6 +15,11 @@ export interface ShardVerdict extends Verdict { readonly ended?: readonly string[]; } +/** What a summary names a lane by: the target, and the Node.js where the run names one. */ +export function laneOf(target: string, node: string | undefined): string { + return node === undefined ? target : `${target}, Node.js ${node}`; +} + /** Whether a shard's verdict fails it: as `fails` has it, or where its runner did not finish. */ export function shardFails(verdict: ShardVerdict, failed: (verdict: Verdict) => boolean): boolean { return failed(verdict) || verdict.runnerFinished === false; diff --git a/tools/deploy-tests/test/judge.test.ts b/tools/deploy-tests/test/judge.test.ts index 19991f06..603af085 100644 --- a/tools/deploy-tests/test/judge.test.ts +++ b/tools/deploy-tests/test/judge.test.ts @@ -281,6 +281,25 @@ const RECORDED: Readonly> = { 'test/e2e/app-dir/turbopack-reports/turbopack-reports.test.ts': [NATIVE_SPLIT_CONTAINER], }; +/** What the native suites failed with under Node.js 26: their addon, built at install, did not build. */ +const NATIVE_ON_26 = + 'Error: Custom deploy script failed: Node.js v26.13.0 builds and serves the application\n' + + '.../node_modules/native-addon install: ../addon.cc:8:35: error: ‘class v8::Context’ has no member named ‘GetIsolate’'; + +/** What a record's cases failed with, where it reads a suite that another record reads otherwise. */ +const RECORDED_RECORDS: Readonly> = { + 'native-addons-on-nodejs-26': [NATIVE_ON_26], + 'preview-mode-on-nodejs-26': [ + 'Error: expect(received).toBe(expected) // Object.is equality\n\nExpected: 200\nReceived: 500', + 'Error: expect(received).toMatch(expected)\n\nMatcher error: received value must be a string\n\nReceived has value: null', + 'Error: expect(received).toEqual(expected) // deep equality\n\n- Expected - 2\n+ Received + 2\n\n Object {\n- "preview": true,\n- "previewData": Object {},\n+ "preview": false,\n+ "previewData": null,\n }', + 'SyntaxError: Unexpected end of JSON input\n at JSON.parse ()', + ], + 'slowbuffer-on-nodejs-26': [ + 'Error: expect(received).toEqual(expected) // deep equality\n\n- Expected - 4\n+ Received + 13\n\n Object {\n "Buffer === B.Buffer": true,\n- "exposedKeys": ArrayContaining [\n- "constants",\n+ "exposedKeys": Array [\n+ "Buffer",\n "kMaxLength",\n- "Buffer",\n- "SlowBuffer",\n+ "btoa",', + ], +}; + /** An inline snapshot's failure in the trace suite, with these lines of its diff. */ function traceDiff(...lines: string[]): string { return [ @@ -338,7 +357,9 @@ function recordedFor(entry: ReviewedCase): readonly string[] { if (entry.reading === 'deploy-gate') { return [GATE_PASSED]; } - return RECORDED_CASES[entry.fullName] ?? RECORDED[entry.suite] ?? []; + return ( + RECORDED_RECORDS[entry.record] ?? RECORDED_CASES[entry.fullName] ?? RECORDED[entry.suite] ?? [] + ); } /** A heading's anchor, as GitHub makes it. */ @@ -349,12 +370,20 @@ function anchorOf(heading: string): string { .replaceAll(/\s/gu, '-'); } -/** A reviewed case judged on its own, failed with `message`. */ -function judgedAlone(entry: ReviewedCase, target: Measured['target'], message: string): Verdict { +/** + * A reviewed case judged on its own, failed with `message`, under the Node.js `node` names: by + * default the first the case is read for, or none where it is read for every one. + */ +function judgedAlone( + entry: ReviewedCase, + target: Measured['target'], + message: string, + node: Measured['node'] = entry.nodes?.[0], +): Verdict { const name = entry.fullName === '*' ? 'any case' : entry.fullName; const result = suiteOf([jestCase(name, 'failed', [message])], {}, entry.suite); return judge( - { target, next: entry.next }, + { target, ...(node !== undefined && { node }), next: entry.next }, [entry.suite], new Map([[entry.suite, result]]), REVIEWED, @@ -380,10 +409,11 @@ describe('the reviewed cases', () => { expect(misread).toStrictEqual([]); }); - it('read no case twice on one target', () => { + it('read no case twice on one target under one Node.js', () => { const keys = REVIEWED.flatMap((entry) => { return (entry.targets ?? ['cloudflare', 'node']).map( - (target) => `${entry.next.commit} ${target} ${entry.suite} ${entry.fullName}`, + (target) => + `${entry.next.commit} ${target} ${entry.nodes?.join(',') ?? 'every'} ${entry.suite} ${entry.fullName}`, ); }); @@ -429,6 +459,49 @@ describe('the reviewed cases', () => { expect(judgedAlone(entry!, 'cloudflare', NATIVE_REFUSED).unexplained).toHaveLength(1); }); + /** + * Under Node.js 26 the native suites fail at install, before they get as far as the refusal the + * reading for every Node.js records on `cloudflare`: the lane's own reading is tried first. + */ + it("read a lane's own failure before the one recorded for every Node.js", () => { + const entry = REVIEWED.find( + ({ suite, record: anchor }) => + suite === 'test/e2e/prerender-native-module.test.ts' && anchor.includes('cloudflare'), + ); + + expect(entry).toBeDefined(); + expect(judgedAlone(entry!, 'cloudflare', NATIVE_ON_26, '26').unexplained).toStrictEqual([]); + expect(judgedAlone(entry!, 'cloudflare', NATIVE_ON_26, '24').unexplained).toHaveLength(1); + expect(judgedAlone(entry!, 'cloudflare', NATIVE_CONTAINER, '24').unexplained).toStrictEqual([]); + }); + + it('read nothing for Node.js 26 under 24, nor where the run names no Node.js', () => { + const lanes = REVIEWED.filter(({ nodes }) => nodes !== undefined); + + expect(lanes).not.toHaveLength(0); + + for (const entry of lanes) { + const target = entry.targets?.[0] ?? 'node'; + const [message = ''] = recordedFor(entry); + + expect(judgedAlone(entry, target, message, '24').unexplained).toHaveLength(1); + } + const [preview] = lanes; + const result = suiteOf( + [jestCase(preview!.fullName, 'failed', [...recordedFor(preview!)])], + {}, + preview!.suite, + ); + const { unexplained } = judge( + { target: 'node', next: preview!.next }, + [preview!.suite], + new Map([[preview!.suite, result]]), + REVIEWED, + ); + + expect(unexplained).toHaveLength(1); + }); + it('read the failures they were read from', () => { for (const entry of REVIEWED) { const messages = recordedFor(entry); @@ -636,6 +709,37 @@ describe('a run’s verdicts summed for its report', () => { expect(run.stdout).toContain('**1 shard left no verdict:** 2.'); }); + /** A night's run has a lane for each Node.js: one target's shards under each are summed apart. */ + it('sums each lane apart, a target under each Node.js the run names', () => { + const base = mkdtempSync(path.join(os.tmpdir(), 'upwind-report-results-')); + dirs.push(base); + for (const [name, verdict] of [ + ['deploy-results-7-1-node-node24-1', { ...verdictOf(1, 10), node: '24' }], + ['deploy-results-7-1-node-node26-1', { ...verdictOf(1, 3), node: '26' }], + ] as const) { + mkdirSync(path.join(base, name)); + writeFileSync(path.join(base, name, 'verdict.json'), JSON.stringify(verdict)); + } + const matrix = { + include: ['24', '26'].flatMap((node) => + [1, 2].map((shard) => ({ target: 'node', node, shard })), + ), + }; + const run = spawnSync(process.execPath, [script, '--dir', base], { + encoding: 'utf8', + env: { PATH: process.env['PATH'], MATRIX: JSON.stringify(matrix) }, + }); + const [under24, under26] = run.stdout.split('### ').slice(1); + + expect(run.status).toBe(0); + expect(under24).toContain('`node, Node.js 24`'); + expect(under24).toContain('| 2 | 10 | 1 | 2 | 0 | 0 | 1 | 0 |'); + expect(under24).toContain('**1 shard left no verdict:** 2.'); + expect(under26).toContain('`node, Node.js 26`'); + expect(under26).toContain('| 2 | 3 | 1 | 2 | 0 | 0 | 1 | 0 |'); + expect(under26).toContain('**1 shard left no verdict:** 2.'); + }); + it('finds the one artifact a run kept, unpacked without a directory of its own', () => { const base = mkdtempSync(path.join(os.tmpdir(), 'upwind-report-results-')); dirs.push(base); diff --git a/tools/deploy-tests/test/workflow-security.test.ts b/tools/deploy-tests/test/workflow-security.test.ts index f6ef3a7d..6306baf3 100644 --- a/tools/deploy-tests/test/workflow-security.test.ts +++ b/tools/deploy-tests/test/workflow-security.test.ts @@ -40,6 +40,15 @@ process.stdout.write(process.env[command === 'curl' ? 'PLAN_REGISTRY' : 'PLAN_RE process.exit(Number(process.env[command === 'curl' ? 'PLAN_CURL_STATUS' : 'PLAN_GIT_STATUS'])); `; +/** Every shard of `targets` under each of `nodes`, in the order the plan lists them. */ +function lanes(targets: readonly string[], nodes: readonly string[], total: number) { + return targets.flatMap((target) => { + return nodes.flatMap((node) => + Array.from({ length: total }, (_, index) => ({ target, node, shard: index + 1 })), + ); + }); +} + describe('workflow security', () => { const repository = path.resolve(import.meta.dirname, '../../..'); const script = path.join(repository, '.github/scripts/verify-release.sh'); @@ -273,6 +282,7 @@ describe('workflow security', () => { ONLY: '', CONCURRENCY: '', CACHE: '', + NODE: '', PROJECT_IDS: '', PROJECT_ID: '', GITHUB_OUTPUT: output, @@ -316,7 +326,7 @@ describe('workflow security', () => { }); it.each([STABLE_SCHEDULE, CANARY_SCHEDULE])( - 'runs both local targets over the whole manifest on schedule %s', + 'runs both local targets under Node.js 24 and 26 over the whole manifest on schedule %s', (schedule) => { const result = plan('ignored', { EVENT: 'schedule', @@ -327,23 +337,63 @@ describe('workflow security', () => { CONCURRENCY: '8', }); const total = Number(workflowSetting('RUNNER_SHARDS')); - const matrix = { - include: ['cloudflare', 'node'].flatMap((target) => - Array.from({ length: total }, (_, index) => ({ target, shard: index + 1 })), - ), - }; + const matrix = { include: lanes(['cloudflare', 'node'], ['24', '26'], total) }; const output = readFileSync(result.output, 'utf8'); expect(result.status).toBe(0); expect(total).toBe(64); - expect(matrix.include).toHaveLength(128); + expect(matrix.include).toHaveLength(256); expect(deployTestsWorkflow()).toContain(` - cron: '${schedule}'`); expect(output).toContain(`matrix=${JSON.stringify(matrix)}\n`); - expect(output).toContain('total=64\nparallel=128\nconcurrency=2\n'); + expect(output).toContain('total=64\nparallel=256\nconcurrency=2\n'); expect(output).toContain('tests=all\ncache=none\n'); }, ); + it('runs node alone under Node.js 24 over the whole manifest on a pull request', () => { + const result = plan('ignored', { + EVENT: 'pull_request', + TARGET: 'host', + TESTS: 'ignored', + NODE: '26', + ONLY: '9', + }); + const matrix = { include: lanes(['node'], ['24'], 64) }; + const output = readFileSync(result.output, 'utf8'); + + expect(result.status).toBe(0); + expect(output).toContain(`matrix=${JSON.stringify(matrix)}\n`); + expect(output).toContain(`nextjs=${workflowSetting('NIGHTLY_NEXTJS_REF')}\n`); + }); + + it('runs a dispatch under the Node.js versions it names', () => { + const result = plan('v16.4.3', { TARGET: 'cloudflare', NODE: '24 26', SHARDS: '2' }); + + expect(result.status).toBe(0); + expect(readFileSync(result.output, 'utf8')).toContain( + `matrix=${JSON.stringify({ include: lanes(['cloudflare'], ['24', '26'], 2) })}\n`, + ); + expect(result.summary).toContain('cloudflare under Node.js 24 26'); + }); + + it.each(['22', '26 24', '24\nmatrix={}', 'latest'])( + 'refuses a Node.js the lanes do not have: %j', + (node) => { + const result = plan('v16.4.3', { NODE: node }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('node must be 24, 26 or both'); + expect(existsSync(result.output)).toBe(false); + }, + ); + + it('refuses Node.js 26 for a host, which builds and serves on its own runtime', () => { + const result = plan('v16.4.3', { TARGET: 'host', NODE: '26', PROJECT_IDS: 'project-1' }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('node must be 24 there'); + }); + function canaryPlan(reply: NetworkReply = {}) { return plan('', { EVENT: 'schedule', SCHEDULE: CANARY_SCHEDULE }, reply); }