Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 56 additions & 11 deletions .github/workflows/deploy-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ name: Deploy tests
# Apart from CI, for two reasons. On a host it needs credentials, which a pull request from a fork must
# never be given. And every target deploys — each suite is an application installed, built and served, a
# minute of it at the least. So it is dispatched by hand, and the two targets on the runner, which need
# no credential, also run every night over the whole manifest. Every push to a pull request runs `node`
# no credential, also run every night over the whole manifest, once for each Node.js the applications
# are built and served under (24 and 26). Every push to a pull request runs `node`
# over the whole manifest against the supported release; a newer push to the same pull request cancels
# the run before it. CI checks the hooks themselves on every pull request (`check:deploy-tests`,
# `check:deploy-tests-local`).
Expand Down Expand Up @@ -44,8 +45,8 @@ on:
# Every push to a pull request: `node` over the whole manifest against the supported release. It
# needs no credential, so a pull request from a fork runs it too.
pull_request:
# Every night, both targets on the runner over the whole manifest (`plan`): the supported
# release first, then the latest published canary in a separate run.
# Every night, both targets on the runner over the whole manifest (`plan`), under Node.js 24 and 26:
# the supported release first, then the latest published canary in a separate run.
schedule:
- cron: '23 17 * * *'
- cron: '23 18 * * *'
Expand Down Expand Up @@ -89,6 +90,14 @@ on:
options:
- none
- hosted
node:
description: 'On the runner: the Node.js each application is built and served under, or both'
default: '24'
type: choice
options:
- '24'
- '26'
- '24 26'

permissions: {}

Expand Down Expand Up @@ -141,6 +150,7 @@ jobs:
ONLY: ${{ inputs.only }}
CONCURRENCY: ${{ inputs.concurrency }}
CACHE: ${{ inputs.cache }}
NODE: ${{ inputs.node }}
# Counted, never printed: the count is not a project anybody can name, the ids are.
PROJECT_IDS: ${{ secrets.ADAPTER_TEST_PROJECT_IDS }}
PROJECT_ID: ${{ secrets.ADAPTER_TEST_PROJECT_ID }}
Expand All @@ -151,13 +161,15 @@ jobs:
# dispatch is what it was asked for.
if [ "$EVENT" = 'pull_request' ]; then
targets='node'
nodes='24'
TESTS='all'
SHARDS="$RUNNER_SHARDS"
ONLY=''
CONCURRENCY=''
nextjs="$NIGHTLY_NEXTJS_REF"
elif [ "$EVENT" = 'schedule' ]; then
targets='cloudflare node'
nodes='24 26'
TESTS='all'
SHARDS="$RUNNER_SHARDS"
ONLY=''
Expand Down Expand Up @@ -190,6 +202,7 @@ jobs:
esac
else
targets="$TARGET"
nodes="${NODE:-24}"
nextjs="$NEXTJS_REF"
fi
# Outputs are line-oriented records. Refuse input that could introduce another record.
Expand All @@ -200,6 +213,16 @@ jobs:
host | cloudflare | node | 'cloudflare node') ;;
*) echo "target must be host, cloudflare or node, not '$targets'" >&2; exit 1 ;;
esac
# The Node.js each application is built and served under. The harness itself runs on 24
# (the job's own steps). A host builds and serves on its own runtime, so it takes 24 alone.
case "$nodes" in
24 | 26 | '24 26') ;;
*) echo "node must be 24, 26 or both, not '$nodes'" >&2; exit 1 ;;
esac
if [ "$targets" = 'host' ] && [ "$nodes" != '24' ]; then
echo 'a host builds and serves on its own runtime: node must be 24 there' >&2
exit 1
fi
selection="$(printf '%s' "$TESTS" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//')"
if [ -z "$selection" ]; then
echo 'no tests were selected: name upstream paths, or `all` for the whole manifest' >&2
Expand Down Expand Up @@ -299,11 +322,18 @@ jobs:
include=''
jobs=0
for target in $targets; do
for shard in $(printf '%s' "$shards" | tr -d '[]' | tr ',' ' '); do
include="${include:+$include,}{\"target\":\"$target\",\"shard\":$shard}"
jobs=$((jobs + 1))
for node in $nodes; do
for shard in $(printf '%s' "$shards" | tr -d '[]' | tr ',' ' '); do
include="${include:+$include,}{\"target\":\"$target\",\"node\":\"$node\",\"shard\":$shard}"
jobs=$((jobs + 1))
done
done
done
# A matrix holds 256 jobs: both targets under both Node.js versions in 64 pieces is all of it.
if [ "$jobs" -gt 256 ]; then
echo "the run would be $jobs jobs, past the 256 a matrix holds: cut it into fewer shards" >&2
exit 1
fi
# On the runner nothing is shared, and every job runs at once; on a host, as many as there
# are projects for (below).
parallel="$jobs"
Expand Down Expand Up @@ -358,7 +388,7 @@ jobs:
if [ -n "$ONLY" ]; then
echo "$targets, $selection: shard $only of $total, on its own" >>"$GITHUB_STEP_SUMMARY"
else
echo "$targets, $selection in $total shard(s) each, $parallel at a time, $concurrency suite(s) at once in each" \
echo "$targets under Node.js $nodes, $selection in $total shard(s) each, $parallel at a time, $concurrency suite(s) at once in each" \
>>"$GITHUB_STEP_SUMMARY"
# Said out loud, because one at a time where more was asked for is a list to fix rather than
# a run to wonder about. Which ids, and how many, stay out of it.
Expand All @@ -375,7 +405,7 @@ jobs:

deploy-tests:
needs: plan
name: ${{ matrix.target }} ${{ matrix.shard }}/${{ needs.plan.outputs.total }}
name: ${{ matrix.target }} ${{ matrix.shard }}/${{ needs.plan.outputs.total }} (Node.js ${{ matrix.node }})
runs-on: ubuntu-latest
# Under the six hours a job is allowed, with room for a slice that runs long to be reported rather
# than cut off mid-suite.
Expand Down Expand Up @@ -403,6 +433,18 @@ jobs:
path: nextjs
fetch-depth: 25
persist-credentials: false
# The Node.js the applications are built and served under, where the lane is not the harness's:
# the deploy hooks put it first on their own `PATH` (`ADAPTER_TEST_NODE_DIR`), and the harness,
# set up after it, keeps 24 for itself. Exactly the version the adapter guards a native Function
# by (`isGuardedNodeVersion`), since a build and the server that loads it must agree on it.
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
if: matrix.node == '26'
with:
node-version: 26.13.0
- name: Note where the applications' Node.js is
id: application-node
if: matrix.node == '26'
run: echo "dir=$(dirname "$(command -v node)")" >>"$GITHUB_OUTPUT"
# Node.js 24, not the 26 this repository builds with: the suite runs out of Next.js's checkout, and
# its harness does not load on 26. `run-tests.js` requires `yargs/yargs`, an extensionless file in
# a `"type": "module"` package, which 26 reads as an ES module ("require is not defined in ES
Expand Down Expand Up @@ -597,6 +639,8 @@ jobs:
NEXT_TEST_CLEANUP_SCRIPT_PATH: ${{ github.workspace }}/adapter/tools/deploy-tests/scripts/e2e-cleanup.sh
# Where each application is served: a host, or this runner (`tools/deploy-tests/src/target.ts`).
ADAPTER_TEST_TARGET: ${{ matrix.target }}
# The Node.js each application is built and served under, where it is not the harness's 24.
ADAPTER_TEST_NODE_DIR: ${{ steps.application-node.outputs.dir }}
# The runtime cache a host's builds are given, as the plan read it (`e2e-deploy.sh`).
ADAPTER_TEST_CACHE: ${{ needs.plan.outputs.cache }}
# The host to deploy to, and nothing on the runner's targets. Secrets rather than anything
Expand Down Expand Up @@ -681,19 +725,20 @@ jobs:
if: always() && matrix.target != 'host' && steps.inventory.outcome == 'success'
env:
TARGET: ${{ matrix.target }}
NODE: ${{ matrix.node }}
SHARD: ${{ matrix.shard }}
run: |
set -euo pipefail
NEXT_COMMIT="$(git -C nextjs rev-parse HEAD)" \
node adapter/tools/deploy-tests/src/judge-results.ts --next-dir nextjs \
--results "$RUNNER_TEMP/deploy-results"
# Named by the shard alone on a host, as a record reads them, and by the target as well on this
# machine, whose two targets share a night's run.
# Named by the shard alone on a host, as a record reads them, and by the target and the Node.js as
# well on this machine, whose two targets under two Node.js versions share a night's run.
- name: Save this shard's results
if: always() && steps.inventory.outcome == 'success'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: ${{ matrix.target == 'host' && format('deploy-results-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.shard) || format('deploy-results-{0}-{1}-{2}-{3}', github.run_id, github.run_attempt, matrix.target, matrix.shard) }}
name: ${{ matrix.target == 'host' && format('deploy-results-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.shard) || format('deploy-results-{0}-{1}-{2}-node{3}-{4}', github.run_id, github.run_attempt, matrix.target, matrix.node, matrix.shard) }}
path: ${{ runner.temp }}/deploy-results
if-no-files-found: error
retention-days: 30
Expand Down
8 changes: 7 additions & 1 deletion tools/deploy-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,13 @@ many pieces (upstream's own `-g n/total`), and `only` runs one piece on its own,
after it was cut short. On the runner's targets nothing is shared, so a run is cut as finely as the
workflow cuts — 64 pieces unless `shards` says otherwise, all at once. Every night both targets run
the whole manifest in two separate runs: the supported release at 17:23 UTC (02:23 JST), and the latest
published `next@canary` at 18:23 UTC (03:23 JST). Each run has 128 jobs. The canary run reads npm's
published `next@canary` at 18:23 UTC (03:23 JST). Each run builds and serves the applications under
Node.js 24 and under Node.js 26, so it has 256 jobs, the most a matrix holds. A dispatch picks either
or both with `node` (24 unless it says otherwise), and a pull request's run takes 24. The harness itself
runs on Node.js 24.22.1 either way, since Next.js's `run-tests.js` does not load on 26. Under 26, the
deploy and cleanup hooks put Node.js 26.13.0 first on their own `PATH` (`ADAPTER_TEST_NODE_DIR`), so the
application's install, its build and the server that serves it all run on it. The job summary and the
report name each lane: `node, Node.js 26`. The canary run reads npm's
dist-tag once and resolves that version's Git tag to a commit SHA, which every shard checks out. Its
plan summary names the version and SHA; a failed lookup stops the plan. Results and summaries stay
separate for each run, and canary failures are judged on their own version and commit, without the
Expand Down
30 changes: 29 additions & 1 deletion tools/deploy-tests/REVIEWED.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ A reading is one of four:

Read at commit `29bbeab09981148ade186e8599ecdede15abc281`, from the night's run of 2026-10-26 (run
40369598264) and from both targets run again at `d2df8807` (runs 40453829649 and 40453835497). Each
reading holds on both targets unless it says otherwise.
reading holds on both targets unless it says otherwise, and under both Node.js versions a night's run
builds and serves the applications under (24 and 26) unless it names one. A reading for the lane's
own Node.js is tried before one for every Node.js: under Node.js 26 a suite can fail before it gets as
far as the failure its other reading records. The readings for Node.js 26 were read from the whole
manifest run under it on both targets on 2026-10-30 (runs 41023438620 and 41023442374).

### Deploy gates

Expand Down Expand Up @@ -121,8 +125,32 @@ runs on `next start` 16.4.3 with no adapter, and in 14 of 20 on each target; sta
every run. vercel/next.js#100776 starts the test from a page whose only links are the accordions'.
The judge reads only `act`'s own timeout ("Timed out waiting for a request to be initiated").

#### Preview mode on Node.js 26

Under Node.js 26, on `node` (12 cases of `prerender-preview`, `preview-fallback` and `prerender`).
`res.setPreviewData` signs its cookie with the `jsonwebtoken` Next.js 16.4.3 compiles in
(`next/dist/compiled/jsonwebtoken`). Its `buffer-equal-constant-time` reads
`require('buffer').SlowBuffer.prototype` as it loads, and Node.js 26 has no `SlowBuffer`.
`require('next/dist/compiled/jsonwebtoken')` throws on Node.js 26.13.0 with no adapter ("Cannot read
properties of undefined (reading 'prototype')"). So the route answers 500 and sets no preview cookie,
and the pages read without one render as no preview. v16.5.0-canary.30 signs with `fast-jwt`
instead. On `cloudflare`, workerd's `buffer` still has `SlowBuffer`, and the cases pass.

### The environment

#### Native addons on Node.js 26

Under Node.js 26, on both targets: `prerender-native-module` (3 of 3) and `turbopack-reports` (1 of
1). Their fixture's addon is built from source at install (`node-gyp rebuild`), and its `addon.cc`
calls `v8::Context::GetIsolate`, which the V8 of Node.js 26 no longer has. The install fails ("has no
member named 'GetIsolate'"), and with it the deploy hook, before anything of the adapter's runs.

#### `SlowBuffer` on Node.js 26

Under Node.js 26, on `node`: `edge-runtime-node-compatibility`'s two `node:buffer` cases expect
`SlowBuffer` among `node:buffer`'s keys. Node.js 26 removed it, and on `node` the edge runtime's
`node:buffer` is the Function's own. On `cloudflare` it is workerd's, which keeps it.

#### Native addons on `cloudflare`

`prerender-native-module` (3 of 3) and `turbopack-reports` (1 of 1) load a native Node.js addon from
Expand Down
4 changes: 4 additions & 0 deletions tools/deploy-tests/scripts/e2e-cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,8 @@
set -euo pipefail

tool_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# Under the Node.js the deploy hook ran under (`e2e-deploy.sh`).
if [ -n "${ADAPTER_TEST_NODE_DIR:-}" ]; then
export PATH="${ADAPTER_TEST_NODE_DIR}:${PATH}"
fi
node "${tool_dir}/src/main.ts" release || true
14 changes: 14 additions & 0 deletions tools/deploy-tests/scripts/e2e-deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ export ADAPTER_TEST_SUITE_ENV
# off (`src/hook.ts`): the build below spends the same time.
export ADAPTER_TEST_HOOK_STARTED_MS="${ADAPTER_TEST_HOOK_STARTED_MS:-$(node -p 'Date.now()')}"

# The Node.js the application is built and served under, where the run names one other than the
# harness's (`ADAPTER_TEST_NODE_DIR`, a directory holding `node`): first on the `PATH` of everything
# below, the server this starts included. A build and the server that loads it then agree on it, as
# the adapter requires of a native Function. Only after the suite's variables are read above, which
# would otherwise take this `PATH` for one of the suite's and hand it to the application.
if [ -n "${ADAPTER_TEST_NODE_DIR:-}" ]; then
if [ ! -x "${ADAPTER_TEST_NODE_DIR}/node" ]; then
echo "ADAPTER_TEST_NODE_DIR holds no node: ${ADAPTER_TEST_NODE_DIR}" >&2
exit 1
fi
export PATH="${ADAPTER_TEST_NODE_DIR}:${PATH}"
fi
echo "Node.js $(node --version) builds and serves the application" >&2

repo_root="$(cd "${tool_dir}/../.." && pwd)"
# The contract's own variable, and it names the adapter's repository: keep it working, since the
# workflow in Next.js's documentation sets it and people copy that workflow.
Expand Down
22 changes: 16 additions & 6 deletions tools/deploy-tests/src/judge-results.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import { parseArgs } from 'node:util';

import { fails, judge } from './judge.ts';
import { type JestSuite, jestSuite, listedSuites } from './results.ts';
import { REVIEWED } from './reviewed.ts';
import { describe, type ShardVerdict, shardFails } from './verdicts.ts';
import { isNodeLine, REVIEWED } from './reviewed.ts';
import { describe, laneOf, type ShardVerdict, shardFails } from './verdicts.ts';

/**
* Judges a shard of a run on this machine's targets (`judge.ts`) where it ran: the suites it was to
Expand All @@ -16,8 +16,8 @@ import { describe, type ShardVerdict, shardFails } from './verdicts.ts';
* left no result, or a runner that did not finish: one that wrote no exit code beside the results
* (`runner-exit-code.txt`), stopped at its step's time limit or never started.
*
* Where it ran, from the environment: the target (`TARGET`), the shard (`SHARD`) and the commit of
* Next.js (`NEXT_COMMIT`).
* Where it ran, from the environment: the target (`TARGET`), the Node.js of its lane where it has one
* (`NODE`), the shard (`SHARD`) and the commit of Next.js (`NEXT_COMMIT`).
*/
const { values } = parseArgs({
options: {
Expand Down Expand Up @@ -45,6 +45,12 @@ if (target !== 'cloudflare' && target !== 'node') {
`judge-results: TARGET is one of this machine's, cloudflare or node, not ${target}`,
);
}
// The lane's Node.js, where the run names one: a major version, as the plan's matrix has it.
const nodeAsked = process.env['NODE'];
const node = nodeAsked === undefined || nodeAsked === '' ? undefined : nodeAsked;
if (node !== undefined && !isNodeLine(node)) {
throw new Error(`judge-results: NODE is 24 or 26, the lanes' Node.js, not ${node}`);
}
const shard = Number(required('SHARD'));
if (!Number.isSafeInteger(shard) || shard < 1) {
throw new Error('judge-results: SHARD must be a positive whole number');
Expand Down Expand Up @@ -83,7 +89,11 @@ function endedByWatchdog(): string[] {

const verdict: ShardVerdict = {
...judge(
{ target, next: { version: next.version, commit: required('NEXT_COMMIT') } },
{
target,
...(node !== undefined && { node }),
next: { version: next.version, commit: required('NEXT_COMMIT') },
},
listed,
results,
REVIEWED,
Expand All @@ -93,7 +103,7 @@ const verdict: ShardVerdict = {
ended: endedByWatchdog(),
};
writeFileSync(path.join(resultsDir, 'verdict.json'), `${JSON.stringify(verdict, undefined, 2)}\n`);
const text = describe(target, [verdict]);
const text = describe(laneOf(target, node), [verdict]);
console.log(text);
const summary = process.env['GITHUB_STEP_SUMMARY'];
if (summary !== undefined && summary !== '') {
Expand Down
Loading
Loading