diff --git a/.github/actions/load-toolchain-pins/action.yml b/.github/actions/load-toolchain-pins/action.yml index e929f2a0..5a55fdba 100644 --- a/.github/actions/load-toolchain-pins/action.yml +++ b/.github/actions/load-toolchain-pins/action.yml @@ -6,4 +6,4 @@ runs: steps: - name: Load toolchain-pins.env shell: bash - run: bash scripts/load-toolchain-pins.sh "${GITHUB_ENV}" + run: grep -E '^[A-Z][A-Z0-9_]*=' toolchain-pins.env >> "${GITHUB_ENV}" diff --git a/.github/actions/setup-guest-toolchain/action.yml b/.github/actions/setup-guest-toolchain/action.yml index 81d21698..34c881c2 100644 --- a/.github/actions/setup-guest-toolchain/action.yml +++ b/.github/actions/setup-guest-toolchain/action.yml @@ -1,45 +1,23 @@ name: "Setup guest toolchain" -description: "Install system deps, xgenext2fs, cartesi-machine, QEMU/Buildx, Rust, just, and cross." +description: "Install system deps, QEMU/Buildx and Rust (rust-toolchain.toml), plus xgenext2fs, cartesi-machine, just and cross at the toolchain-pins.env versions (run ./.github/actions/load-toolchain-pins first)." inputs: - xgenext2fs-version: - description: "xgenext2fs release tag" - required: true - cartesi-machine-version: - description: "cartesi-machine release tag" - required: true - cartesi-machine-sha256-amd64: - description: "SHA-256 for the amd64 .deb" - required: true - cartesi-machine-sha256-arm64: - description: "SHA-256 for the arm64 .deb" - required: true - xgenext2fs-sha256-amd64: - description: "SHA-256 for the amd64 xgenext2fs .deb" - required: true - xgenext2fs-sha256-arm64: - description: "SHA-256 for the arm64 xgenext2fs .deb" - required: true install-foundry: description: "Whether to install Foundry (for anvil)" required: false default: "false" - foundry-version: - description: "Foundry version tag" + cache: + description: "Restore and save the Rust build cache and the QEMU binfmt image cache (release builds pass false)" required: false - default: "v1.5.1" - rust-toolchain: - description: "Rust toolchain version or channel" - required: false - default: "1.95.0" - rust-components: - description: "Extra rustup components (comma-separated)" - required: false - default: "rustfmt" + default: "true" runs: using: composite steps: + - name: Require toolchain pins + shell: bash + run: ': "${CARTESI_MACHINE_VERSION:?run ./.github/actions/load-toolchain-pins first}"' + - name: Install system dependencies shell: bash run: | @@ -57,16 +35,16 @@ runs: run: | set -euo pipefail ARCH="$(dpkg --print-architecture)" - VERSION="${{ inputs.xgenext2fs-version }}" + VERSION="${XGENEXT2FS_VERSION}" case "${ARCH}" in amd64) ASSET_NAME="xgenext2fs_amd64.deb" - EXPECTED_SHA="${{ inputs.xgenext2fs-sha256-amd64 }}" + EXPECTED_SHA="${XGENEXT2FS_SHA256_AMD64}" ;; arm64) ASSET_NAME="xgenext2fs_arm64.deb" - EXPECTED_SHA="${{ inputs.xgenext2fs-sha256-arm64 }}" + EXPECTED_SHA="${XGENEXT2FS_SHA256_ARM64}" ;; *) echo "unsupported arch for xgenext2fs: ${ARCH}" >&2 @@ -82,37 +60,35 @@ runs: - name: Install cartesi-machine uses: ./.github/actions/install-cartesi-machine with: - version: ${{ inputs.cartesi-machine-version }} - sha256-amd64: ${{ inputs.cartesi-machine-sha256-amd64 }} - sha256-arm64: ${{ inputs.cartesi-machine-sha256-arm64 }} + version: ${{ env.CARTESI_MACHINE_VERSION }} + sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} + sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + cache-image: ${{ inputs.cache }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + # rust-toolchain.toml is the only Rust pin. rustup resolves it from the + # working directory, so the cross guest build gets the same toolchain. - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ inputs.rust-toolchain }} - components: ${{ inputs.rust-components }} + shell: bash + run: rustup set profile minimal && rustup toolchain install - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + if: inputs.cache == 'true' + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Install Foundry if: inputs.install-foundry == 'true' - uses: foundry-rs/foundry-toolchain@v1 - with: - version: ${{ inputs.foundry-version }} - - - name: Install just - uses: taiki-e/install-action@v2 + uses: foundry-rs/foundry-toolchain@908c540300062bd5a7e473851cdb4282204cee09 # v1.9.1 with: - tool: just + version: ${{ env.FOUNDRY_VERSION }} - - name: Install cross - uses: taiki-e/install-action@v2 + - name: Install just and cross + uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 with: - tool: cross + tool: just@${{ env.JUST_VERSION }},cross@${{ env.CROSS_VERSION }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d3ef9f2..e696674c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,17 +1,37 @@ name: CI +# Pull requests, plus pushes to main and release tags; a branch is checked +# through its PR, so it does not also run on push. on: push: + branches: [main] + tags: ["v*"] pull_request: +# A new push to a PR cancels that PR's older run. Every other run is its own +# group, so runs on main and tags are never cancelled or queued. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# Builds are not incremental, and cargo output stays coloured in the logs. +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + jobs: rust: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins @@ -19,6 +39,9 @@ jobs: - name: Verify toolchain pin alignment run: bash scripts/verify-toolchain-pins.sh + - name: Verify action pins + run: bash scripts/verify-action-pins.sh + - name: Install system dependencies run: | sudo apt-get update @@ -30,7 +53,11 @@ jobs: liblua5.4-dev \ lua-check \ libcurl4-openssl-dev \ - libslirp-dev + libslirp-dev \ + shellcheck + + - name: Shell lint + run: shellcheck scripts/*.sh watchdog/sequencer-watchdog watchdog/test-guest/nvram.sh # canonical-test links the emulator through testsi's cartesi-machine # bindings, so the workspace build needs the pinned CM .deb and the @@ -42,19 +69,17 @@ jobs: sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} + # rust-toolchain.toml is the only Rust pin; it lists rustfmt and clippy. - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - components: rustfmt, clippy + run: rustup set profile minimal && rustup toolchain install - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 + uses: foundry-rs/foundry-toolchain@908c540300062bd5a7e473851cdb4282204cee09 # v1.9.1 with: - version: v1.5.1 + version: ${{ env.FOUNDRY_VERSION }} - name: Build run: cargo build --workspace --all-targets --locked @@ -66,7 +91,7 @@ jobs: run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - name: Watchdog Lua lint - run: cd watchdog && luacheck --no-color *.lua tests/*.lua + run: cd watchdog && luacheck --no-color *.lua tests/*.lua tests/fixtures/*.lua - name: Watchdog Lua unit tests run: | @@ -81,61 +106,66 @@ jobs: run: bash scripts/ci-c-application-smoke.sh openapi: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins - name: Install Node - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: 22 - name: Lint OpenAPI definition run: npx --yes "@redocly/cli@${REDOCLY_CLI_VERSION}" lint openapi.yaml - canonical-guest: - runs-on: ubuntu-latest - needs: rust - timeout-minutes: 45 + # Policy in deny.toml: crate sources block; advisories report without + # failing, since the RustSec database changes under a fixed Cargo.lock. + cargo-deny: + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins - - name: Setup guest toolchain - uses: ./.github/actions/setup-guest-toolchain + - name: Install Rust toolchain + run: rustup set profile minimal && rustup toolchain install + + - name: Install cargo-deny + uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 with: - rust-toolchain: ${{ env.RUST_TOOLCHAIN }} - xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }} - xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }} - xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }} - cartesi-machine-version: ${{ env.CARTESI_MACHINE_VERSION }} - cartesi-machine-sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} - cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} + tool: cargo-deny@${{ env.CARGO_DENY_VERSION }} - - name: Download canonical app deps - run: just canonical download-deps + - name: Crate sources + run: cargo deny --locked check sources - - name: Run guest tests - run: just canonical-test-guest + - name: Advisories (report only) + continue-on-error: true + run: cargo deny --locked check advisories rollups-e2e: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: rust timeout-minutes: 60 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins @@ -143,24 +173,19 @@ jobs: - name: Setup guest toolchain uses: ./.github/actions/setup-guest-toolchain with: - rust-toolchain: ${{ env.RUST_TOOLCHAIN }} - xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }} - xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }} - xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }} - cartesi-machine-version: ${{ env.CARTESI_MACHINE_VERSION }} - cartesi-machine-sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} - cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} install-foundry: "true" + # setup-guest-toolchain already ran apt-get update. - name: Install native test dependencies - run: | - sudo apt-get update - sudo apt-get install -y faketime libfaketime libclang-dev + run: sudo apt-get install -y faketime libfaketime libclang-dev libcurl4-openssl-dev build-essential pkg-config - - name: Build watchdog Lua deps - run: | - sudo apt-get install -y libcurl4-openssl-dev build-essential pkg-config - just watchdog-lua-deps + - name: Download canonical app deps + run: just canonical download-deps + + # Builds the devnet machine image, which the e2e recipe's + # ensure-machine-image then reuses, so CI builds it once. + - name: Run guest tests + run: just canonical-test-guest - name: Run rollups E2E tests (Rust and C hosts) run: just test-rollups-e2e @@ -172,7 +197,7 @@ jobs: - name: Upload E2E failure logs if: ${{ failure() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: rollups-e2e-logs-${{ github.run_attempt }} path: tests/e2e/results/*.log @@ -181,19 +206,21 @@ jobs: watchdog-docker: name: Watchdog Docker image smoke - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: rust timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build and smoke-test watchdog image run: bash scripts/ci-watchdog-docker-smoke.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 023a8647..d0b356f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,12 @@ # Release workflow: git tag v* builds wallet-sequencer tarballs, CM images, and pushes # watchdog OCI images to ghcr.io + docker.io (requires DOCKERHUB_USERNAME / # DOCKERHUB_TOKEN org or repo secrets, same as cartesi/cli). +# Release builds start cold: no step restores a cache that other refs wrote. +# Before tagging, run `bash scripts/verify-toolchain-pins.sh --upstream`, fix +# what it flags, then run `bash scripts/verify-action-pins.sh --upstream` and +# bump, or knowingly keep, each pin it flags. An action only this workflow uses gets +# no CI run, so a bumped one first runs at the tag; if it fails, fix it on main +# and dispatch this workflow for the same tag. name: Release @@ -11,7 +17,7 @@ on: workflow_dispatch: inputs: tag: - description: "Release tag (e.g. v0.1.0-alpha.1)" + description: "Existing release tag to rebuild (e.g. v0.1.0-alpha.1)" required: true type: string prerelease: @@ -20,14 +26,25 @@ on: default: true type: boolean +# Jobs that write declare it; everything else only reads. permissions: - contents: write - packages: write + contents: read + +# A tag push and a dispatch for the same tag run one after the other, never +# interleaving builds, image pushes or release uploads. +concurrency: + group: release-${{ inputs.tag || github.ref_name }} + cancel-in-progress: false + +# Builds are not incremental, and cargo output stays coloured in the logs. +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always jobs: build-wallet-sequencer: name: Build wallet-sequencer (${{ matrix.arch }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -38,8 +55,13 @@ jobs: target: aarch64-unknown-linux-gnu steps: + # A push builds the pushed commit; a dispatch builds the named tag, fully + # qualified so a branch or SHA input cannot stand in and a missing tag fails. - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.sha }} + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins @@ -56,19 +78,17 @@ jobs: pkg-config \ libssl-dev + # rust-toolchain.toml is the only Rust pin. - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - targets: ${{ matrix.target }} - - - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + run: | + rustup set profile minimal + rustup toolchain install + rustup target add ${{ matrix.target }} - name: Install cross - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 with: - tool: cross + tool: cross@${{ env.CROSS_VERSION }} # `sequencer` is a library crate; the runnable reference binary is the # `wallet-sequencer` example app (sequencer lib + the placeholder wallet). @@ -88,7 +108,7 @@ jobs: cp "target/${TARGET}/release/wallet-sequencer" "package/wallet-sequencer-${TAG}-linux-${ARCH}/wallet-sequencer" bash scripts/generate-release-manifest.sh \ --tag "${TAG}" \ - --git-sha "${GITHUB_SHA}" \ + --git-sha "$(git rev-parse HEAD)" \ --output "package/wallet-sequencer-${TAG}-linux-${ARCH}/RELEASE.json" cat > "package/wallet-sequencer-${TAG}-linux-${ARCH}/RUNNING.md" <<'EOF' @@ -115,17 +135,22 @@ jobs: tar -C package -czf "dist/wallet-sequencer-${TAG}-linux-${ARCH}.tar.gz" "wallet-sequencer-${TAG}-linux-${ARCH}" - name: Upload artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wallet-sequencer-linux-${{ matrix.arch }} path: dist/wallet-sequencer-*.tar.gz build-canonical-machine-image: name: Build canonical machine images - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: + # A push builds the pushed commit; a dispatch builds the named tag, fully + # qualified so a branch or SHA input cannot stand in and a missing tag fails. - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.sha }} + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins @@ -133,13 +158,7 @@ jobs: - name: Setup guest toolchain uses: ./.github/actions/setup-guest-toolchain with: - rust-toolchain: ${{ env.RUST_TOOLCHAIN }} - xgenext2fs-version: ${{ env.XGENEXT2FS_VERSION }} - xgenext2fs-sha256-amd64: ${{ env.XGENEXT2FS_SHA256_AMD64 }} - xgenext2fs-sha256-arm64: ${{ env.XGENEXT2FS_SHA256_ARM64 }} - cartesi-machine-version: ${{ env.CARTESI_MACHINE_VERSION }} - cartesi-machine-sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} - cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} + cache: "false" - name: Download canonical app deps run: just canonical download-deps @@ -162,17 +181,25 @@ jobs: tar -C examples/canonical-app/out -czf "dist/canonical-machine-image-sepolia-${TAG}.tar.gz" canonical-machine-image-sepolia - name: Upload artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: canonical-machine-images path: dist/canonical-machine-image-*.tar.gz build-watchdog-image: name: Build and push watchdog image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write # ghcr.io push steps: + # A push builds the pushed commit; a dispatch builds the named tag, fully + # qualified so a branch or SHA input cannot stand in and a missing tag fails. - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.sha }} + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins @@ -189,27 +216,37 @@ jobs: fi - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + cache-image: false + # The builder records the Actions context (run, ref, commit, actor, event + # payload) in the image's provenance. It stays on: everything in it is + # already public for this public repository, and verifiers use it. - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to GitHub Container Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Docker Hub - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Resolve release commit + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Build and push watchdog image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 env: + # Also skips the build-record artifact, which broke publish's download. DOCKER_BUILD_SUMMARY: "false" with: context: . @@ -221,32 +258,40 @@ jobs: docker.io/cartesi/sequencer-watchdog:${{ inputs.tag || github.ref_name }} build-args: | RELEASE_TAG=${{ inputs.tag || github.ref_name }} - GIT_COMMIT=${{ github.sha }} + GIT_COMMIT=${{ steps.commit.outputs.sha }} CARTESI_MACHINE_VERSION=${{ env.CARTESI_MACHINE_VERSION }} CARTESI_MACHINE_DEB_SHA256_AMD64=${{ env.CARTESI_MACHINE_SHA256_AMD64 }} CARTESI_MACHINE_DEB_SHA256_ARM64=${{ env.CARTESI_MACHINE_SHA256_ARM64 }} - cache-from: type=gha - cache-to: type=gha,mode=max publish: name: Publish GitHub Release - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - build-wallet-sequencer - build-canonical-machine-image - build-watchdog-image + permissions: + contents: write # create the release and upload its assets steps: + # A push builds the pushed commit; a dispatch builds the named tag, fully + # qualified so a branch or SHA input cannot stand in and a missing tag fails. - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.sha }} + persist-credentials: false - name: Load toolchain pins uses: ./.github/actions/load-toolchain-pins - name: Download build artifacts - uses: actions/download-artifact@v6 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: dist pattern: "{wallet-sequencer-linux-*,canonical-machine-images}" + # The default since v8: an artifact whose bytes changed after upload, in + # storage or transit, fails publish. + digest-mismatch: error - name: Flatten artifacts env: @@ -257,7 +302,7 @@ jobs: find dist -type f -name '*.tar.gz' -exec cp -v '{}' out/ \; bash scripts/generate-release-manifest.sh \ --tag "${TAG}" \ - --git-sha "${GITHUB_SHA}" \ + --git-sha "$(git rev-parse HEAD)" \ --output "out/release-manifest-${TAG}.json" - name: Generate checksums @@ -266,13 +311,30 @@ jobs: set -euo pipefail sha256sum *.tar.gz *.json > SHA256SUMS + # gh ships with the runner image, so no third-party code holds contents:write. + # A tag push creates the release. A dispatch rebuild finds it, published or + # draft, replaces its same-name assets, and publishes it if it was a draft. - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.tag || github.ref_name }} - prerelease: ${{ github.event_name == 'workflow_dispatch' && inputs.prerelease }} - fail_on_unmatched_files: true - files: | - out/*.tar.gz - out/*.json - out/SHA256SUMS + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + GH_TELEMETRY: "false" + TAG: ${{ inputs.tag || github.ref_name }} + PRERELEASE: ${{ (github.event_name == 'workflow_dispatch' && inputs.prerelease) || (github.event_name == 'push' && contains(github.ref_name, '-')) }} + run: | + set -euo pipefail + gh --version + # An asset glob that matches nothing fails here, before any API call. + ls -l out/*.tar.gz out/*.json out/SHA256SUMS + assets=(out/*.tar.gz out/*.json out/SHA256SUMS) + # A view that fails for another reason also falls through to create, whose + # check refuses to duplicate a published release. + if gh release view --json url -- "${TAG}" > /dev/null; then + gh release upload --clobber -- "${TAG}" "${assets[@]}" + gh release edit --prerelease="${PRERELEASE}" --draft=false -- "${TAG}" + else + # --verify-tag refuses to create a missing tag; gh uploads to a draft + # and publishes it last, deleting the draft if an upload fails. + gh release create --verify-tag --title "${TAG}" --notes "" \ + --prerelease="${PRERELEASE}" -- "${TAG}" "${assets[@]}" + fi diff --git a/AGENTS.md b/AGENTS.md index 1bdcb0a1..63753a79 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,8 @@ Start here for the repository's mental model and working rules. Read this baseline once, then use [Reading Routes](#reading-routes) to follow the contracts relevant to the work. Those documents own detailed behavior; this guide explains why it -matters and where to look. +matters and where to look. Every fact has one home +([Documentation Practice](#documentation-practice)). ## Mission @@ -32,35 +33,28 @@ In order of importance: - **App-specific sequencer.** The sequencer may link against the application, enabling validation and execution at ingress time. This is a deliberate design choice. - **Soft confirmations may be invalidated.** Under adversarial conditions (network, infrastructure, provider, or L1 outages), soft confirmations can be rolled back via recovery. This is by design, not a bug — it is what makes the sequencer sound in the face of liveness failures. - **App UX may depend on the sequencer.** Without the sequencer, user experience may degrade substantially. This is an acceptable tradeoff: the on-chain scheduler remains the canonical source of truth; the sequencer only accelerates the UX. -- **SQLite-centered local coordination.** Components publish and consume durable local facts through their owned SQLite tables. The on-chain scheduler remains canonical authority; SQLite is the sequencer's local coordination plane. HTTP ingress ↔ inclusion lane MPSC/oneshot is the deliberate exception because low-latency request/response over the lane's in-memory application is unwieldy through SQLite. Do not turn that exception into a general in-memory component bus. Full statement: [ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md). +- **SQLite-centered local coordination.** Components publish and consume durable local facts through their owned SQLite tables, and the on-chain scheduler remains canonical authority. HTTP ingress ↔ inclusion lane is the one deliberate in-memory exception, not a precedent for a component bus ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)). - **Append-oriented storage.** Avoid mutable status flags for open/closed entities; prefer write-once NULL→value transitions with one owner each, and derive cursors and heads from persisted facts rather than duplicating them as mutable fields. - **Assumption-driven robustness.** Every hardening mechanism must name the invariant it protects, the assumptions under which it is needed, and a trigger for revisiting them. Machinery for failures outside the supported model enlarges the state surface developers must audit and can make the system less robust rather than more. - **The complexity budget belongs to concurrency, mutual exclusion, durability, and hostile-L1 robustness.** This sequencer is not algorithmically complex. A large file is a smell: an invariant we're not seeing, a reasonable assumption we're not taking, or plain over-engineering. Judge every mechanism — current or proposed — against its weight. ## Sequencer / Scheduler Duality -The system has two components in an asymmetric relationship: - -### Scheduler — on-chain canonical authority - -The scheduler runs inside the rollup and **defines the canonical transaction ordering**. For each batch read from L1 safe inputs, it processes frames in order: drain all pending direct inputs whose block number is ≤ `safe_block`, then execute the frame's user ops. **The scheduler treats the sequencer as potentially Byzantine** — it enforces ordering and staleness rules regardless of what the sequencer claims. - -### Sequencer — off-chain predictor - -The sequencer knows the scheduler's algorithm. It uses that knowledge to **predict** what the canonical ordering will be once its batches land on L1, and issues soft confirmations to users ahead of time. The sequencer has **write priority on the execution queue**: as long as it keeps advancing `safe_block` and submitting batches, it controls ordering. - -### The `safe_block` synchronization primitive - -Each frame carries a `safe_block` chosen by the sequencer. It serves two purposes: - -- It tells the scheduler how far to drain direct inputs before executing the frame's user ops. -- It is the sequencer's commitment that it has accounted for all direct inputs up to that block. - -The sequencer must advance `safe_block` honestly. If it freezes `safe_block` (to censor deposits) or stops submitting batches, the staleness mechanism detects this and forces recovery. - -### When soft confirmations match canonical order - -Under honest sequencer operation and no infrastructure outages, soft confirmations match the canonical order. This is an **optimistic guarantee** — the sequencer is predicting a future the scheduler has not yet computed. When the sequencer goes offline, submits stale batches, or tries to censor direct inputs, the scheduler's force-drain backstop kicks in and the affected soft confirmations become invalid. +The **scheduler** runs inside the rollup and **defines the canonical transaction +ordering**. For each batch in L1 safe inputs, it processes frames in order: +drain every pending direct input whose block number is ≤ `safe_block`, then +execute the frame's user ops. **It treats the sequencer as potentially +Byzantine** and enforces ordering and staleness regardless of what the +sequencer claims. + +The **sequencer** knows that algorithm and **predicts** the canonical ordering, +issuing soft confirmations before its batches land. It has **write priority on +the execution queue** while it keeps advancing `safe_block` and submitting +batches. A frame's `safe_block` tells the scheduler how far to drain directs +before the frame's user ops, and commits the sequencer to having accounted for +every direct input up to that block. Under honest operation without outages, +soft confirmations match the canonical order; if the sequencer goes offline or +freezes `safe_block`, staleness forces recovery and invalidates them. ### Where the duality lives in code (change one, check all) @@ -75,10 +69,9 @@ Scheduler-acceptance semantics exist in exactly three implementations that must 2. the off-chain acceptance predicate — `ProtocolTiming::scheduler_accepts` ([`sequencer-core/src/protocol.rs`](sequencer-core/src/protocol.rs)), which feeds `safe_accepted_batches`; 3. the inclusion lane's live prediction (drain + execution order). -The submitter's expected-nonce scan also depends on this agreement. The -[scheduler contract](docs/protocol/scheduler-semantics.md#the-three-implementations-and-why-they-agree) -maps the implementations and the deliberate storage-local copy. Changing one -requires checking the others. +[I1](docs/invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) +maps them and their one deliberate difference. Changing one requires checking +the others. Two mechanical facts the agreement rests on: @@ -87,47 +80,21 @@ Two mechanical facts the agreement rests on: ## Batch Staleness and Recovery -### Staleness - -A batch is **stale** when `inclusion_block - first_frame.safe_block >= MAX_WAIT_BLOCKS` (1200 blocks, ~4h). Staleness catches two failure modes: - -1. **Liveness failure** — the sequencer went offline and failed to submit batches in time. -2. **Censorship** — the sequencer kept submitting batches but froze `safe_block` to hold back direct inputs. - -When the scheduler encounters a stale batch, it skips its frames without -consuming the batch nonce. The overdue-direct backstop still runs. Together, -these rules prevent the sequencer from holding write priority indefinitely -without advancing the drain cursor; direct inputs are force-drained at -`MAX_WAIT_BLOCKS`, giving the ~4h censorship-resistance bound. - -### Cascading invalidation - -If a batch is stale, all existing subsequent batches are also invalid. The scheduler's expected-nonce counter does not advance on a stale skip, so every subsequent batch arrives at an unexpected nonce and is rejected. Invalidation is a suffix operation: marking batch `N` invalid cascades to `N+1`, `N+2`, …, including the open batch. New batches created after recovery are unaffected. - -### Two recovery paths - -**Automatic recovery** repairs optimistic history after liveness failures. The -danger detector signals service to stop when a danger check fires; startup recovery -settles outstanding submissions and replaces the invalid suffix using local -SQLite facts and safe L1 history. The danger threshold is a trigger, not proof -that a batch is doomed. Detection uses safe state, with wall-clock checks when -the L1 view stops advancing. Expected recovery exits gracefully; diagnosed -terminal runtime faults abort immediately. The [automatic recovery design](docs/recovery/README.md) -owns detection, startup ordering, dispatch, and admission. - -**Manual cockroach recovery** (`setup --recovery`) rebuilds from a trusted -canonical application checkpoint and L1 history into a fresh data directory. -It also applies after sequencer bugs: fix the bug, choose a trusted canonical -checkpoint, and then run recovery. Historical batches, -including malformed ones, receive the canonical scheduler's treatment. The -result accounts for a fixed input prefix from which sequencing can resume; it -does not need to catch the moving L1 tip or preserve prior soft confirmations. -The [cockroach recovery guide](docs/recovery/cockroach.md) owns the checkpoint -requirements, stopping boundary, and rebuild procedure. - -Before changing recovery code, read its guide and both current bounded TLA+ -models. The automatic recovery guide's [Formal Verification](docs/recovery/README.md#formal-verification) -section explains their scopes; the models are not a proof of every recovery path. +A batch is **stale** when `inclusion_block - first_frame.safe_block >= MAX_WAIT_BLOCKS` +(1200 blocks, ~4h): the sequencer went offline, or kept submitting while +freezing `safe_block` to hold back directs. The scheduler skips a stale batch +without consuming its nonce, and overdue directs are force-drained anyway, which +bounds censorship at ~4h. Every later batch then arrives at an unexpected nonce, +so invalidation cascades through the suffix, open batch included +([scheduler semantics](docs/protocol/scheduler-semantics.md)). + +- **Automatic recovery** repairs optimistic history after liveness failures: the + danger detector stops the process at a threshold before staleness, and startup + settles outstanding submissions and replaces the invalid suffix + ([automatic recovery](docs/recovery/README.md)). +- **Cockroach recovery** (`setup --recovery`) rebuilds a fresh data directory from + a trusted canonical checkpoint and L1 history, including after a sequencer bug + ([cockroach recovery](docs/recovery/cockroach.md)). ## Threat Model (brief) @@ -135,11 +102,8 @@ See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the full mo - **Trusted:** InputBox contract, our own Ethereum node (fail-stop, not byzantine), operator config, batch-submitter key. - **Adversarial:** `POST /tx` callers, direct-input senders, the L1 mempool and block builders (zombie transactions are a first-class threat). -- **RPC endpoint:** single (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`), trusted fail-stop, **must be one consistent node** — no fallback tier exists yet (see the threat model's actor table). -- **Self-trust:** normal operation assumes the sequencer's own code is correct. - Invariant violations fail loud. Bug-induced malformed batches require fixing - the bug and, when rebuilding is necessary, manual cockroach recovery; automatic - recovery does not repair software defects. +- **RPC endpoint:** single (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`), trusted fail-stop, **must be one consistent node** — no fallback tier exists yet ([RPC endpoint](docs/threat-model/README.md#rpc-endpoint)). +- **Self-trust:** normal operation assumes the sequencer's own code is correct; a bug needs a fix and, when state cannot be trusted, cockroach recovery. Automatic recovery does not repair software defects. - **In scope:** correctness bugs *and* exploitation. Under rollup semantics, a correctness bug that causes scheduler/sequencer state divergence is as severe as direct theft. ## Architecture Map @@ -163,6 +127,7 @@ signing, and SSZ batch encoding. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. - `sdk/guest/` — canonical-guest SDK: `trolley` (guest rollup API over `libcmt-sys`), `rollups-types` (rollups ABI encodings), and `testsi` (host-side guest image tests); provenance in its README. +- `watchdog/` — Lua watchdog: re-derives canonical state from L1 and compares it with the sequencer's accepted checkpoint ([watchdog README](docs/watchdog/README.md)). - `tests/{benchmarks,e2e,harness}/` — test infrastructure. ### Sequencer module layout @@ -184,93 +149,23 @@ Paths below are relative to `sequencer/src/`: - **Chunk** — bounded list of user ops processed and persisted together to amortize SQLite cost. - **Frame** — ordering boundary; commits `safe_block` + user ops. - **Batch** — list of frames posted on-chain as one L1 transaction (SSZ-encoded). -- **Inclusion lane** — the single ordering lane, with a latency-critical user-op regime and a slower L1-reconciliation regime ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)); the only writer of open batch/frame state ([I17](docs/invariants.md)) and the system's execution bottleneck. -- **Batch submitter** — stateless worker that bulk-submits all pending batches each tick. Storage assigns each batch's scheduler nonce at creation (`parent.nonce + 1`, or the deployment anchor for a root); the submitter reads it and selects L1 wallet nonces for submission. -- **Danger detector** — polls `Storage::check_danger` and signals the process to stop so startup can recover or refuse. It reads local facts; it never writes the DB or talks to L1. -- **Fee oracle** — setup pins and bootstraps a fixed price or Uniswap V3 TWAP source. The price informs future frame fees; an oracle-only outage is an accepted economic risk. The [threat model's actor table](docs/threat-model/README.md#actors-and-trust) owns the source assumptions and failure policy. -- **Input reader** — ingests safe inputs from L1 InputBox and maintains the durable safe head, accepted-batch projection, and divergence marker in one atomic transaction (`sequencer/src/storage/l1_inputs.rs`); it hands the lane no in-memory cursor. -- **L2 tx feed** — DB-backed application-input stream. HTTP snapshot headers - provide `(EraId, RecoveryGeneration, ExecutedInputCount)`; WS validates that - claim and replays inclusively before following the tip. -- **Application progress** — engine-owned `(ExecutedInputCount, - last_executed_safe_block)`, embedded in every dump. Shared execution verifies - the transition and returns the pre-execution offset; storage commits that - mandatory offset with its source in `application_inputs`. -- **History version** — `(EraId, RecoveryGeneration)`. Setup publishes a complete - baseline with a fresh era; recovery increments the generation exactly once - iff it invalidates at least one valid batch and records the preserved-prefix - cut in the same transaction. `/history` can check a saved checkpoint across - intervening generations; subscription claims still enforce both identifiers. - The [history contract](docs/protocol/application-history.md) owns compatibility. +- **Tip** — the open batch at the end of the valid branch; recovery invalidates a suffix and opens a new Tip ([automatic recovery](docs/recovery/README.md#the-state-being-repaired)). +- **Inclusion lane** — the single ordering lane ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)); the only writer of open batch/frame state ([I17](docs/invariants.md)) and the system's execution bottleneck. - **Soft confirmation** — sequencer's predicted ordering, emitted before the batch lands on L1. -- **Snapshot** — immutable artifact at every batch close, registered with its - local batch identity and application count. Acceptance facts select the - recovery/watchdog checkpoint; the era baseline supplies the initial restore - point. Lifecycle and leases: [`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md). +- **Snapshot** — immutable artifact registered at every batch close; acceptance selects the comparison and recovery checkpoint ([snapshot lifecycle](docs/snapshots/lifecycle.md)). ## Domain Truths - API validates the EIP-712 signature and enqueues a `SignedUserOp`. Method payload decoding happens during application execution, not at ingress. - **Deposits are direct-input-only** (L1 → L2) and must not be represented as user ops. -- Rejections (`InvalidNonce`, `NonceExhausted`, `InvalidMaxFee`, `InsufficientFeeBalance`) produce no state mutation and are not persisted. These are protocol-level rejection semantics every app must implement: nonces prevent user-op replay, fees prevent spam against the sequencer's DA budget. ("Fee", not "gas" — the fee tracks DA; compute metering, if it ever exists, is a separate future concept.) -- **The user-nonce rule is contract, not wallet detail:** accounts start at 0, an op must carry exactly the expected nonce, each included op advances it by one, and direct inputs never touch it. `GET /nonce` derives from persisted user ops under this rule instead of asking the app ([application contract](docs/protocol/application-contract.md#user-nonces)). -- Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `safe_inputs`, and `application_inputs`. Recovery metadata lives in `safe_accepted_batches`; batch lifecycle state (sealed/invalidated) lives on the `batches` row itself as write-once timestamps. -- Frame fee is persisted in `frames.fee` and is fixed for the lifetime of that frame. The next frame's fee is currently sampled from `batch_policy_derived.recommended_fee` at rotation; oracle bootstrap writes the price before any Tip can sample it, and `log_slack` applies the 10× margin in log space. This is present behavior, not a reason for the five-block clock policy; hoisting fee to the batch is a later design with its own trade-offs. -- Wallet balances and nonces live in memory between checkpoints; restart restores - a dump and replays persisted application inputs. -- **EIP-712 domain fields:** `name`, `version`, `chainId`, `verifyingContract`. - Setup pins the chain id and app address from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` - and `CARTESI_SEQUENCER_APP_ADDRESS`, validating the chain id against RPC. All - four fields must be present on both sides; the sequencer and canonical - scheduler share `sequencer_core::build_input_domain`. - -### InputBox payload classification - -- The input reader ingests every `InputAdded` event from InputBox. Each event carries an authenticated `msg_sender` (delivered by the Cartesi framework from `EvmAdvanceCall`). -- **Classification is by sender address**, not by a tag byte: - - Sender == batch-submitter address → SSZ-decoded as `Batch` (scheduler side). The sequencer does not ingest its own batch submissions as direct inputs. - - Any other sender → stored verbatim as a direct input (deposit). -- The payload is opaque to the classification layer. Application-specific decoding happens inside `Application::apply_direct_input`, reached only through the shared `execute_direct_input` boundary. +- **InputBox payload classification** is by authenticated sender, never by a tag byte: the batch submitter's inputs are batches; any other sender's are direct inputs, stored verbatim ([scheduler semantics](docs/protocol/scheduler-semantics.md#the-one-stream-the-one-classification)). +- A rejected user op changes no state and is not persisted; the [application contract](docs/protocol/application-contract.md) owns which layer produces each rejection. The charge is a "fee", not "gas": it tracks DA. +- **The user-nonce rule is contract, not wallet detail** ([user nonces](docs/protocol/application-contract.md#user-nonces)); `GET /nonce` relies on it instead of asking the app. +- **EIP-712 domain:** both sides build it with `sequencer_core::build_input_domain` ([user-op signatures](docs/protocol/scheduler-semantics.md#user-op-signatures)); setup pins the chain id and app address, validating the chain id against RPC. ## Application Trait Contract -Implementors of the `Application` trait must respect these contracts. The shared execution boundary verifies application-owned count/clock progress; application-specific determinism and mutation remain self-trusted. The full, code-grounded contract — method table, dump round-trip durability, the safe-block clock — is **owned by [`docs/protocol/application-contract.md`](docs/protocol/application-contract.md)**; the essentials follow. - -### Replay determinism - -The sequencer persists every included user op and every ingested direct input. On restart, catch-up replays them in order against a fresh `Application` instance to rebuild state. **Any input that succeeded live must succeed on replay.** - -- `apply_direct_input` and `apply_valid_user_op` must not return `AppError::Internal` for any byte sequence that previously executed successfully. The canonical scheduler, catch-up, and recovery fold treat `Internal` as fatal: no canonical successor is defined. -- Validation returns `Accept`, `Reject(InvalidReason)`, or fatal `AppError`. A rejected op changes no state. Included business failures and malformed direct-input no-ops still advance progress; do not turn them into validation rejections. See the application contract for the nonce rule and the wallet's fee semantics. -- `validate_user_op` must be pure over the current app state. No side effects, no time dependence, no randomness. - -### No implicit state - -Logical state changes, including `ApplicationProgress`, flow through the `apply_valid_user_op` and `apply_direct_input` hooks. The engine reports progress by value. Mutating state from `validate_user_op` breaks replay determinism; mutable checkpoint creation may replace backing resources while preserving logical state. - -### One execution entry point - -User ops are executed only through `sequencer_core::application::validate_and_execute_user_op`; already-validated user ops and directs use `execute_valid_user_op` / `execute_direct_input`. The shared boundary preflights the checked successor, then verifies the engine's progress after a successful hook and returns its pre-execution offset. Count zero implies clock zero. Validation purity and native mutation remain self-trusted. `AppError` is fatal and defines no canonical successor; callers discard the instance rather than resume it. The inclusion lane, canonical scheduler, catch-up, and recovery fold all use this boundary — part of the duality agreement. - -`Application` requires `Send`, with neither `Clone` nor `Sync`. Dumps must be durable and immutable, and restored engines must remain independent after source deletion. The opaque app prefix may be a file or directory; checkpoint disposal uses ordinary recursive filesystem deletion. Canonical inspection belongs to the separate `CanonicalState` trait; the native sequencer serves the comparison file in the checkpoint. The [C binding guide](docs/protocol/c-application-binding.md) maps the contract to native engines. - -## Ordering and Storage - -Preserve single-lane deterministic ordering. Do not introduce extra concurrency -in hot-path ordering logic without explicit approval. - -The inclusion lane combines a latency-critical user-op regime with complete L1 -reconciliation. The [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md) -owns that split, acknowledgement rules, runtime ownership, and command admission. -The [scheduler contract](docs/protocol/scheduler-semantics.md#sequencer-frame-clock-policy) -owns the five-safe-block frame clock; the [Application contract](docs/protocol/application-contract.md#5-operational-capacity-for-l1-reconciliation) -owns the assumption that accumulated directs are digestible without preemption. - -Storage changes cross writer boundaries even when the SQL looks local. Read the -[invariant register](docs/invariants.md) for writer ownership, `valid_*` reads, -drain attribution, the content-identity/divergence freeze, `WriteHead` coherence, -and application-history offsets. The [schema](sequencer/src/storage/migrations/0001_schema.sql) -enforces write-once batch lifecycle, Tip uniqueness, and user-op identity. +The [application contract](docs/protocol/application-contract.md) owns the `Application` trait; the [C binding guide](docs/protocol/c-application-binding.md) maps it to native engines. Two rules carry the duality: **any input that succeeded live must succeed on replay** (`AppError` is fatal and defines no canonical successor), and the inclusion lane, canonical scheduler, catch-up, and recovery fold execute inputs only through the shared boundary in `sequencer_core::application`, which checks the engine's reported progress. ## Type Boundaries @@ -285,92 +180,75 @@ enforces write-once batch lifecycle, Tip uniqueness, and user-op identity. ## HTTP Endpoints -- **Ingress** (public-facing): `POST /tx`, `GET /fee`, `GET /nonce`, `GET /domain`. -- **Egress** (internal indexers/watchdog): application-input subscriptions, - snapshot/state downloads, and health probes. Snapshot/state endpoints have no - authentication and **must not be exposed publicly**. Downloads hold a GC lease - for their response lifetime ([snapshot lifecycle](docs/snapshots/lifecycle.md)). - -Today both sides serve from one listener; the planned API split puts each side on its own port (same binary) so internal probes and subscribers can be firewalled from public submit traffic. - -The [README API contract](README.md#api) owns routes, message shapes, caps, -close codes, and health semantics. [`openapi.yaml`](openapi.yaml) is its -machine-readable rendering for client generation and documentation tools; -change both in the same patch. CI lints it; run -`direnv exec . redocly lint openapi.yaml` locally. +Only the ingress routes (`POST /tx`, `GET /fee`, `GET /nonce`, `GET /domain`) are +public; every other route, snapshot and state downloads included, is internal, +unauthenticated, and **must not be exposed publicly**. The [README API contract](README.md#api) +owns routes, listeners, shapes, caps, and codes; [`openapi.yaml`](openapi.yaml) +renders it for client generation and documentation tools, so change both in the +same patch. CI lints it; run `direnv exec . redocly lint openapi.yaml` locally. ## Command Configuration -Configuration follows the command phases: - -- Plain **`setup`** is L1-read-only and never signs. It pins chain/app/submitter - identity and the fee source. **`setup --recovery`** also needs the submitter - key to flush its outstanding transactions; see the [rebuild guide](docs/recovery/cockroach.md). -- **`run`** takes the RPC endpoint and signing key (or key file). It reads the - pinned chain id, app address, and submitter address from the database. - -**Use a dedicated submitter address.** Plain setup refuses an unsettled wallet -nonce, so sharing a busy contract-deployer address can trip the detection gate -while its deployment transactions are not yet safe. The devnet uses Anvil -account 9 (`DEVNET_SEQUENCER_ADDRESS`), separate from the account-0 deployer. - -The [Running guide](README.md#running) gives invocation examples; -[`commands/config.rs`](sequencer/src/commands/config.rs) owns environment-variable -names, defaults, and validation. Check configuration there before documenting or -changing it, including checkpoint and fee-source selection. +Plain `setup` is L1-read-only and never signs; `run` and `flush-mempool` take the +signing key and read the pinned identity from the database. [Running](README.md#running) +owns invocation and why the submitter address must be dedicated (the devnet uses +Anvil account 9). [`commands/config.rs`](sequencer/src/commands/config.rs) owns +every variable name, default, and validation; check it before documenting or +changing configuration. ## Coding Conventions -- Prefer small, composable functions at module boundaries (`ingress::api` → `ingress::inclusion_lane` → `storage::ingress`; `egress::l2_tx_feed` ← `storage::egress`). +- Prefer small, composable functions at module boundaries (`ingress::api` → `ingress::inclusion_lane` → `storage::ingress`; `egress::l2_tx_feed` ← `storage::history`). - Keep application validation and execution deterministic for a given input/state. No `SystemTime::now()`, `HashMap` iteration order, or floating-point in consensus paths. -- Surface user-facing errors via `ApiError` (in `http.rs`); keep internal failures descriptive but safe. +- Surface user-facing errors via `ApiError` (in `http.rs`); keep internal failures descriptive but safe. An error that wraps a cause ends its Display with `: {source}`, because log sinks print Display only. A dependency error whose Display omits its cause (reqwest names only the request kind) is rendered with `l1::provider::error_chain` where it becomes text, in a `String` variant or a log field; it appends only causes the line does not already hold, so it is also safe on our own wrappers of such an error. A failure logs at its verdict's level: ERROR when the cause is a fault or unknown (terminal, unclassified), WARN when the verdict names a planned procedure (a recovery boot, a retry, `setup --recovery`). - Avoid introducing heavy dependencies without strong reason. -- Documentation style: lean. Module headers (1–4 lines) + docs on public methods only when the contract isn't obvious from name+signature. -- **Comment the non-obvious, not the self-evident.** Keep comments concise; avoid redundant and excessive inline commentary. Do not restate what the code already expresses. Explain the why, edge cases, invariants, and subtle behaviors that cannot be inferred from reading the code alone. -- Review-item codenames (finding/decision ids from past review ledgers) never appear in code comments or living docs — state the reason itself, or point at the invariant register entry that owns it. Invariant ids (`I1`–`I20`) are stable register references and are fine. -- **Impossible states fail loud; they are never handled.** Cheap cross-module assertions of *real invariants* are encouraged (assert, trigger `RAISE`, typed error). Failing loud is safety-preserving, not necessarily self-healing: transient failures may clear on restart, while a persistent invariant violation is terminal and may require inspection or cockroach recovery. Silent divergence is never acceptable. Never add graceful fallbacks, neighbor re-validation, or silent absorbers (`INSERT OR IGNORE`, saturating decode of impossible data) for states the contracts rule out; and an assertion must check a real invariant, never an environmental assumption (clock monotonicity is the cautionary tale). Decision test and rationale: [`docs/invariants.md`](docs/invariants.md); trust boundaries: "Self-trust" in [`docs/threat-model/README.md`](docs/threat-model/README.md). +- **Impossible states fail loud; they are never handled or absorbed.** The [check policy](docs/invariants.md#the-check-policy-fail-loud) owns the decision test and its reasons; [Self-trust](docs/threat-model/README.md#self-trust) owns the trust boundary. ## Documentation Practice Write for a reader building a mental model. Start with the purpose, input, result, and governing constraint; introduce implementation detail when it explains a necessary behavior. Keep an intentional baseline here so important -concepts are discoverable before anyone knows to ask about them. Summaries -should point to the owner of a contract rather than becoming a second copy. +concepts are discoverable before anyone knows to ask about them. + +- **One fact, one home.** Elsewhere, name the fact in a clause and link its owner; never restate its conditions, orderings, or lists. +- **Surprising facts go into the owning doc**, so the next reader does not rediscover them. +- **Size caps.** A module header (Rust `//!`, Lua `---`) is 1–4 lines and a rustdoc contract about 6; public methods get docs only when name and signature leave the contract unclear. Longer reasoning goes to the owning doc or invariant, linked. +- **Comment the non-obvious, not the self-evident.** Explain the why, edge cases, invariants, and subtle behavior the code cannot show; do not restate what it expresses. +- **State a non-guarantee once**, at its owner, and only where a reader would otherwise assume the guarantee. +- **Table cells hold one sentence.** +- **Record deliberate absence once**, at the seam where someone would re-add the mechanism, phrased as a positive design statement with its reason. Avoid removal notices scattered across documents. +- **No review codenames** (finding or decision ids from past review ledgers) in code comments or living docs: state the reason, or point at the invariant register entry that owns it. Invariant ids (`I1`–`I20`) are stable references and are fine. Keep three kinds of material distinct: -- **Current contracts and designs** describe what holds now and why. Use present - tense and reasoning inline, without amendment banners, review codenames, or - "previously/no longer" narration. Each topic has one owner. Some current - architecture documents live in `docs/plans/`; the directory name does not - make their established contracts optional. -- **Active plans** name open decisions, dependencies, and remaining work. They - must distinguish proposed behavior from implemented contracts. On completion, - put the durable design in its owner and reduce the plan to its remaining work - and links. -- **Review notes** are temporary working memory. Commit them when they help an - active review or handoff, then distill and delete them when that work ends. - The [review lifecycle](docs/review/README.md) owns the policy: unresolved work - stays in one register or active plan, durable reasoning in its current owner, - completed history in Git. Keep dated evidence only for a named ongoing use. - -**Record deliberate absence once**, at the seam where someone would re-add the -mechanism, phrased as a positive design statement with its reason. Avoid removal -notices scattered across documents. +- **Current contracts and designs** describe what holds now and why, in present + tense, without amendment banners or "previously/no longer" narration. Some + current architecture documents live in `docs/plans/`; the directory name does + not make their established contracts optional. +- **Active plans** name open decisions, dependencies, and remaining work, and + distinguish proposed behavior from implemented contracts. On completion, put + the durable design in its owner and reduce the plan to remaining work and links. +- **Review notes** are temporary working memory. The [review lifecycle](docs/review/README.md) + owns where their conclusions go: unresolved work to the [register](docs/review/register.md), + durable reasoning to its owner, completed history to Git. ## Testing Guidance -Focus tests on: - -- Signature + sender-validation edge cases. -- Nonce progression rules. -- Fee and rejection behavior. -- Included-vs-rejected commit behavior. -- Storage batch atomicity and uniqueness constraints. -- Scheduler/sequencer agreement — any invariant the two sides share should have at least one test that exercises both. - -Prefer black-box tests around `POST /tx` and commit outcomes for integration. +Concentrate tests where consensus risk lives: signature and sender validation, +nonce progression, fee and rejection behavior, included-vs-rejected commits, and +storage batch atomicity and uniqueness. Any invariant the scheduler and +sequencer share gets at least one test that exercises both sides. Prefer +black-box tests around `POST /tx` and commit outcomes for integration. + +Test placement: unit tests are a child module of the code under test, inline +`#[cfg(test)] mod tests` or `/tests.rs` (or `_tests.rs`) once +large, never `#[path]`. Cross-module in-process tests go in +`sequencer/src/integration_tests/`; devnet, machine and binary tests go in the +`tests/*` crates. Shared fixtures live in `crate::test_support`; +`storage::test_helpers` keeps the storage-domain ones (seeding, the recovery +fixture trait, divergence markers); cross-crate signing helpers live in +`sequencer_core::test_support` behind its `test-support` feature. Some `sequencer` tests use Anvil (Foundry). They run by default and fail with a clear message if `anvil` is not on PATH. Use the configured Nix/direnv environment @@ -382,11 +260,15 @@ both) and libslirp. Use the configured Nix/direnv environment for Foundry, TLA+, and other project tools. For noninteractive commands, prefer `direnv exec . `. Rust is -pinned to **1.95.0** in [`rust-toolchain.toml`](rust-toolchain.toml); verify both -`cargo --version` and `rustc --version` in the environment you use. A Nix-provided +pinned in [`rust-toolchain.toml`](rust-toolchain.toml), the only Rust pin (CI and +release install from it); verify both `cargo --version` and `rustc --version` in +the environment you use. A Nix-provided Cargo or rustc may bypass rustup, so direnv alone does not guarantee the pinned compiler is selected. Correct the toolchain selection before interpreting build -or dependency errors. +or dependency errors. `cross` builds the guest with the channel's +`x86_64-unknown-linux-gnu` toolchain on every host; the guest recipes install +it, with the riscv64 target and `rust-src`, before each build +(`just canonical cross-toolchain`). ```bash direnv exec . cargo check @@ -409,7 +291,7 @@ See [Running](README.md#running) for the two-phase `setup` / `run` workflow. - Run at least `cargo check` before finishing. - Read the relevant recovery guide and both current TLA+ models before touching recovery code, and the threat model before touching trust-boundary code. -- Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing, the owning design for its assumptions, and [`docs/review/register.md`](docs/review/register.md) for unresolved work in the code you're about to touch. Verify review claims against current code. +- Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing (storage changes cross writer boundaries even when the SQL looks local), the owning design for its assumptions, and [`docs/review/register.md`](docs/review/register.md) for unresolved work in the code you're about to touch. Verify review claims against current code. ### Ask First @@ -457,8 +339,9 @@ work reaches another boundary. | API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior, with [`openapi.yaml`](openapi.yaml) changed alongside it; [application history](docs/protocol/application-history.md) — era, generation, offsets, and recovery boundaries. | | Snapshots, restart, export, retention, or watchdog checkpoints | [Snapshot lifecycle](docs/snapshots/lifecycle.md) — durable publication, accepted comparison points, leases, and GC; [wallet format](docs/snapshots/format.md) when changing wallet bytes. | | Trust boundaries, provider behavior, or hostile L1 input | [Threat model](docs/threat-model/README.md) — actor assumptions, supported failures, and residual risks. | -| Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat-model actor table](docs/threat-model/README.md#actors-and-trust) — oracle source and outage assumptions. | +| Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat model's fee oracle section](docs/threat-model/README.md#fee-oracle) — oracle source and outage assumptions. | | Command setup or deployment configuration | [Running](README.md#running) and [config.rs](sequencer/src/commands/config.rs) — invocation, identity pinning, defaults, and validation. | | Watchdog development or operation | [Watchdog README](docs/watchdog/README.md) — contract, state sources, commands, and state; [incident runbook](docs/watchdog/incident-runbook.md) — what a latched divergence requires; [local dev](docs/watchdog/getting-started.md) for Anvil; [operator deployment](docs/watchdog/operator-deployment.md) for Sepolia/mainnet. | | Canonical guest, machine images, or a Cartesi Machine bump | [Cartesi Machine facts](docs/cartesi-machine.md) — version pinning, host semantics, storage, memory ranges, and the guest side. | -| A new mechanism, simplification, or work spanning an active track | Owning design and [invariants](docs/invariants.md) — reasons and assumptions; [review register](docs/review/register.md) — unresolved work; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — priorities and dependencies. Follow the [review lifecycle](docs/review/README.md) when recording conclusions. | +| A release, or adding or bumping a GitHub Action | [`release.yml`](.github/workflows/release.yml) header — what to run before tagging; [`verify-action-pins.sh`](scripts/verify-action-pins.sh) — pin form, upstream audit, and why bumps are manual. | +| A new mechanism, simplification, or work spanning an active track | Owning design and [invariants](docs/invariants.md) — reasons and assumptions; [review register](docs/review/register.md) — unresolved work; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — priorities and dependencies. | diff --git a/Cargo.lock b/Cargo.lock index 7ff8e9c4..afbc7d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.2.36" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36ddb69f5e41407e7a93aead3480f7c8ff076e73d01a951ae8f7ea4542c1ca0" +checksum = "c1c4f6f6f020f38a4ef079428b5c7009e913bf8102e4d65a90c6cb2cf93800eb" dependencies = [ "alloy-primitives", "num_enum", @@ -67,12 +67,12 @@ dependencies = [ "either", "k256", "once_cell", - "rand 0.8.7", + "rand 0.8.8", "secp256k1 0.30.0", "serde", "serde_json", "serde_with", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -108,7 +108,7 @@ dependencies = [ "futures", "futures-util", "serde_json", - "thiserror 2.0.19", + "thiserror", "tokio", "tracing", "wasmtimer", @@ -116,9 +116,9 @@ dependencies = [ [[package]] name = "alloy-core" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8421a5ee9019b6d89f92935d0f8facd9f6ca088b41d7cc47244a02ccde4545f" +checksum = "c6a70852055d66af821d28497e690c95950020ed1ee4c7a0ab9b02a01ac75ee3" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a04eb4abc2b5074a18e687ee63918f407cc7990083cba9b999445f839796060" +checksum = "14b3b9990889736af898bd4e51f7d0c0228d6a9b9c3a5fecb3e6b25a7abc791c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -140,7 +140,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow 1.0.4", + "winnow", ] [[package]] @@ -153,14 +153,14 @@ dependencies = [ "alloy-rlp", "crc", "serde", - "thiserror 2.0.19", + "thiserror", ] [[package]] name = "alloy-eip2930" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9441120fa82df73e8959ae0e4ab8ade03de2aaae61be313fbf5746277847ce25" +checksum = "e64579d931b3f8eacc7c9ab0b220e87e9c4816e5c724ede1947b55c2f8e92ae5" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -178,7 +178,7 @@ dependencies = [ "alloy-rlp", "borsh", "serde", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -192,7 +192,7 @@ dependencies = [ "borsh", "once_cell", "serde", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -205,7 +205,7 @@ dependencies = [ "alloy-rlp", "borsh", "serde", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -243,7 +243,7 @@ dependencies = [ "alloy-provider", "alloy-sol-types", "async-trait", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cee30dd4c2f4b23f434fdf675e7bf9681b86768141277266c6f548ef25cba0a" +checksum = "858acd7fdad1e4a7057fd9c1b39c1f2cd6bcd57ebf3c56e2853c02ead049e816" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -297,7 +297,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", "tracing", ] @@ -324,7 +324,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -354,19 +354,19 @@ dependencies = [ "alloy-signer-local", "k256", "libc", - "rand 0.8.7", + "rand 0.8.8", "serde_json", "tempfile", - "thiserror 2.0.19", + "thiserror", "tracing", "url", ] [[package]] name = "alloy-primitives" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f007e257069855bdf21d27762fd3f3705a613f805c9a08309bf353503f081d71" +checksum = "d5e9dbd49258ac3ab893a481d46be29b58be7f734dcac46cd80b6b13ee36566c" dependencies = [ "alloy-rlp", "bytes", @@ -376,7 +376,7 @@ dependencies = [ "fixed-cache", "foldhash", "hashbrown 0.17.1", - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "k256", "keccak-asm", @@ -423,10 +423,10 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest 0.13.4", + "reqwest", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", "tokio", "tracing", "url", @@ -467,7 +467,7 @@ dependencies = [ "alloy-transport-http", "futures", "pin-project", - "reqwest 0.13.4", + "reqwest", "serde", "serde_json", "tokio", @@ -537,7 +537,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -563,7 +563,7 @@ dependencies = [ "either", "elliptic-curve", "k256", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -578,15 +578,15 @@ dependencies = [ "alloy-signer", "async-trait", "k256", - "rand 0.8.7", - "thiserror 2.0.19", + "rand 0.8.8", + "thiserror", ] [[package]] name = "alloy-sol-macro" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5655c38d5f84955bf727b2eeb62fddd91ebb98fd1d7ae6eb77f73ea88f9b9cf" +checksum = "60dd79f578c3912f1fc2a150dbeb8110b8cfb976c60f98ebf6de9d5da5965a2b" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -598,15 +598,15 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6277c780e07b76951e09a59788dde230d1582612324177d11a43a61e21a6bb83" +checksum = "9edb8520f2f94275e1caa73c85207dcc78ec402a9a4c429240f9a0783a8f16c0" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.14.0", + "indexmap 2.14.2", "proc-macro-error3", "proc-macro2", "quote", @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9762b2ad3e5a0c09886de54fe549ab0056681df843cb082e2df7e1c0eb270d30" +checksum = "66af2d9344882172993be5f5cbfd349fdfa52cb548f7af8715b446fb35ef6001" dependencies = [ "alloy-json-abi", "const-hex", @@ -635,19 +635,19 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4c7130f0f01f4719678bda3db3bc7267fc2f7f9d0565e3bd964cd2bb45050d" +checksum = "77c2a8abc3334044013b23d5e438cc5dc802e5544cc42b713d0840d5185967cc" dependencies = [ "serde", - "winnow 1.0.4", + "winnow", ] [[package]] name = "alloy-sol-types" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96e74d6213180f78dbdccddce8af02a639c160c94b0a543fa35c77c58b8a7fc" +checksum = "3e6f87cf007caa54e95455875fbd25879b008d0d9c7c340a2c258b3a3400a28c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -663,14 +663,14 @@ checksum = "e10be13ab13ddadcfaec3320cc6cee83548a1ae32ada214ff864604cbf5c6f77" dependencies = [ "alloy-json-rpc", "auto_impl", - "base64", + "base64 0.22.1", "derive_more", "futures", "futures-utils-wasm", "parking_lot", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", "tokio", "tower", "tracing", @@ -687,7 +687,7 @@ dependencies = [ "alloy-json-rpc", "alloy-transport", "itertools 0.14.0", - "reqwest 0.13.4", + "reqwest", "serde_json", "tower", "tracing", @@ -706,7 +706,7 @@ dependencies = [ "nybbles", "serde", "smallvec", - "thiserror 2.0.19", + "thiserror", "tracing", ] @@ -724,9 +724,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -791,6 +791,7 @@ dependencies = [ "ethereum_ssz_derive", "rollups-types", "sequencer-core", + "serde_json", "tracing", ] @@ -1024,7 +1025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -1034,7 +1035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -1044,7 +1045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -1054,7 +1055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "367c9c827ed431bff6868b7aa926e05b16eb46603cc8b6e768e4a5553fa1d155" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -1087,13 +1088,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -1104,13 +1105,13 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auto_impl" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +checksum = "683bf733a032aec4f8954e5c0ec9d5c2183c341c49d0939ad77acc0a19fa338a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1121,9 +1122,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "zeroize", @@ -1131,9 +1132,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -1149,7 +1150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -1171,7 +1172,7 @@ dependencies = [ "sha1", "sync_wrapper", "tokio", - "tokio-tungstenite 0.29.0", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", @@ -1209,6 +1210,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -1232,7 +1239,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite", ] [[package]] @@ -1241,7 +1248,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1257,20 +1264,20 @@ dependencies = [ [[package]] name = "bitcoin-consensus-encoding" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "207311705279250ba465076a1bac4b1ac982855fff73fc5f67e22158ac58cdc9" +checksum = "9daa31138eb443d5751b207f3f64154e2bb09cd59960562ccc7a7112be38147f" dependencies = [ "bitcoin-internals", - "hex-conservative 1.2.0", + "hex-conservative 1.3.0", "serde", ] [[package]] name = "bitcoin-internals" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d573f4cf32996a8dce612e4348cece65a241f1882ed594047c9ba348e8869fa5" +checksum = "e8bea3a9f0cfece4564e37cb49a38cc245ca5184719e50d7d0dda3268722c4e2" [[package]] name = "bitcoin-io" @@ -1288,14 +1295,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bca4c7abb40c8817d77403c880988cfd484f23ab2365726afb2f798363e2c4a2" dependencies = [ "bitcoin-io", - "hex-conservative 0.2.2", + "hex-conservative 0.2.3", ] [[package]] name = "bitflags" -version = "2.13.1" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitvec" @@ -1341,9 +1354,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ "borsh-derive", "bytes", @@ -1352,15 +1365,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1442,6 +1455,7 @@ dependencies = [ "app-core", "c-app-engine", "ethereum_ssz", + "rollups-types", "sequencer-core", "tempfile", ] @@ -1477,27 +1491,29 @@ dependencies = [ "k256", "rollups-types", "sequencer-core", + "serde", + "serde_json", "testsi", ] [[package]] name = "cartesi-machine" version = "2.0.0" -source = "git+https://github.com/cartesi/dave?rev=79874c0a1b40678465daf533acc976c88301c6d4#79874c0a1b40678465daf533acc976c88301c6d4" +source = "git+https://github.com/cartesi/dave?rev=612c54cf22d73b364cd57a7058ba6aa449e5c77a#612c54cf22d73b364cd57a7058ba6aa449e5c77a" dependencies = [ - "base64", + "base64 0.22.1", "cartesi-machine-sys", "derive_builder", "hex", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", ] [[package]] name = "cartesi-machine-sys" version = "2.0.0" -source = "git+https://github.com/cartesi/dave?rev=79874c0a1b40678465daf533acc976c88301c6d4#79874c0a1b40678465daf533acc976c88301c6d4" +source = "git+https://github.com/cartesi/dave?rev=612c54cf22d73b364cd57a7058ba6aa449e5c77a#612c54cf22d73b364cd57a7058ba6aa449e5c77a" dependencies = [ "bindgen", "link-cplusplus", @@ -1516,9 +1532,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" dependencies = [ "find-msvc-tools", "jobserver", @@ -1537,9 +1553,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -1549,12 +1565,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -1583,9 +1599,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -1593,9 +1609,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -1605,21 +1621,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmake" @@ -1638,9 +1654,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -1648,9 +1664,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.19.1" +version = "1.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -1710,6 +1726,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1721,9 +1743,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -1745,9 +1767,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crunchy" @@ -1806,6 +1828,16 @@ dependencies = [ "darling_macro 0.23.0", ] +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + [[package]] name = "darling_core" version = "0.20.11" @@ -1834,6 +1866,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + [[package]] name = "darling_macro" version = "0.20.11" @@ -1856,6 +1901,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -1876,6 +1932,37 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + [[package]] name = "der" version = "0.7.10" @@ -1888,9 +1975,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" dependencies = [ "serde_core", ] @@ -1999,7 +2086,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -2043,9 +2130,9 @@ dependencies = [ [[package]] name = "either" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" dependencies = [ "serde", ] @@ -2087,14 +2174,14 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -2106,6 +2193,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + [[package]] name = "ethereum_serde_utils" version = "0.8.1" @@ -2208,9 +2301,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fixed-cache" @@ -2229,7 +2322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.7", + "rand 0.8.8", "rustc-hex", "static_assertions", ] @@ -2269,9 +2362,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -2284,9 +2377,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -2294,15 +2387,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -2311,38 +2404,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -2462,9 +2555,9 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" dependencies = [ "hashbrown 0.17.1", ] @@ -2477,9 +2570,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -2489,18 +2582,18 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-conservative" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" +checksum = "db3fef046dca3ca91ee1408a8c1b80ab777e80a4d308d1bf4e7adb3fcb047e08" dependencies = [ "arrayvec", ] [[package]] name = "hex-conservative" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35431185f361ccf3ffc58254628af5f1f5d5f28531da2e02e5d6c82bbc282a10" +checksum = "271e0d19bcb473b6675739a2b536076b24a082316cb5199ad918edce10c599e8" dependencies = [ "arrayvec", ] @@ -2536,9 +2629,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -2561,22 +2654,21 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", - "futures-channel", "futures-core", "http", "http-body", @@ -2591,9 +2683,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -2606,16 +2698,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -2653,9 +2746,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2667,9 +2760,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -2680,9 +2773,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2694,16 +2787,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2714,15 +2808,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -2793,9 +2887,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -2805,18 +2899,18 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -2857,6 +2951,60 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -2869,7 +3017,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror", "walkdir", "windows-link", ] @@ -2918,9 +3066,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -2944,20 +3092,21 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] name = "keccak-asm" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5dc2c0d691cbf7595cde551ced329cca99c2387c2cbc97754c5d0cd045d3ee" +checksum = "f32890f646914a263e39064295005972f0e95b928254061b2aca98445f304ee9" dependencies = [ + "cfg-if", "digest 0.10.7", "sha3-asm", ] @@ -2979,15 +3128,15 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libcmt-sys" @@ -3014,9 +3163,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.38.1" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -3025,13 +3174,14 @@ dependencies = [ [[package]] name = "libtest-mimic" -version = "0.6.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" dependencies = [ + "anstream", + "anstyle", "clap", - "termcolor", - "threadpool", + "escape8259", ] [[package]] @@ -3051,9 +3201,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -3066,9 +3216,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" @@ -3081,9 +3231,9 @@ dependencies = [ [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "macro-string" @@ -3131,9 +3281,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "1.2.2" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -3177,9 +3327,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -3322,11 +3472,13 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.8" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +checksum = "b568374ba38b33a6c627141f891faf16902b08d2db26b8ede1bcb0a15b1919fa" dependencies = [ "memchr", + "psm", + "stacker", "ucd-trie", ] @@ -3376,9 +3528,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" [[package]] name = "pkcs8" @@ -3392,24 +3544,39 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -3447,7 +3614,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit", ] [[package]] @@ -3483,24 +3650,34 @@ dependencies = [ [[package]] name = "proptest" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ - "bitflags", + "bitflags 2.13.2", + "chacha20", + "core_detect", "num-traits", - "rand 0.9.5", - "rand_chacha 0.9.0", + "rand 0.10.3", "rand_xorshift", "regex-syntax", "unarray", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -3510,7 +3687,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.19", + "thiserror", "tokio", "tracing", "web-time", @@ -3518,22 +3695,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror", "tinyvec", "tracing", "web-time", @@ -3541,9 +3718,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -3582,9 +3759,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3605,9 +3782,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3670,11 +3847,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -3692,27 +3869,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.2", ] [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -3746,11 +3923,11 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", "futures-util", @@ -3758,40 +3935,6 @@ dependencies = [ "http-body", "http-body-util", "hyper", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] - -[[package]] -name = "reqwest" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" -dependencies = [ - "base64", - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", "hyper-rustls", "hyper-util", "js-sys", @@ -3804,15 +3947,18 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -3884,7 +4030,7 @@ dependencies = [ "ethereum_ssz", "futures-util", "k256", - "reqwest 0.12.28", + "reqwest", "rusqlite", "sequencer-core", "sequencer-rust-client", @@ -3893,7 +4039,7 @@ dependencies = [ "tar", "tempfile", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite", "toml", ] @@ -3912,14 +4058,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown 0.16.1", - "thiserror 2.0.19", + "thiserror", ] [[package]] name = "ruint" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e99bff0393163bb25029a6af25d3d8d202ba5b5438a74d1bd8789f5c822970" +checksum = "2973657b5127d510e230f5c63d2d106af9c8f79393d8b9f4647323e8196bdde5" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", @@ -3935,7 +4081,7 @@ dependencies = [ "parity-scale-codec", "primitive-types", "proptest", - "rand 0.8.7", + "rand 0.8.8", "rand 0.9.5", "rlp", "ruint-macro", @@ -3952,11 +4098,11 @@ checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" [[package]] name = "rusqlite" -version = "0.40.1" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ - "bitflags", + "bitflags 2.13.2", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -4007,11 +4153,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -4020,9 +4166,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -4034,9 +4180,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -4056,9 +4202,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation", "core-foundation-sys", @@ -4077,15 +4223,15 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -4137,9 +4283,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -4175,7 +4321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ "bitcoin_hashes", - "rand 0.8.7", + "rand 0.8.8", "secp256k1-sys 0.10.1", "serde", ] @@ -4215,7 +4361,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -4261,7 +4407,6 @@ name = "sequencer" version = "0.1.0" dependencies = [ "alloy", - "alloy-network-primitives", "alloy-primitives", "alloy-sol-types", "alloy-transport", @@ -4274,7 +4419,7 @@ dependencies = [ "ethereum_ssz_derive", "futures-util", "k256", - "reqwest 0.12.28", + "reqwest", "rollups-harness", "rusqlite", "rusqlite_migration", @@ -4285,11 +4430,12 @@ dependencies = [ "sha2", "tar", "tempfile", - "thiserror 2.0.19", + "thiserror", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite", "tokio-util", "toml", + "tower", "tower-http", "tracing", "url", @@ -4307,7 +4453,7 @@ dependencies = [ "ruint", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -4315,13 +4461,13 @@ name = "sequencer-rust-client" version = "0.1.0" dependencies = [ "alloy-primitives", - "reqwest 0.12.28", + "reqwest", "sequencer-core", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite", ] [[package]] @@ -4351,14 +4497,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -4380,11 +4526,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -4401,18 +4547,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" dependencies = [ - "base64", + "base64 0.23.1", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.2", + "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -4421,14 +4568,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -4475,9 +4622,9 @@ dependencies = [ [[package]] name = "sha3-asm" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6287fd675f713484342a89cbf0a386abef5f15919cfad607e5e1f19e1e15331" +checksum = "471668161349031e3d415412f996b030c477488eec267cc3cadae3d06c0a367f" dependencies = [ "cc", "cfg-if", @@ -4542,9 +4689,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "slab" @@ -4554,9 +4701,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" dependencies = [ "serde", ] @@ -4599,6 +4746,19 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -4641,9 +4801,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -4652,9 +4812,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.6.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083be3061e64d362cbe6ef12cfe1307ba3884326d8856448fe8a120fa2c44ebf" +checksum = "eb6a2e3c7f7a3e4e83d1752cec5d1e357ced0cf96e85419b6a07f227db3def3a" dependencies = [ "paste", "proc-macro2", @@ -4673,13 +4833,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -4712,15 +4872,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "testsi" version = "0.1.0" @@ -4730,7 +4881,7 @@ dependencies = [ "libtest-mimic", "rollups-types", "testsi-macros", - "thiserror 1.0.69", + "thiserror", ] [[package]] @@ -4744,42 +4895,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" -dependencies = [ - "thiserror-impl 2.0.19", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -4832,9 +4963,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -4842,24 +4973,15 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -4879,14 +5001,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -4904,18 +5026,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "tokio-tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.28.0", -] - [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -4925,14 +5035,14 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.29.0", + "tungstenite", ] [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -4943,74 +5053,54 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "1.1.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" dependencies = [ - "serde", + "indexmap 2.14.2", + "serde_core", "serde_spanned", - "toml_datetime 0.6.11", - "toml_edit 0.22.27", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned", - "toml_datetime 0.6.11", - "toml_write", - "winnow 0.7.15", -] - -[[package]] -name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ - "indexmap 2.14.0", - "toml_datetime 1.1.1+spec-1.1.0", + "indexmap 2.14.2", + "toml_datetime", "toml_parser", - "winnow 1.0.4", + "winnow", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ - "winnow 1.0.4", + "winnow", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_writer" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" [[package]] name = "tower" @@ -5034,7 +5124,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -5135,23 +5225,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.9.5", - "sha1", - "thiserror 2.0.19", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.29.0" @@ -5165,7 +5238,7 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror 2.0.19", + "thiserror", ] [[package]] @@ -5200,9 +5273,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-segmentation" @@ -5235,12 +5308,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5293,9 +5360,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -5317,9 +5384,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -5330,19 +5397,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5350,31 +5418,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -5408,9 +5476,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -5585,15 +5653,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "1.0.4" @@ -5611,9 +5670,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wyz" @@ -5647,30 +5706,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -5688,21 +5747,21 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] @@ -5720,9 +5779,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -5731,9 +5790,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -5742,13 +5801,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b67553a8..00843a57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ license = "Apache-2.0" description = "Deterministic sequencer prototype with low-latency soft confirmations" homepage = "https://github.com/cartesi/sequencer" repository = "https://github.com/cartesi/sequencer" -readme = "../README.md" +readme = "README.md" authors = [ "Gabriel Coutinho de Paula ", "Pedro Argento ", @@ -45,29 +45,43 @@ trolley = { version = "0.1", path = "sdk/guest/trolley" } testsi = { version = "0.1", path = "sdk/guest/testsi" } rollups-types = { version = "0.1", path = "sdk/guest/rollups-types" } -# ── Contracts (exact pin — the single place the version lives) ── +# ── Contracts (exact pin; equals ROLLUPS_CONTRACTS_VERSION in toolchain-pins.env, +# checked by scripts/verify-toolchain-pins.sh) ── cartesi-rollups-contracts = "=3.0.0-alpha.10" # ── Alloy family (must stay on the 2.x line: the contracts crate # requires alloy ^2; the alloy-core crates below stay on 1.x; # feature lists are member-specific) ── alloy = "2.5" -alloy-primitives = { version = "1.6", features = ["serde", "k256"] } -alloy-sol-types = "1.6" +alloy-primitives = { version = "1.7", features = ["serde", "k256"] } +alloy-sol-types = "1.7" # ── Shared crates (floors track what CI actually tests) ────── clap = { version = "4.6", features = ["derive"] } futures-util = "0.3" +# The k256 that alloy-primitives recovers guest signatures with; it moves with +# alloy-core, behind the consensus vectors. k256 = "0.13.4" -reqwest = { version = "0.12", default-features = false } +libtest-mimic = "0.8.2" +reqwest = { version = "0.13", default-features = false } rusqlite = { version = "0.40.1", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" ssz = { package = "ethereum_ssz", version = "0.10" } ssz_derive = { package = "ethereum_ssz_derive", version = "0.10" } +tar = "0.4" tempfile = "3.10" thiserror = "2" tokio = "1.53" -tokio-tungstenite = "0.28" +# Same tungstenite stack as axum's `ws` feature; move to 0.30 with the +# axum release that requires it. +tokio-tungstenite = "0.29" +toml = "1.1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Line tables keep file:line in backtraces without full variable debuginfo, +# which shrinks dev and test targets. Release is untouched on purpose: the +# guest is built with --release and its bytes feed the template hash. +[profile.dev] +debug = "line-tables-only" diff --git a/README.md b/README.md index d60fff13..86e99183 100644 --- a/README.md +++ b/README.md @@ -47,18 +47,12 @@ application-visible clock. Safe-head publication is best effort: if the node exposes a multi-block jump, the sequencer creates one frame at the observed tip and never fabricates intermediate frames. -Soft confirmations are an **optimistic prediction**: the sequencer also -cross-checks every at/above-anchor batch its off-chain scheduler simulation -accepts on L1 against the batch it sealed locally (a content-identity check). -When a foreign or byte-different landing reaches L1 *safe* finality and the -input reader ingests it, the same transaction records canonical divergence and -freezes the accepted frontier; the runtime stops when it next observes that -fact, and every later boot refuses until an operator performs cockroach -recovery. User-op chunks committed before runtime observation may still -acknowledge and be rolled back. This check is a narrow zombie/foreign-batch -backstop, not proof that arbitrary application or scheduler divergence cannot -exist, and it does not replace the watchdog. The mechanism and its bounds are -recorded in [`docs/invariants.md`](docs/invariants.md) (I9 and I15). +Soft confirmations are an **optimistic prediction**. A narrow content-identity +check compares each batch that L1 accepts at the sequencer's nonces with the +batch it sealed locally. A foreign or byte-different landing freezes the +accepted frontier and stops the runtime, and every later boot refuses until an +operator performs cockroach recovery. The check does not replace the watchdog; +its bounds are recorded in [`docs/invariants.md`](docs/invariants.md) (I9 and I15). ## Recovery @@ -83,7 +77,7 @@ between normal operation's self-trust and manual repair after a bug. The sequencer is designed to handle: -- **L1 provider outages** — workers retry with exponential backoff. The inclusion lane and API continue operating locally. A wall-clock fallback detects when an outage pushes batches into the danger zone. +- **L1 provider outages** — each worker retries on its next tick; the RPC client resends a rate-limited or temporarily unavailable request at most twice, a second apart or after the server's requested wait when that is 5 s or less. The inclusion lane and API continue operating locally. A wall-clock fallback detects when an outage pushes batches into the danger zone. - **Undiagnosed interruptions (OOM, SIGKILL, reboot)** — restart can recover automatically: every boot derives any required recovery from SQLite and L1 safe state through startup recovery, never assuming the previous exit was clean. Terminal errors returned through a command bracket best-effort record their cause in `terminal_faults`; terminal runtime aborts leave only process diagnostics. - **Extended downtime** — startup syncs to the current L1 safe head, flushes if needed, and recovers before admission. A terminal exit requires operator investigation; rebuilding untrustworthy state follows the cockroach recovery procedure above. - **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. Recovery waits until every covered wallet-nonce slot is consumed at safe depth, whether the original transaction or a flush no-op wins, so delayed "zombie" submissions cannot land later. @@ -126,24 +120,46 @@ For rebuilding from a trusted checkpoint, follow the CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ -CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 \ +CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE=0 \ +CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xa0Ee7A142d267C1f36714E4a8F75612F20a79720 \ cargo run -p wallet-sequencer -- setup # Phase B — run the sequencer. CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ -CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac09...f2ff80 \ +CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0x2a87...d409c6 \ cargo run -p wallet-sequencer -- run ``` +`setup` exits 20 (retryable) until the application's deployment block is at or below the L1 safe head, so no reorg can move the block it pins; on Ethereum that is a one-time wait of an epoch or two after deployment, and on Anvil `--slots-in-an-epoch` mined blocks (one in the repo's devnet stack, which sets it to 1; Anvil's default is 32). + A third subcommand, **`flush-mempool`**, settles the batch-submitter wallet nonce on demand (keyed operator tool). It is flush-only: it requires a completed setup and no canonical divergence, and it never performs Sync/Cascade or launches runtime workers. -`setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`. -`run` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `_FILE`); it refuses to boot until `setup` has completed. +`setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`. On chains other than Ethereum mainnet and Sepolia it also requires a fee-oracle source (listed below). +The submitter address must equal the `sequencer_address` compiled into the canonical machine; nothing checks this at runtime ([scheduler semantics](docs/protocol/scheduler-semantics.md#the-one-stream-the-one-classification)). + +**Use a dedicated submitter address.** Plain `setup` refuses an unsettled wallet +nonce, so sharing a busy contract-deployer address can trip the detection gate +while its deployment transactions are not yet safe. The devnet uses Anvil +account 9 (`DEVNET_SEQUENCER_ADDRESS`), separate from the account-0 deployer. +The Sepolia reference reserves [`SEPOLIA_SEQUENCER_ADDRESS`](examples/app-core/src/application/wallet.rs) +for this deployment's batch submissions and application fee revenue; do not share +its signing key with other app or node instances. Changing that compiled address +requires a rebuilt canonical image and a matching new application deployment, +not just a runtime signing-key change. + +`run` and `flush-mempool` require `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` and the signing key, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (whose first line is the key); both refuse to start until `setup` has completed. -Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32012,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`. +Optional variables, by name; each subcommand's `--help` shows their defaults and validation from [`commands/config.rs`](sequencer/src/commands/config.rs): + +- Every subcommand: `CARTESI_SEQUENCER_DATA_DIR` (the SQLite file is `sequencer.db` inside; `setup` creates it, and `run` and `flush-mempool` refuse a missing one), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`, and `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC` (below). +- `setup` and `run`: `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS`, `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS`. +- `setup`: the fee-oracle source it pins, either `CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE` or a Uniswap source (the overrides `CARTESI_SEQUENCER_FEE_TOKEN_ADDRESS`, `CARTESI_SEQUENCER_WETH_ADDRESS` and `CARTESI_SEQUENCER_UNISWAP_V3_POOL`, given together and required on other chains, and `CARTESI_SEQUENCER_FEE_ORACLE_TWAP_WINDOW_SECS`); `CARTESI_SEQUENCER_CHECKPOINT_BLOCK`; and, for a [rebuild](docs/recovery/cockroach.md), `CARTESI_SEQUENCER_RECOVERY`, `CARTESI_SEQUENCER_CHECKPOINT_DUMP_DIR` and the signing key. +- `run`: `CARTESI_SEQUENCER_HTTP_ADDR` (public listener), `CARTESI_SEQUENCER_INTERNAL_HTTP_ADDR` (internal listener), `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH` and `CARTESI_SEQUENCER_FEE_ORACLE_POLL_INTERVAL_MS`. + +Logs and errors show only the endpoint's scheme, host and port, since its path, query and userinfo can hold an API key or a password, unless the RPC server echoes them inside a JSON-RPC error message or a malformed result, which alloy builds above the redaction layer, or quotes in an error body only part of them or a different URL that carries them, such as a redirect target on another path. An L1 error's text stays on one line, with each cause cut at 256 bytes. The client follows no redirects: an endpoint that answers 3xx fails with that status, so configure the provider's final URL (setup exits 30 at its chain-id check, since a retry gets the same redirect). Debug-level logging for the `alloy_transport_http` target records the full URL, so keep that target at `info` where logs leave the host. By default the blockchain endpoint must be `https://` unless its host is loopback (`localhost`, `127.0.0.0/8`, `::1`) — a guard against accidentally sending L1 traffic to a public RPC in the clear. Set `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC=true` (or `--allow-insecure-rpc`) to permit plaintext `http://` to a non-loopback host on a **trusted private network** — e.g. a Docker Compose / Kubernetes service name (`http://anvil:8545`), `host.docker.internal`, or a private-VPC IP. @@ -155,15 +171,9 @@ environment: CARTESI_SEQUENCER_ALLOW_INSECURE_RPC: "true" ``` -Process exit codes follow the orchestrator exit-code contract: `0` clean shutdown, `10` restart (expect a recovery boot), `20` transient refusal (retry with backoff), `30` terminal command refusal (operator required — e.g. setup not complete, identity mismatch, canonical divergence, persistent storage/application invariant failure), `40` a previous instance left work past the checkpoint (wipe the data directory and run `setup --recovery`), and `1` for an unclassified operational failure. Diagnosed terminal runtime faults, including worker panics, immediately call `abort()` (SIGABRT, status 134), without worker drain or database settlement. Supervisors must treat SIGABRT as terminal class. Ordinary shutdown, expected recovery, and transient errors still drain gracefully. Startup panics caught by the command harness are projected to `30`; panics after runtime scope creation abort; `101` remains possible before the command harness starts. The constants live in `sequencer/src/commands/error.rs`; supervisor recipes are in [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md). - -Fixed protocol identity (EIP-712): +Process exit codes follow the orchestrator exit-code contract: `0` clean shutdown, `10` restart (expect a recovery boot), `20` transient refusal (retry with backoff), `30` terminal command refusal (operator required — e.g. setup not complete, identity mismatch, canonical divergence, persistent storage/application invariant failure), `40` a previous instance left work past the checkpoint (wipe the data directory and run `setup --recovery`), and `1` for an unclassified operational failure. Diagnosed terminal runtime faults, including worker panics and an undecodable batch landed by the submitter key, immediately call `abort()` (SIGABRT, status 134), without worker drain or database settlement. Supervisors must treat SIGABRT as terminal class. Ordinary shutdown, expected recovery, and transient errors still drain gracefully. Startup panics caught by the command harness are projected to `30`; panics after runtime scope creation abort; `101` remains possible before the command harness starts. The constants live in `sequencer/src/commands/error.rs`; supervisor recipes are in [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md). -- domain name: `CartesiAppSequencer` -- domain version: `1` -- `chain_id` and `verifying_contract` come from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` and `CARTESI_SEQUENCER_APP_ADDRESS` - -Most queue sizes, polling intervals, and safety limits are now internal runtime constants instead of public launch-time configuration. +The [EIP-712 domain](docs/protocol/scheduler-semantics.md#user-op-signatures) is fixed except `chainId` and `verifyingContract`, which come from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` and `CARTESI_SEQUENCER_APP_ADDRESS`. ## API @@ -171,6 +181,13 @@ This section owns the wire contract. [`openapi.yaml`](openapi.yaml) renders it as OpenAPI 3.1 for client generation and documentation tools; change both together. +The sequencer serves two listeners. The public one +(`CARTESI_SEQUENCER_HTTP_ADDR`) serves only `POST /tx`, `GET /fee`, +`GET /nonce`, and `GET /domain`. Every other route, health probes included, is +internal and served only on the internal listener +(`CARTESI_SEQUENCER_INTERNAL_HTTP_ADDR`), without authentication; keep it off +the public network. + Errors use one JSON shape, `{ "ok": false, "code": "", "message": "" }`, with these codes: `400` `BAD_REQUEST` or `INVALID_SIGNATURE`, `413` `PAYLOAD_TOO_LARGE`, `422` `EXECUTION_REJECTED`, `429` `OVERLOADED`, `500` @@ -441,9 +458,8 @@ checkpoint selection, and subscription. ### Operator snapshot endpoints (internal only) These serve application state to the operator's watchdog and indexers. -**They are operator-internal — no auth — and must not be exposed publicly** -(gated by network controls today; bound to a separate internal port once the -api split lands). +**They are operator-internal — no auth — and must not be exposed publicly**: +they are served only on the internal listener, behind network controls. - `GET /finalized_state/inclusion_block` — cheap JSON the watchdog polls: `{ "inclusion_block": , "executed_input_count": }`. @@ -498,50 +514,27 @@ They do not certify L1 freshness, submitter balance, or canonical agreement. - `safe_inputs`: every raw InputBox observation, including batch envelopes - `history_state`: immutable era baseline (application count and accounted L1 block) plus recovery generation - `snapshots` and `dumps`: immutable batch-close/baseline artifacts and streaming leases; accepted status is derived from `safe_accepted_batches` -- `batch_policy`: singleton knobs and constants for DA-style batch sizing and fee derivation; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target`. A batch closes on whichever fires first: the derived `batch_size_target` byte budget or the `max_batch_open` wall-clock deadline (an inclusion-lane setting, `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, not a `batch_policy` column). Setup writes the first `log_gas_price` (and observation stamp) for both Fixed and Uniswap modes, failing if the initial Uniswap quote cannot be read. Fixed local pricing has no oracle worker; Uniswap starts from the persisted price and refreshes lazily via the setup-pinned WETH/fee-token TWAP source, retaining that price across transient source failures. Host tick-to-price conversion uses approximate floating-point arithmetic before checked integer gas-cost calculation and log encoding; recorded frame fees execute with deterministic integer arithmetic. `log_slack = log(10)` applies the 10× safety margin in log space. Fees are app-token smallest units — initially USDC (6 decimals) for the wallet prototype — not a protocol-level USDC invariant. +- `batch_policy`: singleton fee and batch-sizing knobs; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target` ([formula](sequencer/src/storage/migrations/0001_schema.sql)). A batch closes when its `batch_size_target` byte budget fills or `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS` elapses, whichever comes first. The [threat model](docs/threat-model/README.md#fee-oracle) owns how the oracle prices and refreshes. ## Project Layout -- `sequencer/src/lib.rs`: public crate surface (`run`, `RunConfig`) — the sequencer is a library; app crates build the binary (see `examples/wallet-sequencer/`) -- `examples/wallet-sequencer/`: binary crate composing the sequencer library with the placeholder wallet app -- `sequencer/src/http.rs`: shared HTTP error type, JSON error shape, and `axum::serve` orchestration -- `sequencer/src/runtime/`: process lock and shutdown scope; command bootstrap and config live in `commands/`, the shared clock in `clock.rs`, and EIP-712 domain construction in `sequencer-core/` -- `sequencer/src/ingress/`: public-facing — `POST /tx`, `GET /fee`, `GET /nonce`, and `GET /domain` (`api.rs`) and the inclusion lane (`inclusion_lane/`: hot-path loop, chunk/frame/batch rotation, catch-up, snapshot lifecycle) -- `sequencer/src/egress/`: internal read path — WS subscribe + health probes (`api/`) and the DB-backed ordered-L2Tx feed (`l2_tx_feed/`) -- `sequencer/src/l1/`: L1 client surface — input reader, batch submitter, fee oracle, shared EIP-1559 estimation, provider, partition helper -- `sequencer/src/recovery/`: preemptive recovery startup, runtime danger detector, mempool flusher -- `sequencer/src/storage/`: schema, migrations, SQLite persistence (split per writer role), and replay reads -- `sequencer-core/src/`: shared domain types and interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, feed message types) -- `examples/app-core/src/`: wallet prototype implementing `Application` -- [`bindings/c-app-engine/`](bindings/c-app-engine/README.md): reusable C ABI adapter and external static-archive integration guide -- `bindings/c-app-sequencer/`: optional C-engine CLI host and external-archive binary -- `examples/c-wallet-engine/`: reference wallet C ABI exports, genesis tool, and conformance tests -- `examples/c-wallet-sequencer/`: binary composing the C-engine host with the reference wallet engine -- `tests/benchmarks/`: benchmark harnesses and benchmark spec - -Related docs: - -- C application binding: [`docs/protocol/c-application-binding.md`](docs/protocol/c-application-binding.md) -- App snapshots (format + lifecycle): `docs/snapshots/` -- Watchdog — local dev: [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md); Sepolia/mainnet: [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) - -The watchdog ships as a multi-arch container image per release tag `vX`: +The sequencer is a library: an application crate composes it with its +`Application` implementation into a binary, as +[`examples/wallet-sequencer/`](examples/wallet-sequencer/) does with the +placeholder wallet in [`examples/app-core/`](examples/app-core/). Native engines +bind through the [C ABI adapter](bindings/c-app-engine/README.md). The +[architecture map](AGENTS.md#architecture-map) covers the workspace and the +sequencer's modules. -```bash -docker pull ghcr.io/cartesi/sequencer-watchdog:vX -# mirror: docker.io/cartesi/sequencer-watchdog:vX -``` +The watchdog ships as a multi-arch container image per release tag; the +[operator deployment guide](docs/watchdog/operator-deployment.md) gives the +image names and how to run it. ## Prototype Limits - The `Application` trait defines dump/load behavior ([format](docs/snapshots/format.md)). Every batch close registers a durable snapshot atomically with the seal. Restart restores the latest valid snapshot and replays application inputs from its count. Acceptance determines the recovery checkpoint and garbage-collection frontier without mutating artifact metadata. [Snapshot lifecycle](docs/snapshots/lifecycle.md) documents leases and crash ordering. - Schema and migrations are still in prototype mode and may change. -## Local Test Prerequisites - -- Some `sequencer` tests spin up `Anvil`; install Foundry locally if you want the full test suite: -- Self-contained benchmarks also spawn `Anvil` from a preloaded rollups state dump. - ## Development The shared [development commands](AGENTS.md#shell-and-commands) cover Rust @@ -551,18 +544,9 @@ validation for a change; some tests require Anvil or libslirp. ## Further Reading -- [`AGENTS.md`](AGENTS.md) — developer guide: architecture, conventions, duality, recovery, invariants, rules. -- [`CLAUDE.md`](CLAUDE.md) — Claude entrypoint to the shared agent guide. -- [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. -- [`docs/recovery/README.md`](docs/recovery/README.md) — automatic recovery, TLA+ formal verification, design history. -- [`docs/recovery/cockroach.md`](docs/recovery/cockroach.md) — manual rebuild after lost state or a sequencer bug. -- [Application history and replay](docs/protocol/application-history.md) — progress, history identity, and replica bootstrap/resume. -- [Snapshots](docs/snapshots/README.md) — engine checkpoints, lifecycle, accepted comparison, and wallet encoding. -- [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — step-by-step: run the watchdog with a local sequencer. -- [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — watchdog on live L1 (Sepolia staging, mainnet production). -- [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, modules, and test commands. -- [`sequencer-core/`](sequencer-core/) — shared domain types (`Application`, `SignedUserOp`, `Batch`, `Frame`). -- [`examples/app-core/`](examples/app-core/) — placeholder wallet app implementing the `Application` trait. +[`AGENTS.md`](AGENTS.md) is the developer guide: architecture, conventions, and +rules. Its [Reading Routes](AGENTS.md#reading-routes) name the document that +owns each contract. ## License diff --git a/bindings/c-app-engine/Cargo.toml b/bindings/c-app-engine/Cargo.toml index 98f6443a..fed6a699 100644 --- a/bindings/c-app-engine/Cargo.toml +++ b/bindings/c-app-engine/Cargo.toml @@ -11,7 +11,8 @@ authors.workspace = true [build-dependencies] # Generates the FFI declarations from the engine header at build time. It decides how this host -# reads every record crossing the seam, so a bump changes the ABI interpretation. +# reads every record crossing the seam, so a bump changes the ABI interpretation. 0.73 changed +# layout and bitfield code generation; take it with seam v1. bindgen = "0.72" [dependencies] diff --git a/bindings/c-app-engine/build.rs b/bindings/c-app-engine/build.rs index 34c43379..ba0a04c5 100644 --- a/bindings/c-app-engine/build.rs +++ b/bindings/c-app-engine/build.rs @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) //! Links the application's engine archive and generates the FFI declarations from its header. -//! //! The two environment variables are the whole application-specific binding, documented in -//! `README.md`. With none of them set this crate links no archive, -//! and the binary that uses it supplies the engine instead. +//! `README.md`. With none of them set this crate links no archive, and the binary that uses it +//! supplies the engine instead. use std::env; use std::path::{Path, PathBuf}; diff --git a/bindings/c-app-engine/src/lib.rs b/bindings/c-app-engine/src/lib.rs index b6a4b378..2167c64e 100644 --- a/bindings/c-app-engine/src/lib.rs +++ b/bindings/c-app-engine/src/lib.rs @@ -67,7 +67,19 @@ fn internal(reason: impl Into) -> AppError { } } +/// Validation, execution and draining. NOT_FOUND and INVALID_DUMP describe +/// lifecycle failures only, so here they break the contract and are fatal. fn check(status: i32, operation: &str) -> Result<(), AppError> { + check_status(status, operation, false) +} + +/// `from_dump` and `create_dump`, where a missing or invalid dump is an +/// operational I/O failure. +fn check_lifecycle(status: i32, operation: &str) -> Result<(), AppError> { + check_status(status, operation, true) +} + +fn check_status(status: i32, operation: &str, lifecycle: bool) -> Result<(), AppError> { if status == sys::APPLICATION_ENGINE_STATUS_OK { return Ok(()); } @@ -77,8 +89,8 @@ fn check(status: i32, operation: &str) -> Result<(), AppError> { ); let kind = match status { sys::APPLICATION_ENGINE_STATUS_IO_ERROR => std::io::ErrorKind::Other, - sys::APPLICATION_ENGINE_STATUS_NOT_FOUND => std::io::ErrorKind::NotFound, - sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP => std::io::ErrorKind::InvalidData, + sys::APPLICATION_ENGINE_STATUS_NOT_FOUND if lifecycle => std::io::ErrorKind::NotFound, + sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP if lifecycle => std::io::ErrorKind::InvalidData, _ => return Err(internal(reason)), }; Err(AppError::Io(std::io::Error::new(kind, reason))) @@ -257,7 +269,7 @@ impl Application for EngineApp { fn from_dump(prefix: &Path) -> Result { let prefix = path_to_cstring(prefix); let mut engine = std::ptr::null_mut(); - check( + check_lifecycle( unsafe { sys::application_engine_from_dump(prefix.as_ptr(), &mut engine) }, "from_dump", )?; @@ -270,7 +282,7 @@ impl Application for EngineApp { fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { let prefix = path_to_cstring(prefix); - check( + check_lifecycle( unsafe { sys::application_engine_create_dump(self.engine, prefix.as_ptr()) }, "create_dump", ) diff --git a/bindings/c-app-engine/src/sys.rs b/bindings/c-app-engine/src/sys.rs index 87325e0d..82d7c2ed 100644 --- a/bindings/c-app-engine/src/sys.rs +++ b/bindings/c-app-engine/src/sys.rs @@ -1,12 +1,9 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! The engine C ABI, generated by bindgen from the header. Nothing here is written by hand, so -//! to change what crosses, change the header. -//! -//! Enums arrive as plain integer constants, never Rust enums, so a value an engine adds later is -//! a number this host can refuse rather than undefined behavior. Records carry generated layout -//! assertions. +//! The engine C ABI, generated by bindgen from the header: change the header, not +//! this file. Enums arrive as integer constants, so a value an engine adds later is +//! refused rather than undefined behavior; records carry generated layout assertions. // Generated code follows C's naming and carries the whole surface, including what this host has // no call for yet diff --git a/bindings/c-app-engine/src/tests.rs b/bindings/c-app-engine/src/tests.rs index 194743f3..a3934548 100644 --- a/bindings/c-app-engine/src/tests.rs +++ b/bindings/c-app-engine/src/tests.rs @@ -111,3 +111,24 @@ fn drain_preserves_output_order_and_copies_reused_payloads() { ] ); } + +#[test] +fn lifecycle_statuses_are_fatal_outside_dump_lifecycle_calls() { + for status in [ + sys::APPLICATION_ENGINE_STATUS_NOT_FOUND, + sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP, + ] { + assert!(matches!( + check(status, "execute_direct_input"), + Err(AppError::Internal { .. }) + )); + assert!(matches!( + check_lifecycle(status, "from_dump"), + Err(AppError::Io(_)) + )); + } + assert!(matches!( + check(sys::APPLICATION_ENGINE_STATUS_IO_ERROR, "drain_output"), + Err(AppError::Io(_)) + )); +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 00000000..71929faf --- /dev/null +++ b/deny.toml @@ -0,0 +1,19 @@ +# cargo-deny gate on the crate graph (the cargo-deny job in +# .github/workflows/ci.yml). Sources are deterministic for a given Cargo.lock +# and block. Advisories change with the RustSec database under a fixed lock, +# so CI reports them without failing. + +[graph] +all-features = true + +[advisories] +# Unmaintained notices count only for direct dependencies of our crates. +unmaintained = "workspace" +# Each entry: an advisory id plus why it does not reach a shipped binary. +ignore = [] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +# testsi's cartesi-machine bindings, pinned by rev (sdk/guest/testsi/Cargo.toml). +allow-git = ["https://github.com/cartesi/dave"] diff --git a/docs/cartesi-machine.md b/docs/cartesi-machine.md index 288dbebf..bef11873 100644 --- a/docs/cartesi-machine.md +++ b/docs/cartesi-machine.md @@ -5,32 +5,93 @@ drives it, the canonical guest is built for it, and cockroach recovery exports from it. This page records the CM behavior those components depend on, especially behavior that is easy to assume wrongly. Everything here holds for the pinned version (`CARTESI_MACHINE_VERSION` in -[`toolchain-pins.env`](../toolchain-pins.env), currently v0.21.0) and was -checked against the emulator source at that tag. Re-check it on every bump. +[`toolchain-pins.env`](../toolchain-pins.env), currently v0.22.0) and was +checked against the emulator source at that tag. Re-check it, and the Dave pin +under [rollup host semantics](#rollup-host-semantics), on every bump of either. ## Versions are all-or-nothing -A stored machine records a config archive version and the emulator's machine -identifiers; loading rejects any mismatch. A CM bump therefore invalidates every -stored image, release tarball, and watchdog checkpoint: rebuild the canonical -images and re-`init` watchdog state directories. The kernel -(`machine-linux-image`) and guest tools (`machine-guest-tools`) are paired with -the emulator release and move with it. The development shell pins the same -emulator through the parent flake. +A stored machine records a config archive version (8) and the emulator's +machine identifiers (`marchid` 22, and `mimpid`, major × 1000 + minor); loading +rejects any mismatch, and nothing converts a stored machine to another version +(`skip_version_check` skips only the identifiers). A bump to another minor +release therefore invalidates every stored image, release tarball, and +watchdog checkpoint: rebuild the canonical images and the watchdog test guest +(the recipes reuse an image that exists), and start each new deployment's +watchdog from a new state directory +([upgrades](watchdog/operator-deployment.md#upgrades)). A patch release of the +same minor loads them and, by the emulator's design since 0.15.2, keeps the +root hash; the pins name the exact release all the same. The kernel and guest +tools are the ones the emulator release's `dependencies.lock` names, and move +with it: kernels are +[`cartesi/linux`](https://github.com/cartesi/linux/releases) releases named by +kernel version, guest tools are `machine-guest-tools` releases; +`scripts/verify-toolchain-pins.sh --upstream` checks the pairing. The +development shell pins the same emulator through the parent flake. **A deployed application is pinned to its emulator version.** Its on-chain template hash is the root hash of a machine built with one emulator release, -and the machine identifiers are part of that hash. A machine rebuilt with -another release has a different root hash, and the old release's stored -machines do not load. Every canonical machine for a deployment, the -watchdog's included, must run the release its template was built with; moving -to a new emulator release means a new application deployment. +and the machine identifiers are part of that hash. A machine rebuilt with a +release of another minor has a different root hash, and the old release's +stored machines do not load; within a minor, only the pins enforce the +release. Every canonical machine for a deployment, the watchdog's +included, must run the release its template was built with; moving to a new +emulator release means a new application deployment. + +**What moves the template hash.** The hash covers every byte of the stored +machine, so it moves when any input to the image moves, even when behavior does +not: + +- the emulator's minor release, even with every other input fixed (the machine + identifiers and the microarchitecture's state are part of the root hash, and + execution fixes change what runs), and its kernel (`toolchain-pins.env`); +- the rootfs: the guest Dockerfile, `machine-guest-tools`, and `xgenext2fs`; +- whether the cross toolchain has `rust-src`: with it, rustc embeds local + standard-library paths in panic locations instead of the remapped + `/rustc//` ones. cross 0.2.5 adds it only on a run after the one that + installs the riscv64 target, so the guest recipes install both first + (`cross-toolchain`) and a cold build is reproducible; +- the dapp binary: the `rust-toolchain.toml` channel, every `Cargo.lock` entry + in `canonical-app`'s build graph, the cross image digest in + `examples/canonical-app/Cross.toml`, and `CROSS_VERSION`. The build graph + includes the proc macros and build-dependencies that only run on the host + (`syn`, `bindgen` for libcmt, `ruint` for the fee table): a version bump + anywhere in the graph can move the binary, most likely because each crate's + metadata hash covers its dependencies and feeds symbol names and code + layout. The host half of + the graph is `x86_64-unknown-linux-gnu`, cross's container, whatever the + machine; `cargo tree -p canonical-app --target all -e normal,build` lists a + superset from any machine, and an update outside it leaves the binary + byte-identical; +- any source change in a guest crate (`sequencer-core`, `sdk/guest/*`, + `examples/app-core`, `examples/canonical-app`), even in code the guest never + calls: recompiling a crate shifts its constants and code layout. + +The checkout path does not enter the hash, and `just canonical-print-build-hashes` +prints all four inputs to compare; compare cold builds, since a warm `target/` +can hold crates compiled before `rust-src` arrived. A move that is meant to preserve behavior is +checked against [`tests/fixtures/consensus_vectors_v1.json`](../tests/fixtures/consensus_vectors_v1.json), +which pins the consensus bytes (EIP-712 hashes, signature recovery, SSZ, fee +table, output payloads) natively and, through `canonical-test`'s golden cases, +in the built image, where each signature variant is checked for running as the +signer or not. The fixture is never regenerated: a mismatch is a behavior +change to explain. A release that moves the hash says so, and deployed +applications keep their old image, which the watchdog checks against the +on-chain hash. ## Rollup host semantics -The reference host loop is the `cartesi-machine` CLI (`run_advance_state_epoch` -in `src/cartesi-machine.lua`); Dave's PRT client implements the same rules. -Any other host, including the watchdog, must match it exactly: +The Solidity step defines the per-input transition: Dave treats it as the +authority, and the release CLI as no independent reference (Dave's +`docs/computation-hash.md`). The watchdog follows two pinned implementations of +it: Dave's sling node, `MachineStf` (`feed`, `run_big`, `restore_rejected`, +`terminal_fixed`, `manual_yield_reason`) at dave@dbcca7d4, and the v0.22.0 +CLI's epoch driver without a hash builder (`run_advance_state_epoch` and +`run_advance_state_input`, running `run_to_stop` with the null builder), which +is the test oracle. They differ only at the corners stated below, at the rule +each qualifies; there the watchdog follows the step and sling, apart from the +deliberate differences the +[watchdog README](watchdog/README.md#why-it-is-built-this-way) lists. | Guest outcome after an input | Host action | |---|---| @@ -38,26 +99,80 @@ Any other host, including the watchdog, must match it exactly: | `RX_REJECTED` manual yield | restore the pre-input snapshot | | `TX_EXCEPTION`, halt, any other manual yield, or mcycle overflow | fixed point: the machine stays there forever, with no revert | +A pending manual yield decides the outcome, as the step reads it: an +`RX_ACCEPTED` yield takes the next input and an `RX_REJECTED` one reverts, even +on the budget's last cycle or on a machine preset halted. A halt or an +exhausted budget is a fixed point only with no yield pending (sling's +`MachineStf::terminal_fixed` at dave@dbcca7d4). + +A host classifies a pending yield as the step does: `iflags.Y`, then tohost's +device, command and reason, never its data field (sling's +`MachineStf::manual_yield_reason` at dave@dbcca7d4). A guest that bypasses +libcmt can declare data past the 2 MiB TX buffer, which `receive_cmio_request` +refuses; the CLI reads every yield through it and aborts there. `iflags.Y` +under another device or command, which only a preset template carries, is a +fixed point. + +The emulator's run break reason ranks a halt first (halt, then manual yield, +then an exhausted budget), and the CLI's epoch loop follows it. The two orders +differ only on a preset image with both `iflags.H` and `iflags.Y` set, which +execution never produces and our image recipes reject +(`--assert-rolling-template` exits 2 on it): the step delivers the next input +and the machine then stays halted, while the CLI ends the epoch at the halt, a +case Dave's `docs/computation-hash.md` lists among the CLI's named exclusions. + - The host reads the root hash, snapshots, then sends the input together with - that root as the *revert root hash*. The emulator refuses an advance unless - the machine sits at an `RX_ACCEPTED` yield and the revert root hash equals the - current root, so a host cannot silently continue from a rejected state. -- Each input gets an mcycle budget of 2^48 (`imcyclemax`). Exhausting it is a - fixed point, not a reject. At realistic speeds this takes days, so a stuck - guest shows up as a hung host long before the budget ends. + that root as the *revert root hash*. At an `RX_ACCEPTED` yield the emulator + refuses an advance without a revert root hash or with one that differs from + the current root, and at any state it refuses a revert root hash on any + other response. Every other undeliverable response is a silent no-op that + leaves the machine unchanged, like the logged and proven transition: no + pending yield, an advance at any yield other than `RX_ACCEPTED` (an + unreverted rejection included), or data larger than the 2 MiB RX buffer. + Only delivery clears `iflags.Y`. Sling asserts an `RX_ACCEPTED` header + before sending, and the CLI trusts the last yield it read; neither checks + afterwards. At an `RX_ACCEPTED` yield the one undeliverable advance is data + over the RX buffer, which both take as an accept with the state unchanged. + No chain input reaches it: the pinned InputBox (rollups-contracts + `ROLLUPS_CONTRACTS_VERSION` in `toolchain-pins.env`) refuses an encoded + input over 65,536 bytes (`InputTooLarge`, as Dave's + `LeafTournamentGasFixture` pins for the same release). +- Each delivered advance sets the budget `imcyclemax` to the current mcycle + plus 2^48 − 1. Exhausting it with no halt or yield pending is a fixed point, + not a reject. At realistic speeds this takes days, so a stuck guest shows up + as a hung host long before the budget ends. - An inspect query is legal only at an `RX_ACCEPTED` yield and runs on a snapshot the host discards. Inspect execution is never part of canonical - history, and its single report is bounded by the 2 MiB CMIO TX buffer. + history, and its single report is bounded by the 2 MiB CMIO TX buffer. The + emulator itself delivers an inspect at any pending manual yield; the host + enforces `RX_ACCEPTED` (the CLI asserts it). A fixed point is permanent: once the canonical machine reaches one, no later -input is ever processed for that application. +input is ever processed for that application. The inputs still arrive: sling +records each later input as a window that carries the frozen state and +defends the epoch, which can never be staged (Dave's `docs/epoch-lifecycle.md`), +and the CLI ends the epoch at the fixed point. Both hold the same state. + +Epochs change no machine state. Under Dave each epoch starts from the previous +one's final state (`DaveConsensus` instantiates the next tournament from it; +sling's `roll_epoch` re-registers it as the next epoch's first snapshot), so +the uninterrupted InputBox sequence is the machine's history. The one exception +is the epoch's input span: an epoch admits at most 2^24 inputs, and past that +the contracts never feed the tail and a Dave node panics (Dave's +`docs/dimensioning.md`), while a host replaying the whole InputBox sequence +executes it. Reaching it takes about 5e11 gas of inputs in one epoch on L1, and +a cheaper throughput flood breaks Dave settlement first, so the gap is +accepted on cost; revisit it before deploying on a cheap L2, where such floods +cost about a thousand times less. ## Storage, sharing, and snapshots - **Loading** defaults to `SHARING_NONE`: every backing file is mapped privately and never modified, under a shared `flock`. `SHARING_ALL` maps the files shared and runs the machine in place on disk, under an exclusive - `flock`, so one directory cannot be open both ways at once. + `flock` on each writable file (read-only ranges keep a shared one), so one + directory cannot be open both ways at once. `SHARING_ALL` refuses a writable + range whose backing file has no write permission or more than one hard link. - **`store`** writes every range in full as sparse files, refuses an existing directory, and does **not** fsync. Durable publication uses `sync_stored` (fsyncs files, the directory, and its parent), then `rename_stored` (atomic, @@ -65,16 +180,19 @@ input is ever processed for that application. - **`clone_stored`** reflinks writable files (`clonefile` on macOS APFS, `FICLONE` on Linux btrfs/XFS), hardlinks read-only ones, and falls back to a sparse copy when the filesystem cannot link (`ENOTSUP`, `EXDEV`, `EPERM`). It - always works; it is cheap only on copy-on-write filesystems. + always works; it is cheap only on copy-on-write filesystems. It hard-links + only files without write permission, so a clone still loads with + `SHARING_ALL`; a copy hard-linked by hand (`cp -l`) does not. - **Snapshots come in two families.** An in-place (`SHARING_ALL`) machine snapshots by closing, cloning its directory, and reopening; this is the CLI's `--revert-mode=stored` and Dave's sling node. The CLI's default `--revert-mode=fork` instead forks a JSON-RPC machine server process, which - refuses to fork a machine with shared ranges (both processes would see the - writes). We run the machine in-process and use the clone family only. + refuses to fork a machine with shared backing stores, its hash-tree files + included (both processes would see the writes). We run the machine + in-process and use the clone family only. -Measured on APFS (Apple M5 Max) with a machine holding a 1 GiB NVRAM, 768 MiB -of it non-zero: +Measured with v0.21.0 on APFS (Apple M5 Max) with a machine holding a 1 GiB +NVRAM, 768 MiB of it non-zero: | Operation | Cost | |---|---| @@ -93,8 +211,8 @@ answering an inspect query: - **NVRAM** (`--nvram=label:,...`) appears in the guest as `/dev/uioN`. It has no filesystem layer and no page cache: guest writes through `mmap` are - immediately in the machine state. It requires the paired kernel with UIO - (ctsi-2 and later). + immediately in the machine state. It requires a kernel with UIO + (`CONFIG_UIO`, `CONFIG_UIO_PDRV_GENIRQ`), which the pinned kernel has. - **Flash drives** appear as `/dev/pmemN`. Guest writes go through the page cache, so the application must flush (`fsync`, `msync(MS_SYNC)`, or `O_DIRECT`) before finishing each input; otherwise the drive holds a stale @@ -112,11 +230,11 @@ answering an inspect query: ## The guest side -- **The guest–host interface is unchanged from v0.20.** Yield reasons, CMIO - buffer addresses, and ioctl numbers are the same; only the names moved to - `HTIF_*`. The revert root hash and the per-input cycle budget are host-side. - libcmt 0.18 bundles its ioctl header, so the guest build needs no kernel - headers. +- **The guest–host interface is the same from v0.20 through v0.22.** Yield + reasons, CMIO buffer addresses, and ioctl numbers match; only the names moved + to `HTIF_*`. The revert root hash and the per-input cycle budget are + host-side. libcmt 0.18 bundles its ioctl header, so the guest build needs no + kernel headers. - **Some guest tools are glibc builds.** The `nvram`/`flashdrive` label helpers, which the CLI's generated init script calls for every NVRAM and for flash drives it formats, mounts, or chowns, and `xhalt`, which `cartesi-init` calls @@ -151,14 +269,24 @@ The CLI is a script, not a module: it parses the global `arg` and ends in `cartesi.util`, `cartesi.hash-tree`, `cartesi.jsonrpc`). When tests run the CLI as a reference: -- `--cmio-advance-state` checks the outputs Merkle root from genesis by default; - resuming from a mid-history checkpoint needs `check_outputs_merkle_root:false`. -- `--max-mcycle` reached mid-input exits 0 and stores the mid-input state. +- `--cmio-advance-state` reads input files `input_file_index_begin` to + `input_file_index_end`−1 (the end defaults to the begin, so name both) and + rejects unknown keys. It checks the outputs Merkle root, starting from the + empty tree unless `last_output_proof` (or, with `resumable:true` and + `begin_epoch:false`, an `outputs_frontier` file) seeds it; a run resumed + from a mid-history checkpoint with neither needs + `check_outputs_merkle_root:false`. +- `--max-mcycle` reached mid-input exits 0 and stores the mid-input state + (without `resumable`). - Exit codes do not distinguish outcomes (a halt with payload 0 exits 0); load the stored machine and read its state instead, cross-checked with `--final-hash`. -- An inspect query is delivered even to a machine at an exception yield, and - its own outcome is not reported. +- `--assert-rolling-template` runs after `--store`: it requires an + `RX_ACCEPTED` yield whose TX buffer holds the expected outputs Merkle root + (by default the empty tree's, which libcmt writes on its first accept) and + exits 2 otherwise, leaving the stored image in place. +- `--cmio-inspect-state` asserts an `RX_ACCEPTED` yield before the query and + exits nonzero otherwise; the query's own accept or reject does not set the + exit code. - `--remote-spawn` leaves machine servers running unless `--remote-shutdown` is - also given. `--no-rollback` no longer exists; use `--no-revert`, which cannot - get past a rejected input: the emulator refuses the next advance. + also given. `--no-revert` aborts the run at the first rejected input. diff --git a/docs/invariants.md b/docs/invariants.md index 0a68a8d9..664dcec5 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -29,34 +29,12 @@ When you change anything listed under *enforced by*, re-check every line under - **Never absorb silently.** No `INSERT OR IGNORE`, saturating decode, or `unwrap_or_default` on data the contracts make impossible; use the loud variant of the same operation. -- **Command admission is fact-derived; terminal runtime faults abort the - process.** Three facts (the kernel process lock, two-sided - `setup_complete`, `canonical_divergence`), no admission state machine, no - operator acknowledgement, and a verdict-neutral black box; the statement, - the accepted trade, and the termination policy are owned by the - [authority-boundary ADR](plans/2026-08-authority-boundary-adr.md) - (mechanisms 1 and 2). What this policy adds: telemetry writes are - verdict-neutral — a failed black-box record loses only the black-box copy, - and the exit code and logs still carry the verdict. Runtime aborts skip - settlement entirely. The [I15](#i15-divergence-marker-present--acceptance-frontier-frozen) - freeze applies immediately to its named tables; runtime reaction still - requires a worker to observe the committed marker. -- **Maintenance is flush-only.** `flush-mempool` is an operator command, - not a startup-recovery alias: it settles the wallet nonce and never acquires - Sync/Cascade semantics. It requires completed setup and no divergence. - There is no verdict state for a flush to erase, and a successful wallet - flush proves nothing about the rest of the runtime. -- **Normal run repair and admission share one selection policy.** Local - absorbing facts are inspected before fallible provider facts; startup - selects one repair and checks its result. The flush's safe-block witness - is boot-local, and Sync must catch the persisted view up through it before - Cascade. Final admission checks one consistent fact set and yields - the single-use `RuntimeAdmission` witness consumed by the infallible, - non-yielding launch; no refusal or retry can construct it, and raw worker - and HTTP launch surfaces are crate-private. Design: - [`docs/recovery/README.md`](recovery/README.md). Mutation and output - authorization remains role-local at the durable boundaries documented - below; public low-level storage helpers are not an authority API. +- **Admission, terminal faults, the maintenance flush, and startup repair** + follow the [authority-boundary ADR](plans/2026-08-authority-boundary-adr.md) + (mechanisms 1–3) and the [recovery design](recovery/README.md). Raw worker + and HTTP launch surfaces are crate-private, and public low-level storage + helpers are not an authority API: mutation and output authorization stay + role-local at the durable boundaries this register lists. - An assertion must check a **real invariant** — true in every legitimate execution, including crash-recovery, replays, and clock steps — never an environmental assumption. (Cautionary tale: `sealed_at_ms >= created_at_ms` @@ -87,7 +65,6 @@ by writer and are write-once (`0001_schema.sql`). | setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows and rebuild root `batches`/`frames` (atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | | snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | obsolete `snapshots` and unreferenced `dumps` row deletion (`gc_unreferenced_dumps`), including a superseded baseline artifact | | command brackets (run, setup, flush) | `terminal_faults` (append-only, best-effort at settlement) | -| admin | `batch_policy` alpha knobs (`log_alpha`, `log_one_plus_alpha`) | | fee oracle | `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (Uniswap mode only; stamps on every successful refresh) | ### I1. Scheduler-acceptance semantics agree across all implementations @@ -97,21 +74,17 @@ by writer and are write-once (`0001_schema.sql`). `sequencer-core/src/scheduler/mod.rs`), the off-chain predicate (`ProtocolTiming::scheduler_accepts`, `sequencer-core/src/protocol.rs`), and the inclusion lane's live prediction produce the same - accept/reject/ordering decisions for every input. (Known, documented + accept/reject/ordering decisions for every input. The recovery fold + (`fold_replay`) drives `Scheduler` itself, not a copy. (Known, documented exception: the predicate omits the two structural rejections — self-trust, since the simulator only runs over the sequencer's own well-formed batches; - the omission is documented in `scheduler-semantics.md` and test-pinned by + the reason is recorded at `scheduler_accepts`, and it is test-pinned by the I1 duality test in `sequencer-core/src/scheduler/mod.rs`, which asserts the canonical fold and the predicate diverge exactly and only there.) - **Enforced by:** review + the duality test. No structural mechanism. - **Depended on by:** everything — the gold frontier, recovery's cascade pivot, checkpoint selection, soft-confirmation honesty. - **Breaks:** silent permanent scheduler/sequencer divergence. -- The expected-nonce fold is homed next to `scheduler_accepts` as - `advance_expected_batch_nonce`; `decide_submit_start` consumes it, while - `populate_safe_accepted_batches` keeps a deliberate inline copy (its advance - interleaves with storage-only side effects that can't move below the protocol - layer — see the call-site comment). ### I2. Drain attribution: accumulated directs land in the clock-advanced frame @@ -131,19 +104,14 @@ by writer and are write-once (`0001_schema.sql`). ### I3. Frame `safe_block`s are non-decreasing along the spine -- **Holds:** during an admitted live run, logical frame time advances directly - to the latest observed safe head `H` only when `H - S >= 5`, where `S` is the - open frame's persisted `safe_block`. An observation jump creates one frame at - `H` and resets the anchor; no intermediary frames are synthesized. Batch - closure may create a structural successor frame at the unchanged `S`, so - equality is valid. Bootstrap and recovery are anchoring transitions, not - live clock ticks: they may open a fresh Tip at a proven checkpoint/current - safe head without applying the five-block delta. -- **Enforced by:** lane flow + `append_safe_inputs`' monotonicity asserts +- **Holds:** frame `safe_block`s never decrease along the valid spine; equal + adjacent clocks are valid. The [frame-clock policy](protocol/scheduler-semantics.md#sequencer-frame-clock-policy) + says when the lane advances the clock, including the anchoring exception + for bootstrap and recovery. +- **Enforced by:** lane flow + `append_ingested_safe_inputs_with_timestamp`'s monotonicity asserts (`storage/l1_inputs.rs`) + - `ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS` (homed with its timing - siblings in `sequencer-core/src/protocol.rs`; prose owner is the - scheduler-semantics frame-clock section). The lane's frontier time gate + `ProtocolTiming::FRAME_CLOCK_INTERVAL_SAFE_BLOCKS` + (`sequencer-core/src/protocol.rs`). The lane's frontier time gate bounds SQLite observation load and is not part of the clock semantics. - **Depended on by:** `check_danger`'s arm ordering (see I4); the scheduler's within-batch monotonicity check; "if the frontier batch is fresh, all are". @@ -172,9 +140,12 @@ by writer and are write-once (`0001_schema.sql`). identifies the completely accounted L1 prefix. Reconciliation executes all external directs in the newly safe interval before committing its next frame and application rows. Envelopes-only intervals advance the clock with no rows. -- **Enforced by:** complete-block ingestion; the lane's indivisible reconciliation - turn; storage range and execution-receipt checks. Batch closure preserves the - current frame clock. +- **Enforced by:** complete-block ingestion: every committed `safe_inputs` / + `l1_safe_head` pair holds every input through its head, witnessed by index + contiguity plus the InputBox count at that head (`l1/reader.rs`), above a + scan floor setup pins only once the deployment is safe; the lane's + indivisible reconciliation turn; storage range and execution-receipt checks. + Batch closure preserves the current frame clock. - **Breaks:** skipping or double-applying directs after restart/recovery. ### I7. Every committed batch close has an immutable snapshot @@ -219,9 +190,9 @@ by writer and are write-once (`0001_schema.sql`). - **Depended on by:** the gold frontier, cascade pivot selection, checkpoint selection, local-state ↔ canonical-state agreement. - **Breaks:** would be silent divergence (a zombie replay of our own stale tx - winning a nonce slot; a power-loss re-seal at the same nonce with different - content); instead it is a detected `CanonicalDivergence` refusal whose - remedy is cockroach recovery. + winning a wallet-nonce slot; a power-loss re-seal at the same batch nonce + with different content); instead it is a detected `CanonicalDivergence` + refusal whose remedy is cockroach recovery. - **Completeness boundary:** the check completely enforces the accepted-batch identity predicate above; it is intentionally not a general canonical/application divergence oracle. The entire L1 prefix through baseline block `C` is opaque, @@ -254,9 +225,10 @@ by writer and are write-once (`0001_schema.sql`). - **Holds:** `safe_inputs` retains every InputBox observation. The storage/lane boundary selects external directs by the setup-pinned submitter address; only those inputs and included user ops enter `application_inputs`. -- **Enforced by:** classified direct reads and complete receipt validation at - append. Standard startup recovery attributes undrained directs to the new Tip; - lane catch-up executes them before processing queued user operations. Manual +- **Enforced by:** append-only `safe_inputs` triggers; classified direct reads + and complete receipt validation at append. Standard startup recovery + attributes undrained directs to the new Tip; lane catch-up executes them + before processing queued user operations. Manual rebuild represents the folded prefix through `C` in its baseline snapshot, with no application-history rows for that prefix. Replay and WS need no envelope filter because every row executes. @@ -265,8 +237,8 @@ by writer and are write-once (`0001_schema.sql`). ### I12. Safe head advances only on real observation; `synced_at_ms` is genuine progress time - **Holds:** the reader early-returns when the fetched head doesn't advance; - `append_safe_inputs` asserts monotonicity and stamps `synced_at_ms` only on - commit. + `append_ingested_safe_inputs_with_timestamp` asserts monotonicity and stamps + `synced_at_ms` only on commit. - **Enforced by:** reader floor check + asserts (`storage/l1_inputs.rs`). - **Depended on by:** the wall-clock danger arms (`L1ViewStale`, `EstimatedBatchInDanger`) — their baseline must be true progress time, or @@ -308,63 +280,57 @@ by writer and are write-once (`0001_schema.sql`). ### I15. Divergence marker present ⇒ acceptance frontier frozen -- **Holds:** a fully-accepted landing that - fails the content-identity check writes the `canonical_divergence` - singleton **in the same transaction** as the sync that detected it, and - `populate_safe_accepted_batches` returns early whenever the marker exists — - so no acceptance row or gold-frontier advance can ever - happen past a detected divergence. -- **Enforced by:** the `trg_*_frozen_on_divergence` trigger family - (`0001_schema.sql`) — specifically batch-tree writes and snapshot collection RAISE in the engine while the marker exists. This is - the immediate persisted freeze for those named tables, not a general - user-op hot-path barrier. The accepted frontier itself has no trigger: its - single writer refuses past the marker — the guard at the top of - `populate_safe_accepted_batches`. The typed error surface also includes - `check_danger`'s first arm (`CanonicalDivergence`, ranked ahead of every - other arm). Startup inspection refuses before any provider query, and - each repair transaction reasserts its durable preconditions and the - absence of divergence before writing. A clean worker drain waits for - in-flight reader appends: `run` then re-reads the marker on - its Ok path (`refuse_divergence_on_clean_exit`) and exits terminal rather - than 0, the one code that would break the supervisor's restart-then-refuse - rediscovery. The admission and preemptive TLA+ models verify the - controller ordering (`LocalTerminalDominance` in `admission.tla`) and - slot/batch safety respectively. -- **Runtime reaction:** the danger detector owns prompt process-wide reaction, - reading `check_danger` on its poll interval (`DANGER_DETECTOR_POLL_INTERVAL`). Independently, the inclusion lane's existing time-gated - SQLite read returns `SafeFrontierState::CanonicalDivergence` instead of an - `Open` frontier when the marker is already present. The lane then exits - with a terminal error, causing the supervisor to abort, before direct execution or the five-block rotation decision. This is opportunistic - refusal at an existing read, not another detector or a timing guarantee. - One bounded dequeue chunk (`max_user_ops_per_chunk`) is the fast-turn limit, so rejected traffic cannot - starve the read once its time gate is due. There is deliberately no - per-chunk marker query or extra poll. -- **Race bound:** a lane turn that already read `Open` may finish if the reader - commits divergence concurrently. Preventing that would require a lock or - transaction spanning application execution. Existing freeze triggers stop - conflicting batch-tree writes; the detector and next typed read - stop the process. A chunk committed before either runtime observation may - acknowledge and later roll back. -- **Watchdog boundary:** accepted-checkpoint selection checks for divergence - in the same transaction as selection and any download lease, refusing while - the marker is present. A matching batch - before a divergent acceptance in the same L1 block cannot represent that - block's final state. Because the watchdog skips replay when the finalized - inclusion block is unchanged, it - does not subsume this wire-identity detector. Conversely, the check does - not subsume the watchdog's broader independent application-state - comparison. +- **Holds:** a fully accepted landing that fails the content-identity check + writes the `canonical_divergence` singleton **in the detecting sync's + transaction**, and `populate_safe_accepted_batches` returns early while the + marker exists, so no acceptance row or gold-frontier advance follows a + detected divergence. +- **Enforced by:** the `trg_*_frozen_on_divergence` triggers + (`0001_schema.sql`) make batch-tree writes and snapshot collection RAISE + while the marker exists: an immediate, persisted freeze of those tables, not + a user-op hot-path barrier. The accepted frontier has no trigger; its single + writer's guard refuses. `check_danger`'s first arm is `CanonicalDivergence`. + Startup inspection refuses before any provider query, and each repair + transaction reasserts its preconditions and the marker's absence before + writing. A clean worker drain waits for in-flight reader appends; `run` then + re-reads the marker (`refuse_divergence_on_clean_exit`) and exits terminal, + not 0, the one code that would break the supervisor's restart-then-refuse + rediscovery. `admission.tla` checks the controller ordering + (`LocalTerminalDominance`); `preemptive.tla` checks slot and batch safety. +- **Runtime reaction:** the danger detector owns the prompt process-wide + reaction, polling `check_danger` every `DANGER_DETECTOR_POLL_INTERVAL`. + Independently, the lane's existing time-gated SQLite read returns + `SafeFrontierState::CanonicalDivergence` instead of `Open`, and the lane + exits terminal, so the supervisor aborts, before direct execution or the + five-block rotation decision. This is opportunistic refusal at an existing + read, not another detector or a timing guarantee, and there is deliberately + no per-chunk marker query or extra poll. One `max_user_ops_per_chunk` chunk + is the fast-turn limit, so rejected traffic cannot starve the read once its + time gate is due. +- **Race bound:** a lane turn that already read `Open` may finish while the + reader commits divergence; preventing that would need a lock or transaction + spanning application execution. The freeze triggers stop conflicting + batch-tree writes, and the detector or the next typed read stops the + process, so a chunk committed before either may acknowledge and later roll + back. +- **Watchdog boundary:** accepted-checkpoint selection checks the marker in + the same transaction as selection and any download lease, and refuses while + it is present. A matching batch before a divergent acceptance in the same L1 + block is not that block's final state, and the watchdog skips replay when + the finalized inclusion block is unchanged, so it does not subsume this + wire-identity detector. Nor does the check subsume the watchdog's broader, + independent application-state comparison. - **Depended on by:** standard recovery never running on a diverged frontier - (a flush+cascade there would compound the divergence); egress never - publishing a diverged landing; the remedy being cockroach recovery only. -- **Breaks:** silent permanent scheduler/sequencer divergence — the + (a flush and cascade there would compound the divergence); egress never + publishing a diverged landing; cockroach recovery as the only remedy. +- **Breaks:** silent permanent scheduler/sequencer divergence, the theft-equivalent failure. - **Baseline-aware frontier:** accepted-batch scanning starts strictly after - the immutable baseline L1 block `C`, seeded with anchor nonce `N'`. The whole - prefix is opaque, including previously rejected future-nonce batches; it must - never be reinterpreted using a later expected nonce. Subsequent scans resume - after the last acceptance. Rebuild defers the projection until the complete - baseline and anchor are published. + the immutable baseline L1 block `C`, seeded with anchor nonce `N'`; the + prefix through `C` is opaque + ([I9](#i9-acceptance-identity-accepted-nonce-n-means-our-valid-batch-n)). + Later scans resume after the last acceptance. Rebuild defers the projection + until the complete baseline and anchor are published. ### I16. The batch tree has exactly one valid parentless root, carrying the deployment's anchor nonce @@ -449,29 +415,6 @@ by writer and are write-once (`0001_schema.sql`). Checkpoint state, inclusion block, and next nonce are trusted operator inputs; neither clone detection nor distributed fencing is implied. -### Do-not-simplify (deliberate shapes that look like cleanup targets) - -The refactorer-facing mirror of the register above — each of these *looks* -like a simplification and would break a registered invariant: - -- **Don't move filesystem work into `storage/snapshot_dumps.rs`** — the - module boundary *is* the GC crash-ordering guarantee (I13). -- **Don't reorder `check_danger`'s arms** or merge its two `find_*` helpers - into one that consults the Tip first — the closed-frontier-first order is - the dispatch table's meaning (I4). -- **Don't derive application order from L1 positions** — optimistic user ops - precede their envelope and have no general one-to-one L1 mapping. -- **Don't retain a numeric resume offset without its history identity** — - recovery deliberately reuses suffix counts (I18). -- **Don't discard a valid snapshot beyond the accepted frontier** — that exact - batch can become the next required recovery checkpoint (I7). -- **Don't add internal retry loops to the flusher/submitter for provider - errors** — the orchestrator respawn is the retry mechanism; internal - retries mask exactly the failures the danger machinery routes on. -- **Don't unify the two staleness references** (inclusion-relative vs - current-relative) — deliberately different formulas for different - questions. - ### I19. Application progress follows the shared execution contract - **Holds:** `ApplicationProgress` is the pair @@ -519,3 +462,26 @@ like a simplification and would break a registered invariant: - **Performance boundary:** head discovery uses the integer primary-key maximum; replay seeks directly by offset and joins bounded source rows. Neither scans invalidated history. Chunk insertion adds no durability transaction or actor. + +## Do-not-simplify (deliberate shapes that look like cleanup targets) + +The refactorer-facing mirror of the register above — each of these *looks* +like a simplification and would break a registered invariant: + +- **Don't move filesystem work into `storage/snapshot_dumps.rs`** — the + module boundary *is* the GC crash-ordering guarantee (I13). +- **Don't reorder `check_danger`'s arms** or merge its two `find_*` helpers + into one that consults the Tip first — the closed-frontier-first order is + the dispatch table's meaning (I4). +- **Don't derive application order from L1 positions** — optimistic user ops + precede their envelope and have no general one-to-one L1 mapping. +- **Don't retain a numeric resume offset without its history identity** — + recovery deliberately reuses suffix counts (I18). +- **Don't discard a valid snapshot beyond the accepted frontier** — that exact + batch can become the next required recovery checkpoint (I7). +- **Don't add internal retry loops to the flusher/submitter for provider + errors** — the orchestrator respawn is the retry mechanism; internal + retries mask exactly the failures the danger machinery routes on. +- **Don't unify the two staleness references** (inclusion-relative vs + current-relative) — deliberately different formulas for different + questions. diff --git a/docs/l1-fee-policy.md b/docs/l1-fee-policy.md index 2292317b..0602c721 100644 --- a/docs/l1-fee-policy.md +++ b/docs/l1-fee-policy.md @@ -51,14 +51,13 @@ the Ethereum node's pending transactions. ## Why this policy -The [#34 review history](https://github.com/cartesi/sequencer/pull/34) explored -asymmetric bumps, remembered suffix hashes, compounding fee floors, and a -market-relative ceiling. Those implementations introduced invalid fee pairs, -payload/hash-association bugs, funding pressure, and recovery incompatibility. -[#35](https://github.com/cartesi/sequencer/pull/35) retains the independent -gas-estimation and flusher fee-validity fixes while removing that escalation -state. These failures motivate the current choice; they do not prove that -every bounded replacement policy would be unsound. +Escalating replacements were tried in +[#34](https://github.com/cartesi/sequencer/pull/34): asymmetric bumps, +remembered suffix hashes, compounding fee floors, and a market-relative +ceiling. They introduced invalid fee pairs, payload/hash-association bugs, +funding pressure, and recovery incompatibility. These failures motivate the +current choice; they do not prove that every bounded replacement policy would +be unsound. Revisit with evidence of sustained batch delays, recovery frequency or duration, and submission cost. Record the logs and affected L1 block range. diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 873892b6..a2b43f9e 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -12,38 +12,45 @@ freely at this stage — no backward-compatibility constraints. | # | Track | Owner | Status | |---|-------|-------|--------| -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **repository API implemented; post-merge adoption and deployment work remain** — [follow-up sequence](2026-07-track3-feed-replay-design.md#follow-up-sequence) and [ownership](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership) | -| 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented; client adoption and deployment work remain; frozen until the shared recovery tool and a new client sync** — [scope](#track-3--feed--replay-protocol-redesign) | +| 5 | Fee exponentiation LUT | us | **unblocked, needs Bart** — fees stay in log space; the batch-level alternative is deferred with [batch-scoped replay protection](2026-10-batch-scoped-replay-protection.md). Shape decided: exact-floor table, algorithm-free; a consensus change agreed with Bart before implementation; the log-additive `batch_policy` design is frozen until then | | 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | **Current campaign order:** -1. Merge the implemented egress API after repository review/checks. Bart can then integrate his client; adjust the API from concrete feedback without waiting for downstream completion. +1. Bart's client integrates the feed and treats cockroach recoveries as out of scope; we fix bugs it finds, and other feedback is input to the new sync. A shared recovery tool, still to be designed, will produce both the sequencer's and the client's startup state from a checkpoint. Until it lands, `setup --recovery`, `fold.rs` and the cockroach reader seams get bug fixes only. 2. Application integrators/operators validate the private engine, canonical-to-native exporter and recovery drill, and representative capacity before production use. Reference C-host lifecycle coverage is part of repository CI. -3. Track 5 (fee LUT) only after the log-space-fees decision. +3. Track 5 (fee LUT) after a sync with Bart, batched with other guest-image changes. Full restore archives now support file and directory application prefixes. Additional snapshot retention or transport mechanisms require a measured consumer need. ## Track 3 — Feed & replay protocol redesign -The current [history contract](../protocol/application-history.md) owns replica -bootstrap, history identity, replay, and recovery boundaries. The -[API contract](../../README.md#api) owns wire behavior. The wallet's cold replica -and canonical recovery/watchdog gates have a -[validation record](../review/2026-09-16-track3-validation.md). +The egress API is implemented. The [history contract](../protocol/application-history.md), +the [projection contract](../protocol/projection-replay.md) and the +[API contract](../../README.md#api) own its behavior; the +[integration plan](2026-07-track3-feed-replay-design.md) owns deployment follow-ups. -Readers whose projections contain information absent from the latest application -state can use the implemented fixed-prefix historical L1 API and checkpoint -metadata. The [projection contract](../protocol/projection-replay.md) owns that -workflow; the history contract owns implemented checkpoint compatibility across -standard recoveries. The [integration plan](2026-07-track3-feed-replay-design.md) -owns follow-up requirements and their owners. Native-engine integration and -representative deployment latency remain open after merge; they are not egress -API merge prerequisites. Other transport or retention mechanisms require a -measured need. +A shared recovery tool will produce the sequencer's and a client's startup +state from one checkpoint, and a new client sync follows its design. Until +then, what they replace gets bug fixes only, and its docs are not polished: -## Track 5 — Fee exponentiation LUT (deferred) +- Deleted when the tool lands: projection replay across a rebuild + (`/historical-l1-inputs`, the SDK's `historical_l1_inputs`, the main + `historical_bootstrap` test, the projection contract, and the integration + plan). First move the projection contract's seed/replay rules and worked + boundary into the tool's design and tests, move the integration plan's + deployment follow-ups to the register, and repoint the links to both files. +- Reshaped or retired with the new sync: the WS wire format, the SDK + snapshot-then-subscribe bootstrap, `/latest_snapshot` and its history headers, + `/history`'s baseline, accepted checkpoint and compatibility query, and the + era/generation identity types. In-era generation signalling survives in some form. + +Not frozen: the ingress routes, `/finalized_state` and its probes, +`/finalized_snapshot`, download leases, and the canonical page reader with `HistoryClaim`. + +## Track 5 — Fee exponentiation LUT `fee_to_linear` is consensus-critical (scheduler fold, guest agreement, app fee charging) and must be bit-identical across the Rust sequencer, the @@ -56,8 +63,10 @@ If built, the shape is decided: a full lookup table of exact 550 KB). The checked-in table *is* the cross-implementation spec artifact (algorithm-free, golden-hash-tested); `build.rs` verifies rather than generates; C++ consumes the same file byte-identically; and replay -continuity across the upgrade is explicitly not preserved. Do not implement -until the pending log-space-fees decision lands (with Bart). +continuity across the upgrade is explicitly not preserved. Fees stay in log +space; [batch-scoped replay protection](2026-10-batch-scoped-replay-protection.md) +records the deferred design that would move the fee to the batch. Agree the +table with Bart and ship it with the other guest-image changes. ## Track 6 — Dump / `Application` API redesign diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 31e06fe6..a4ed71df 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -14,12 +14,14 @@ The repository delivery includes the egress contracts, storage, SDK, and referen replay/recovery tests. Its merge criteria are review and the relevant repository checks. Private-engine integration, operational rehearsal, and representative capacity measurements are follow-up work; they do not block merging this API. -Merge the implementation, let consumers integrate, and adjust the API from -concrete feedback. There are no live deployments requiring compatibility. +Merge the implementation and let consumers integrate. Bugs they find are fixed; +other feedback is input to the new client sync +([Track 3](2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign)). +There are no live deployments requiring compatibility. | Follow-up | Owner | When it is needed | |---|---|---| -| Private DEX scheduler, indexer, and database backups | Bart / application integration | After merge, while adopting the API. Verify complete checkpoint/claim association and scheduler replay; report missing fields or awkward workflow for adjustment. | +| Private DEX indexer and database backups | Bart / application integration | While adopting the API. Report bugs for fixing; missing fields or awkward workflow feed the new client sync. | | Canonical-to-native export and incident rehearsal | Application integration and operators, under Track 6 | Before relying on that application's recovery procedure in production. | | Ingress latency, indexing headroom, and recovery capacity | Sequencer/application maintainers and deployment operators | Before claiming support for the target deployment workload; measure historical serving alongside ordinary traffic. | @@ -76,20 +78,18 @@ replacement directs, empty/no-op invalidation, nonzero baselines, and rollback. |---|---| | Historical bootstrap | Implemented: replay the complete fixed prefix and join the feed at `K`, including after a nonzero rebuild. | | Standard recovery | Implemented: choose the nearest retained checkpoint whose prefix survived every intervening generation change, then restore/resubscribe. | -| Cockroach reader recovery | Core replay/metadata workflow and reference checkpoint restore implemented; Bart's actual checkpoint preparation, incident validation, and fallback rehearsal remain. | +| Cockroach reader recovery | Core replay/metadata workflow and reference checkpoint restore implemented. Superseded for clients by the planned shared recovery tool; bug fixes only, with no client follow-up. | | Stable coordinates | Existing era/generation/application count for claims; separate InputBox indices for raw paging. Counts alone do not certify cross-era compatibility. | | Bounded serving cost | Page/item/response bounds implemented; representative bootstrap must preserve the ingress latency target and demonstrate catch-up headroom. | | Operational readiness | Each production application supplies its canonical-to-native exporter, runbook, and non-genesis recovery drill under Track 6. | ### Follow-up sequence -1. **Consumer adoption after merge.** Bart integrates accepted-boundary checkpoint - preparation and scheduler replay using the implemented API. Sequencer - maintainers address concrete feedback as it arrives; downstream completion - is not a prerequisite for repository delivery. Before production use, rehearse - identifying an unsound projection checkpoint, including one below new `K`, - and restoring an earlier trusted backup or genesis. The API does not certify - the client's projection. +1. **Consumer adoption.** Bart's client follows the feed and treats cockroach + recoveries as out of scope. A planned shared recovery tool will produce both + the sequencer's and the client's startup state from a checkpoint, so clients + never resume across a rebuild; new client work should not build on the + projection-replay path, which receives bug fixes only. 2. **Deployment readiness.** Validate the private native engine and measure latency, historical serving cost, projection throughput, catch-up headroom, and recovery time. Track 6 independently owns the versioned diff --git a/docs/plans/2026-10-batch-scoped-replay-protection.md b/docs/plans/2026-10-batch-scoped-replay-protection.md new file mode 100644 index 00000000..89510eed --- /dev/null +++ b/docs/plans/2026-10-batch-scoped-replay-protection.md @@ -0,0 +1,137 @@ +# Batch-scoped replay protection (deferred) + +**Status:** deferred design, not implemented. The current rule is the +per-sender nonce in the [application contract](../protocol/application-contract.md#user-nonces). +Revisit when replay state must become prunable, or when the fee moves from the +frame to the batch. + +## Why + +The nonce rule makes replay protection depend on all history. Every account's +expected nonce lives in application state forever, so state cannot be pruned. +That state is the "historical fold" each implementation must carry. + +Moving the fee to the batch, signed by the user, hits a race. A user who reads +the current fee, signs, and submits can find the batch already sealed. The +batch-scoped design fixes that race too. + +## Design + +**Signed coordinates.** The user signs `(b, i)` in place of a nonce: + +- `b` is the scheduler batch nonce the op targets: the canonical batch sequence + number, not the sequencer's local `batch_index`. +- `i` orders the sender's ops within `b`. + +**Look-back.** The sequencer adds an unsigned field `k` (2 bits, so `k` ∈ 0..=3) +when it builds the batch. Executing batch `c`, the scheduler recovers +`b = c - k` and verifies the signature over it. `b` itself never goes on the +wire. A user who signed for a batch that sealed meanwhile is still included, +in a later batch with `k ≥ 1`. + +**Acceptance.** Per sender, keep the last consumed `(b, i)`. Accept an op only +if its `(b, i)` is lexicographically greater, then store it. + +- `i` need not equal the op's position or start at 0, so a rejected op does not + break the ones after it. +- Ops of one sender targeting the same `b` may split across batches `b..b+3`; + order is still enforced. + +**Pruning.** Drop a sender's entry once its stored `b < c - 3`. This is sound: +any op still in the window has `b ≥ c - 3`, so it would pass the comparison +against the dropped entry anyway. State is bounded by the senders active in the +last four batches, not by every account ever seen. + +**Why the state is needed.** Without look-back (`k = 0` always), uniqueness of +`i` within batch `b` is checkable inside the batch, and the rule is stateless. +With look-back, a check confined to the including batch is replayable by a +Byzantine sequencer: + +``` +user signs X with (b=10, i=0) +batch 10: X with k=0 → recovers b=10, accepted +batch 11: X with k=1 → recovers b=10, accepted again under a within-batch check +``` + +The per-sender `(b, i)` entry rejects the second inclusion. + +## Fee at the batch + +- Each batch carries one fee, fixed when the batch opens and published before + users sign. Signing `b` commits the user to `fee(b)`; no `max_fee` is signed. +- An op included through look-back pays `fee(b)`, not `fee(c)`. The scheduler + therefore keeps the fees of the last four batches, a bounded addition to its + state. +- Users minimize cost by targeting the cheapest batch in the window, so a fee + increase fully applies only after `look-back` batches. The 10× fee headroom + over DA cost absorbs that delay. + +## Wire and DA + +- Per op, the 4-byte nonce and 2-byte `max_fee` are replaced by a small `i` + plus 2 bits of `k`. The fee appears once per batch, not per frame. +- The distribution of `i` is skewed toward 0, which leaves room for later + compression. + +## Recovery and invalidation + +A stale batch does not consume its nonce, so a cascade rolls the open batch +back and replacements reuse the nonces. An op from an invalidated batch never +executed, but its signature stays executable while canonical `c <= b + 3`, +including when its `b` is above the rolled-back open batch. That breaks +re-signing, which today's nonce makes safe: + +``` +user signs X with (15, 0); batches 10..15 are invalidated; open batch is 10 +user re-signs X' with (10, 0); it executes and stores (10, 0) +canonical c reaches 15: X arrives with k=0, (15, 0) > (10, 0), accepted again +``` + +The coordinates name a nonce, not a batch's content, so a lower re-signed copy +does not exclude the original. Adopting the design requires binding ops to +their branch; [open questions](#open-questions) lists tentative fixes. + +The fee commitment has the same gap: a replacement batch at a reused nonce +`b` opens with its own fee, so an op signed against the original `fee(b)` can +be charged a fee it never saw. + +The window state (per-sender `(b, i)`, recent fees, and any anchors) is +canonical state. Checkpoints and the cockroach recovery tool must carry it. + +## What changes together + +This is one coordinated contract change, agreed with the engine owner: + +- the EIP-712 `UserOp` type; +- `WireUserOp`, and `Frame`/`Batch` (the fee moves to the batch); +- scheduler acceptance, the application contract's nonce rule, and the C ABI's + validation inputs; +- `GET /nonce` and `GET /fee` (they become the open batch's nonce and fee, plus + the sender's next `i`), and the SDK. + +## Open questions + +- **Binding ops to their branch (required before adoption).** Tentative fixes, + not yet analyzed in depth. Most promising: each op also signs an anchor, the hash of the canonical batch at nonce `b - 1`, which the + scheduler keeps for the window. A rebuilt `b - 1` then invalidates every + signature made against the old branch, as a recent block hash does elsewhere, + and re-signing becomes safe. Alternatives: clients and `GET /nonce` never + quote coordinates below a sender's outstanding signed op until `c > b + 3`; + or the sequencer re-includes invalidated ops itself and users never re-sign. +- **Signing the fee.** Sign `fee(b)` without putting it on the wire: the + scheduler fills it from its window, so a replacement batch with another fee + makes the op recover a different address and skip, instead of overcharging. + Or state that a replacement may charge its own fee, within policy. +- **Window width in time.** Four batches shrink to seconds under load, when + batches seal on size. Size `k` against a minimum batch lifetime, or bound the + sealing cadence. +- **Owner of the window state.** Replay protection is protocol-level. The + scheduler could own it at the shared execution boundary, which would remove + nonce logic from every application. +- **Width and encoding of `i`,** and its exhaustion rule. + +## Why deferred + +The per-sender nonce works, and the engine owner no longer needs to prune +replay state. The change spans signing, wire format, scheduler, ABI and API at +once, so it waits for a concrete trigger. diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 7776b9e3..add97ae6 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -119,12 +119,20 @@ its post-rebuild ops. An engine outside the rule (gapped or keyed nonces, a deposit that resets one) makes `GET /nonce` quote nonces the engine rejects. +The rule keeps every account's nonce in state forever. A batch-scoped +alternative makes replay state prunable and lets the fee move to the batch; it +is a [deferred design](../plans/2026-10-batch-scoped-replay-protection.md). + ### 3. The safe-block clock — `last_executed_safe_block` The clock is the maximum block carried by an executed input: frame `safe_block` for user ops, L1 inclusion block for directs, or zero at genesis. On success, -the engine advances it with `max(old_clock, input_clock)`. Count zero implies -clock zero. Both fields survive checkpoint round-trips. +the engine advances it with `max(old_clock, input_clock)`. Input clocks are +not monotone: [gate d](scheduler-semantics.md#processing-an-input) orders +frame clocks only within a batch, so a later batch's user ops may run at a +`safe_block` below an earlier input's clock (a direct never runs below the +stored clock); the `max` keeps the stored clock monotone. Count zero implies clock zero. Both +fields survive checkpoint round-trips. The [frame-clock policy](scheduler-semantics.md#sequencer-frame-clock-policy) may deliver a delayed head jump as one step. Newly covered directs execute diff --git a/docs/protocol/application-history.md b/docs/protocol/application-history.md index ed3e3bff..f249d67e 100644 --- a/docs/protocol/application-history.md +++ b/docs/protocol/application-history.md @@ -181,7 +181,7 @@ writers become supported. | Boundary | Code and tests | |---|---| | Coordinates and claim ordering | [`sequencer-core/src/history.rs`](../../sequencer-core/src/history.rs) — identity before position, inclusive head, checked counts. | -| Coherent application pages | [`storage/egress/canonical.rs`](../../sequencer/src/storage/egress/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | +| Coherent application pages | [`storage/history/canonical.rs`](../../sequencer/src/storage/history/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | | Replay followed by live delivery | [`l2_tx_feed`](../../sequencer/src/egress/l2_tx_feed/) — bounded deep replay, identity refusals, shutdown and persistent faults; [`catch_up.rs`](../../sequencer/src/ingress/inclusion_lane/catch_up.rs) for native replay. | | Artifact and claim association | [`snapshot_endpoints.rs`](../../sequencer/src/integration_tests/snapshot_endpoints.rs) — headers, restore, archive contents, and lease lifetime. | | Checkpoint compatibility | [`storage/history.rs`](../../sequencer/src/storage/history.rs) and recovery tests — immutable cuts, complete lineage, and transaction rollback; [`recovery_compatibility.rs`](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) — saved projections across missed recoveries and HTTP lookup/WS admission races. | diff --git a/docs/protocol/projection-replay.md b/docs/protocol/projection-replay.md index 7278c485..1667c59b 100644 --- a/docs/protocol/projection-replay.md +++ b/docs/protocol/projection-replay.md @@ -8,6 +8,10 @@ scheduler remains the ordering authority. The [README API](../../README.md#histo owns routes, fields, limits, and refusals; the [history contract](application-history.md) owns application coordinates. +The planned shared recovery tool supersedes this workflow, so it gets bug fixes +only; new client work should not build on it +([Track 3](../plans/2026-07-coordination-tracks.md#track-3--feed--replay-protocol-redesign)). + ## Bootstrap and handoff 1. Read `/history`. Pin the deployment identity and era, baseline application diff --git a/docs/protocol/scheduler-semantics.md b/docs/protocol/scheduler-semantics.md index b76a43d6..195cf00f 100644 --- a/docs/protocol/scheduler-semantics.md +++ b/docs/protocol/scheduler-semantics.md @@ -1,131 +1,189 @@ # Scheduler acceptance semantics -The canonical ordering rule. The on-chain scheduler reads a stream of L1 safe -inputs and folds them into application state in one deterministic order. Every -other component that *predicts* that order — the gold frontier, the inclusion -lane, the recovery fold — must reproduce this algorithm exactly. Disagreement is -silent scheduler/sequencer divergence, the most severe failure in the system -([I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)). - -This document **owns** that algorithm. [`AGENTS.md`](../../AGENTS.md) §"Sequencer / -Scheduler Duality" is the map; this is the detail. The reference implementation -is [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs) — the same source -compiled into the on-chain canonical machine and run bare-metal by the recovery -fold, so those two targets agree *by construction*. This prose is the intended -protocol contract; the reference implementation embodies it but remains code -that can contain bugs and require hardening. - ---- +The canonical scheduler folds an application's L1 inputs, in L1 order, into +application state. This page specifies that fold for anyone who reimplements +it. The reference is [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs), +run in the canonical machine by the [canonical-app harness](../../examples/canonical-app/src/scheduler/mod.rs). +This text is the contract and the reference can have bugs, so report any +disagreement. The [application contract](application-contract.md) owns what +the application does with each input. Rules marked **pending** describe the +reference today; changing one changes the canonical machine image or its wire +format, so they are settled with the C++ port's author +([register](../review/register.md#canonical-rules-open-for-the-c-port)). ## The one stream, the one classification -The scheduler consumes `InputAdded` events from the InputBox in L1 order. Each -event carries an authenticated `msg_sender`. Classification is **by sender -address, never by a tag byte** ([`process_input`](../../sequencer-core/src/scheduler/mod.rs)): +Each input is a Cartesi Rollups `EvmAdvance` from the InputBox. The scheduler +reads its `msgSender`, its `blockNumber` (the *inclusion block*), its `chainId` +and `appContract` (the signature domain), and its `payload`. It classifies by +sender only, never by payload content: -| Sender | Treated as | Effect | -|---|---|---| -| `== sequencer_address` | a **batch** (SSZ-encoded `Batch`) | folded through the acceptance algorithm below | -| anything else | a **direct input** (deposit) | appended to the *fridge* (the direct-input FIFO), drained later | +- `msgSender == sequencer_address`: the payload is a **batch**. +- Any other sender: a **direct input**, appended unchanged to the *fridge*, + the FIFO of directs not yet executed. -The payload is opaque to classification; app-specific decoding happens inside -the `Application::apply_direct_input` hook, reached through -the shared `execute_direct_input` boundary (see the -[application contract](application-contract.md)). +`sequencer_address` is deployment data compiled into the canonical machine. It +must equal the batch-submitter address that `setup` pins; nothing checks this +at runtime. If they differ, the machine treats every batch as a direct input +while the sequencer still reports its batches accepted, so only a watchdog +comparison reveals it. ---- +The scheduler's state is `next_batch_nonce` (`u64`, 0 at genesis), the fridge, +and the application, which holds its executed-input count and clock. Nothing +else carries over between inputs: no earlier batch's `safe_block` is kept. The +fridge has no size bound (**pending**). -## The algorithm (`process_input`) +## Wire format -Every input — batch or direct — first runs the **censorship backstop**, then is -classified. +A batch payload is the SSZ encoding of `Batch` +([`batch.rs`](../../sequencer-core/src/batch.rs)), fields in this order: -``` -process_input(input): - 1. force_execute_overdue(input.inclusion_block) ── backstop, runs for EVERY input - 2. if input.sender != sequencer_address: - fridge.push_back(input) ── DirectEnqueued - else: - process_batch_payload(input) ── the acceptance algorithm +```text +Batch { nonce: uint64, frames: List[Frame] } +Frame { user_ops: List[WireUserOp], safe_block: uint64, fee_price: uint16 } +WireUserOp { nonce: uint32, max_fee: uint16, data: List[uint8], signature: List[uint8] } ``` -### Step 1 — Force-execute overdue directs (the backstop) +Integers are little-endian. Each variable-size field has a 4-byte offset in its +container's fixed part, and the variable parts follow in field order. A list of +containers starts with one 4-byte offset per element. Lists have no length +limit. Decoding is strict: offsets must start right after the fixed part, never +decrease and stay in bounds, so only the canonical encoding of a `Batch` +decodes, and re-encoding returns the same bytes. +`max_fee` and `fee_price` are fee exponents ([constants](#constants)). A +`signature` of any length decodes, but only 65 bytes can recover a sender; a +fixed 65-byte field is **pending**. + +## User-op signatures + +A user op signs the EIP-712 struct `UserOp(uint32 nonce,uint16 max_fee,bytes data)` +([`user_op.rs`](../../sequencer-core/src/user_op.rs)) under the domain +`EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)`: +name `CartesiAppSequencer`, version `1`, the input's `chainId`, its +`appContract` as `verifyingContract`, and no salt +([`build_input_domain`](../../sequencer-core/src/lib.rs)). The sender is not +signed; it is recovered from `r ‖ s ‖ v` (alloy's `Signature::from_raw` and +`recover_address_from_prehash`), and any failure drops the op: + +1. The signature must be 65 bytes, and `v` must be 0, 1, 27, 28 or at least 35. + The y-parity is `v` for 0 and 1; for `v ≥ 27` it is 1 when `v` is even, + else 0. +2. `r` and `s` must lie in `[1, n − 1]`, where `n` is the secp256k1 order. +3. If `s > n / 2`, `s` becomes `n − s` and the parity flips, so a high-`s` + signature and its low-`s` twin of opposite parity recover the same sender. +4. The sender is the address of the public key recovered from `(r, s, parity)` + and the EIP-712 signing hash. + +This acceptance set is the library's, pinned on both the batch path and +`POST /tx` by the signature vectors except for `s ≥ n` and `v` from 29 to 34; +it is **pending**. Batches the sequencer +builds carry `v ∈ {27, 28}`, but keep a user's `s` as signed. + +## Processing an input + +Every input first runs the **censorship backstop**: pop and execute directs +from the front of the fridge while +`input.inclusion_block − front.inclusion_block ≥ MAX_WAIT_BLOCKS`. This bounds how +long a sequencer that stops submitting, or freezes `safe_block`, can hold a +direct back. Then a direct joins the fridge, and a batch runs these gates in +order; the first that applies decides the outcome: + +| # | Gate | Outcome | `next_batch_nonce` | +|---|---|---|---| +| a | payload does not SSZ-decode as `Batch` | rejected | kept | +| b | `batch.nonce != next_batch_nonce` | rejected | kept | +| c | `frames` is empty | executed, as a no-op | +1 | +| d | frame structure is invalid (below) | rejected | kept | +| e | `inclusion_block − frames[0].safe_block ≥ MAX_WAIT_BLOCKS` | skipped as stale | kept | +| f | otherwise: execute every frame (below) | executed | +1 | + +Ages saturate at zero. An empty batch is valid at any age, since it has no +first frame (**pending**). Gate d rejects a batch whose frames include a +`safe_block` above `inclusion_block`, or a `safe_block` lower than the previous +frame's. It looks only inside the batch: a later batch may start below an +earlier one's last `safe_block`, and its user ops then see the lower value. A +stale batch keeps the nonce, so every later batch fails gate b until one +reuses that nonce, as a batch rebuilt after sequencer recovery does. A +rejected and a skipped batch have the same effect: no state change, nonce +kept. The reject reason the reference reports is diagnostic. + +### Frame execution (gate f) + +For each frame, in order: + +1. **Drain covered directs.** Pop and execute directs from the front of the + fridge while their `inclusion_block ≤ frame.safe_block`. The `safe_block` is + the sequencer's claim that it accounted for every direct up to that block; + the scheduler enforces the drain whatever the sequencer did. +2. **Execute the user ops** in order. Skip an op, with no effect, if (checked + in this order) its sender cannot be recovered, `max_fee < frame.fee_price`, + the application rejects it at fee `frame.fee_price`, or its `nonce` is + `u32::MAX`, which has no successor. Otherwise apply it with fee + `frame.fee_price` and clock `frame.safe_block` + ([`validate_and_execute_user_op`](../../sequencer-core/src/application/mod.rs)). + +The scheduler does not range-check `fee_price`; the reference wallet halts on +converting an exponent above `MAX_EXPONENT` (**pending**). + +## Outputs and the executed-input count + +An input's outputs are the application's notices and vouchers in execution +order: overdue directs, then each frame's drained directs and applied ops. +Reports are diagnostics outside the outputs tree: the reference reports a +rejected batch with one fixed string and a stale skip not at all, and the +[golden guest tests](../../examples/canonical-test/src/golden.rs) ignore them. + +The executed-input count is canonical state too. It advances by one for each +executed direct and each applied user op, including an app-level no-op or +business failure, and for nothing else: not for queueing a direct, a skipped +op, a batch envelope or an empty batch. A rejected or stale batch can still +advance it through the backstop. The [application contract](application-contract.md#4-canonical-history-cursor-executed_input_count) +owns the count; the [history contract](application-history.md) owns how +catch-up, recovery and the API use it. + +## Fatal conditions + +The reference panics, halting the machine, when the application returns an +error; when the batch nonce or the executed-input count has no `u64` +successor; and when `blockNumber` or `chainId` does not fit `u64`. A halt is a +permanent fixed point, not a reject ([rollup host semantics](../cartesi-machine.md#rollup-host-semantics)), +so a port must halt on the same inputs. No canonical successor state exists. + +## Constants + +| Constant | Value | Home | +|---|---|---| +| `MAX_WAIT_BLOCKS` | 1200 L1 blocks, about 4 h | [`lib.rs`](../../sequencer-core/src/lib.rs) | +| `MAX_EXPONENT` | 17101 | [`fee.rs`](../../sequencer-core/src/fee.rs) (`build.rs` generates it) | +| `sequencer_address` | per deployment | compiled into the canonical machine | -Before *anything* else, drain every fridge direct that has gone **overdue**: -`current_block − direct.inclusion_block ≥ MAX_WAIT_BLOCKS` -([`force_execute_overdue`](../../sequencer-core/src/scheduler/mod.rs)). This is -the censorship-resistance guarantee: a direct cannot be held in the fridge -forever by a sequencer that freezes `safe_block` or stops submitting batches — -once it ages out, it executes regardless of any frame. It runs on every input -tick (even a malformed batch advances `current_block` and can trip the -backstop). Drained directs execute in FIFO (ascending `inclusion_block`) order. +A fee exponent `n` stands for `(129/128)^n` smallest token units. The +application converts it; [`fee_to_linear`](../../sequencer-core/src/fee.rs) +owns the bit-exact conversion. A checked-in lookup table in its place is +**pending** ([Track 5](../plans/2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut)). -### Step 2 — Batch acceptance (`process_batch_payload`) +## Conformance -A batch from the sequencer address runs this gauntlet, in order. The first -failing gate decides the outcome; only `BatchExecuted` (including the empty-batch -case) **consumes the nonce**. +[`consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json) +pins EIP-712 hashes, keccak256, the signature cases above (every accepted `v` +form, `v = 2`, high `s`, 64 bytes, `r = 0`, `s = 0`, `r = n`) with the sender +each path recovers, SSZ bytes, fee conversions, `EvmAdvance` calldata, and +guest scenarios with their exact outputs. It is never regenerated: a mismatch +is a consensus change. Gate edge cases are pinned by the reference's unit tests +and the [guest tests](../../examples/canonical-test/src/main.rs); vectors for +them are **pending**. -| # | Gate | On failure | Nonce? | -|---|---|---|---| -| a | SSZ-decode the payload as `Batch` | `BatchRejected(DecodeFailed)` | not consumed | -| b | `batch.nonce == next_expected_batch_nonce` | `BatchRejected(WrongNonce)` | not consumed | -| c | *empty frames?* → accept as a no-op | `BatchExecuted` | **consumed** | -| d | structural frame check (below) | `BatchRejected(SafeBlockAboveInclusionBlock \| NonMonotonicSafeBlocks)` | not consumed | -| e | staleness: `inclusion_block − frames[0].safe_block ≥ MAX_WAIT_BLOCKS` | `BatchSkippedStale` | **not consumed** | -| f | execute every frame (below) | `BatchExecuted` | **consumed** | - -**Gate d — structural frame check** -([`batch_reject_reason_for_block`](../../sequencer-core/src/scheduler/mod.rs)): - -- every frame's `safe_block ≤ inclusion_block` (a frame cannot claim to have - accounted for L1 state it was included before) — else - `SafeBlockAboveInclusionBlock`; -- frame `safe_block`s are **non-decreasing** across the batch — else - `NonMonotonicSafeBlocks`. - -**Gate e — staleness** is measured against the **first** frame only -(`has_elapsed_since`, mirrored off-chain by -[`ProtocolTiming::is_scheduler_stale`](../../sequencer-core/src/protocol.rs)). A -stale batch is a **true no-op in nonce space**: the expected nonce does *not* -advance, so the next batch — including a freshly-rebuilt one reusing the same -nonce after recovery — is what the scheduler accepts next. This is the -mechanism behind cascading invalidation (AGENTS.md §"Cascading invalidation"). - -### Step 2f — Frame execution order (drain-before-ops) - -For each frame, in order ([`process_batch_payload`](../../sequencer-core/src/scheduler/mod.rs)): - -1. **Drain covered directs first**: pop every fridge direct with - `inclusion_block ≤ frame.safe_block` and execute it - ([`drain_directs_safe_at`](../../sequencer-core/src/scheduler/mod.rs)). The - `safe_block` is the sequencer's commitment that it has accounted for all - directs up to that block; the scheduler enforces the drain regardless. -2. **Then execute the frame's user ops**, each through the single entry point - [`validate_and_execute_user_op`](../../sequencer-core/src/application/mod.rs) - with `(frame.fee_price, frame.safe_block)`. A user op is silently skipped - (no state change, no output) when its signature is unrecoverable, when app - `validate_user_op` rejects it, or when a protocol guard fails - (`max_fee ≥ fee_price`, or a nonce of `u32::MAX`, which has no - successor) — the fold is a pure deterministic function and emits no - diagnostics at the library seam. - -An `AppError` from either execution path is fatal: there is no canonical -successor to the partially evaluated input, so the canonical harness, -inclusion lane, catch-up, and recovery fold all stop rather than continue. - -This ordering — directs ≤ `S_K` then ops validated on top of them — is exactly -what the inclusion lane writes into frame K's wire content -([I2](../invariants.md#i2-drain-attribution-accumulated-directs-land-in-the-clock-advanced-frame)), -which is what keeps the off-chain prediction faithful. +## Sequencer side + +The rest of this page is for sequencer developers. ### Sequencer frame-clock policy -The scheduler constrains frame clocks to be non-decreasing and no later than -the batch inclusion block; it does not require a frame for every L1 block. -During an admitted live run, the sequencer advances logical frame time when the -latest persisted safe head `H` is at least five blocks beyond the open frame -clock `S`. It then opens exactly one frame at `H`, drains the complete +Gate d constrains frame clocks only within a batch: non-decreasing and no later +than its inclusion block. The scheduler does not require a frame for every L1 +block. During an admitted live run, the sequencer advances logical frame time +when the latest persisted safe head `H` is at least five blocks beyond the open +frame clock `S`. It then opens exactly one frame at `H`, drains the complete newly-covered direct range, and uses `H` for subsequent user ops. An observation jump from 100 to 132 therefore creates one frame at 132, not synthetic frames at each missed five-block boundary. Intermediate empty frames would execute @@ -147,156 +205,15 @@ deposit budget. If actual safe-head publication cadence threatens that budget, revisit the constant from measurements rather than adding interpolation or a second timer. ---- - -## The application-history offset - -The feed offset is semantically -[`Application::executed_input_count()`](application-contract.md#4-canonical-history-cursor-executed_input_count). -It starts at zero and names the next application input to execute. An -application at count `X` is ready for history entry `X`; successfully executing -that entry advances the application to `X + 1`. - -The scheduler determines which InputBox material becomes an executed -application input. Its count transitions are therefore: - -| Scheduler event | Application execution | Count effect | -|---|---|---| -| Direct arrives and is appended to the fridge | none yet | unchanged | -| Covered or overdue direct is successfully executed | `execute_direct_input` returns `Ok` | exactly `+1` | -| User op passes signature, protocol, and app validation and is successfully executed | `execute_valid_user_op` returns `Ok` | exactly `+1` | -| User op has an unrecoverable signature or is rejected by protocol/app validation | none | unchanged | -| Batch envelope is decoded, accepted, skipped, or rejected | none for the envelope itself; contained executions are counted by the rows above | unchanged for the envelope | -| Empty batch | none | unchanged | -| Either execution method returns `AppError` | fatal invariant failure | no canonical successor is defined | - -The censorship backstop still runs before classification, so a malformed, -stale, or otherwise rejected batch can indirectly advance the count by causing -overdue directs to execute first. The batch itself never contributes an entry. - -This coordinate must agree across the canonical fold, the inclusion lane, -catch-up/replay, and recovery. Standard recovery may replace an invalidated -suffix at the same application offsets; cockroach recovery resumes from the -absolute count persisted in the recovered application state even when older -history is no longer locally available. - -The durable `application_inputs` sequence uses these same offsets. HTTP -snapshots carry the history version and application count; WS subscriptions -must claim both before inclusive replay. See the [history contract](application-history.md). - ---- - -## The three implementations (and why they agree) - -I1 names three places this algorithm lives. They are not three rewrites; two are -*derived* from the first. - -| # | Implementation | Role | Source | -|---|---|---|---| -| 1 | **Canonical fold** `Scheduler` | on-chain authority + bare-metal recovery fold | [`scheduler/mod.rs`](../../sequencer-core/src/scheduler/mod.rs) | -| 2 | **Off-chain predicate** `scheduler_accepts` + `advance_expected_batch_nonce` | builds the gold frontier (`safe_accepted_batches`) | [`protocol.rs`](../../sequencer-core/src/protocol.rs) | -| 3 | **Inclusion lane live prediction** | drains + executes ahead of L1 to issue soft confirmations | [`ingress/inclusion_lane/`](../../sequencer/src/ingress/inclusion_lane/) | - -- **#1 is the authority.** It is the literal scheduler source; the recovery - engine [`fold_replay`](../../sequencer-core/src/scheduler/fold.rs) *drives* it - (seeds the fridge, replays the stream), so recovery is consistent with L1 by - construction rather than by a parallel re-implementation. It is not a fourth - copy. -- **#2 is a predicate, not the full fold.** `scheduler_accepts` answers only - "would the scheduler accept this batch into the frontier?" — it checks sender, - decode, staleness, and nonce, and **deliberately omits gate d** (the - structural frame checks). That omission is sound by *self-trust*: the frontier - simulator runs over the sequencer's **own** batch submissions, which are - well-formed by construction; a structurally-malformed batch would be a - self-bug (a fault state to crash on), not an adversarial input to predict. - This document is the cross-reference home for the omission — the asymmetry is - intentional, not a missing check. -- **The content-identity check is complete relative to #2, not an independent oracle for #1.** For - every at/above-anchor landing that `scheduler_accepts` accepts, the frontier - builder exhaustively finds a byte-identical valid local closed batch - (`Match`), no local batch (`Foreign`), or different bytes (`Mismatch`); the - latter two durably freeze the frontier. Because the check shares #2 and its - structural omissions, it cannot prove that #1, application state, or trusted - collapsed history is correct. Its false-positive boundary is recorded once - at I9; see I9/I15 for the runtime and recovery boundary. -- **The expected-nonce fold** is homed once, next to `scheduler_accepts`, as - [`advance_expected_batch_nonce`](../../sequencer-core/src/protocol.rs). The - submitter's `decide_submit_start` consumes it directly. The frontier builder - `populate_safe_accepted_batches` keeps a deliberate inline copy of the same - advance — its loop interleaves the advance with two storage-only side effects - (the content-identity check and the `canonical_divergence` freeze) that - cannot move below the protocol layer; sharing the fold there would force a - callback contract. The duplication is intentional and documented at the call - site. - -**Why the agreement is load-bearing:** the gold frontier (#2) is what -recovery's cascade pivots on and what accepted-checkpoint selection trusts; the lane (#3) is what -users see as soft confirmations. If any of the three computes a different -accept/reject/order than the canonical fold (#1), the sequencer will have -promised users a future the scheduler will not produce. No mechanism enforces -the agreement — only review and the duality tests. **Change one, check all.** - ---- - -## Invariants this rests on - -- [I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) - — the three implementations agree (this document is its prose). -- [I2](../invariants.md#i2-drain-attribution-accumulated-directs-land-in-the-clock-advanced-frame) - — drained directs land in the new frame, so the lane's wire content matches - the fold's drain-before-ops order. -- [I3](../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) - — frame `safe_block`s are non-decreasing, the lane-side mirror of gate d's - monotonicity rule. -- [I19](../invariants.md#i19-application-progress-follows-the-shared-execution-contract) - — every successful application input advances one typed count/clock pair; - rejected inputs do not, and `AppError` is terminal. - -## Test-pinned properties - -The duality's load-bearing edge cases each have at least one test -([`scheduler/mod.rs`](../../sequencer-core/src/scheduler/mod.rs) tests, -[`protocol.rs`](../../sequencer-core/src/protocol.rs) tests): - -- **Empty batches are valid no-ops that consume the nonce** — no first frame to - measure staleness against (`empty_batches_are_valid_noops`, - `scheduler_accepts_empty_frames_batch_regardless_of_age`). -- **Stale batches do not consume the nonce** and the next batch reuses it - (`stale_batch_is_skipped_without_consuming_nonce`). -- **Drain uses an inclusive `inclusion_block ≤ safe_block` rule** - (`frame_drain_uses_consistent_inclusive_safe_block_rule`). -- **The backstop drains all overdue directs before the batch** - (`pre_batch_backstop_executes_overdue_directs_before_user_ops`, - `backstop_drains_all_overdue_directs`). -- **Staleness boundary is `≥ MAX_WAIT_BLOCKS`** — accepted at `MAX_WAIT − 1`, - skipped at `MAX_WAIT` (`scheduler_accepts_boundary_just_below_stale`, - `is_scheduler_stale_reports_true_at_and_past_threshold`). -- **Structural rejects don't consume the nonce** - (`non_monotonic_safe_blocks_invalidate_batch`, - `frame_safe_block_above_inclusion_block_invalidates_batch`, - `wrong_batch_nonce_is_rejected_without_consuming_nonce`). - ---- - -## Reference-scheduler count discipline - -Every canonical input crosses the same typed execution -boundary in the scheduler, live lane, catch-up, and recovery fold: - -1. Successful directs and validated user ops return their pre-execution - `ExecutedInputCount` and advance exactly once with checked arithmetic. - Rejection, bad signatures, envelopes, empty batches, and stale/structural - skips leave it unchanged. -2. `AppError` is fatal everywhere; there is no parallel scheduler-only - transition after a possibly partial hook failure. -3. The engine advances its embedded progress in each successful apply hook. - The shared boundary preflights overflow and asserts the exact expected - successor after success. Validation purity and native-engine mutation - remain self-trusted. -4. Tests pin direct/user advancement, every skip/reject family, pre-batch - overdue-drain ordering, overflow, dump round-trips, nonzero recovery bases, - and live/replay mapping agreement. - -Additional scheduler changes may still be batched separately, but they must -preserve the transition table above and the shared boundary rather than -reintroducing a scheduler-local count. +### The three implementations and why they agree + +The sequencer predicts this fold in two more places: the off-chain predicate +`scheduler_accepts`, which builds the gold frontier, and the inclusion lane's +live prediction. The recovery fold drives `Scheduler` itself. +[I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) +maps all three and their one deliberate difference: the predicate skips gate d. +[I2](../invariants.md#i2-drain-attribution-accumulated-directs-land-in-the-clock-advanced-frame) +and [I3](../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) +are the lane's mirrors of the drain order and of frame monotonicity, and +[I19](../invariants.md#i19-application-progress-follows-the-shared-execution-contract) +covers the shared execution boundary. Change one, check all. diff --git a/docs/recovery/README.md b/docs/recovery/README.md index 1a45fa52..f12e6a47 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -127,6 +127,14 @@ in [`TimingArgs`](../../sequencer/src/commands/config.rs): with `MAX_WAIT_BLOCKS 1200 and margin 300, observed danger starts at age 900 blocks. Neither that margin nor the fee policy bounds the time required to finish recovery. +A Tip ages from its first frame, so `run` also refuses a +`max_batch_open_seconds` that, rounded up to blocks at `seconds_per_block`, +exceeds the danger threshold minus `BATCH_SUBMISSION_MARGIN_BLOCKS` +([`RunConfig`](../../sequencer/src/commands/config.rs)); otherwise a low-volume +Tip reaches danger before it can be sealed, submitted and safe-accepted. The +check assumes L1 blocks are no faster than `seconds_per_block` and does not +bound inclusion time. + ### When safe-head progress stops A responsive RPC endpoint can keep returning an old view. The detector uses @@ -159,7 +167,7 @@ may cover a slot that was never used; it must not leave a sent slot uncovered ([I14](../invariants.md#i14-watermark--wallet-nonce-of-every-tx-ever-broadcast)). The flusher submits zero-value self-transfers at unresolved slots from the -account's Latest nonce through `max(Pending, W + 1) - 1`. It completes only when: +account's Latest wallet nonce through `max(Pending, W + 1) - 1`. It completes only when: ```text Pending <= Safe && Safe >= W + 1 @@ -216,7 +224,9 @@ the same suspected prediction alive without solving that cycle. `EnsureOpenTip` is a separate action. Its transaction requires `Safe`, no Tip, no divergence, and a rollback-safe checkpoint. It opens the Tip without invalidating history. Guarded writes matter even without concurrent writers: -wall-clock aging alone can change the decision after inspection. +wall-clock aging alone can change the decision after inspection, which retries. +An open Tip found by the guard refuses instead: aging cannot create one, so +only another writer or an inspection bug can. ### Atomic history change and replay @@ -348,8 +358,8 @@ just -f docs/recovery/justfile check-all | Concern | Owner and useful tests | |---|---| | Startup dispatch, error classification, final admission | [`recovery/mod.rs`](../../sequencer/src/recovery/mod.rs); procedure tests substitute only L1 sync/flush, keeping real SQLite inspections and repairs. | -| Detection, mutation guards, pivot and atomic cascade | [`storage/recovery.rs`](../../sequencer/src/storage/recovery.rs), [`recovery_tests.rs`](../../sequencer/src/storage/recovery_tests.rs); exact Tip guard, safe-view floor, unconditional post-flush policy, generation rollback, root nonce, and direct replay. | -| Observed danger versus estimates, accepted frontier and content identity | [`storage/l1_submission.rs` tests](../../sequencer/src/storage/l1_submission.rs), [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs); stale-view precedence, clock faults, reused nonces, divergence freeze. | +| Detection, mutation guards, pivot and atomic cascade | [`storage/recovery.rs`](../../sequencer/src/storage/recovery.rs), [`recovery/tests.rs`](../../sequencer/src/storage/recovery/tests.rs); exact Tip guard, safe-view floor, unconditional post-flush policy, generation rollback, root nonce, and direct replay. | +| Observed danger versus estimates, accepted frontier and content identity | [`recovery/tests.rs`](../../sequencer/src/storage/recovery/tests.rs) (`check_danger`): stale-view precedence, clock faults; [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) tests: accepted frontier, content identity, divergence freeze; [`l1_submission.rs`](../../sequencer/src/storage/l1_submission.rs) tests: reused nonces. | | Slot settlement and broadcast coverage | [`recovery/flusher.rs`](../../sequencer/src/recovery/flusher.rs), [`l1/watermark.rs`](../../sequencer/src/l1/watermark.rs), [`submitter/poster.rs`](../../sequencer/src/l1/submitter/poster.rs). | | Task-free preparation, launch, restore and catch-up | [`commands/run/`](../../sequencer/src/commands/run/), [`inclusion_lane/mod.rs`](../../sequencer/src/ingress/inclusion_lane/mod.rs). | diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index c714065e..2a33cb4d 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -243,8 +243,9 @@ make normal reconciliation execute already-accounted direct inputs again. ### Flush and stopping block -The lost database cannot supply its previous wallet-nonce watermark. Flushing -therefore depends on the provider's pool view. A transaction dropped there but +Recovery reads the wallet-nonce watermark from the fresh data directory, so the +previous database's watermark is not used, even when that database survives. +Flushing therefore depends on the provider's pool view. A transaction dropped there but alive elsewhere may escape; a later accepted foreign or mismatched landing freezes the rebuilt instance and requires another rebuild. The provider is trusted fail-stop, as specified in the [threat model](../threat-model/README.md). diff --git a/docs/recovery/justfile b/docs/recovery/justfile index 723602ba..e3d078c6 100644 --- a/docs/recovery/justfile +++ b/docs/recovery/justfile @@ -8,9 +8,10 @@ check-admission: check-preemptive: {{tlc}} -workers auto -deadlock preemptive.tla -# Check the optimistic recovery spec (~3min) +# Check the historical optimistic design (~3min). A dev analysis tool for +# revisiting that design, not a check of the current procedure. check-optimistic: {{tlc}} -workers auto -deadlock history/optimistic.tla -# Check all specs -check-all: check-admission check-preemptive check-optimistic +# Check the specs of the current recovery procedure +check-all: check-admission check-preemptive diff --git a/docs/review/2026-09-16-track3-validation.md b/docs/review/2026-09-16-track3-validation.md index b5491328..b1498882 100644 --- a/docs/review/2026-09-16-track3-validation.md +++ b/docs/review/2026-09-16-track3-validation.md @@ -12,10 +12,12 @@ measurements must not be attributed to its rebased trees. The latency baseline already contains the initial versioned-history foundation; it isolates the subsequent application-history refactor, not the cost of the complete stack. -Retained for the [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): -this is the wallet baseline against which native-engine and deployment results -can be assessed. Replace or delete it when those decisions no longer use these -measurements. It describes the named revisions, not ongoing validation of HEAD. +Retained for the [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md) +and the ADR's [performance posture](../plans/2026-08-authority-boundary-adr.md#performance-posture), +which cites its ACK latency as regression evidence: this is the wallet baseline +against which native-engine and deployment results can be assessed. Replace or +delete it when those decisions no longer use these measurements. It describes +the named revisions, not ongoing validation of HEAD. ## Environment and canonical agreement diff --git a/docs/review/2026-09-18-stack-review-validation.md b/docs/review/2026-09-18-stack-review-validation.md deleted file mode 100644 index b4845fd3..00000000 --- a/docs/review/2026-09-18-stack-review-validation.md +++ /dev/null @@ -1,76 +0,0 @@ -# Stack review closeout validation - -Evidence for reviewing and landing the closing change on -`codex/stack-review-fixes`, above reviewed tip -`62ec150f18a27697220158f1ca4d84ec4a6fee06`. The implementation and regression tests -are at `18e99e4fe2d03426954f72df5641196d12103e99`; the closing documentation commit -adds contract corrections, this record, and a storage rustdoc correction only. -Retire this record after stack landing when no ongoing review decision uses it. - -## Environment and results - -Run on macOS arm64 through the project's parent Nix/direnv shell. Cargo and -rustc were both 1.95.0, Anvil was 1.5.1, and the locally rebuilt canonical guest -used the repository-pinned Cartesi Machine 0.20.0. Local Anvil differs from CI's -1.4.3 pin; these are local results, not a new CI run. - -| Check | Result | -|---|---| -| `cargo check --locked --workspace --all-targets` | Passed | -| `cargo fmt --all -- --check` and `git diff --check` | Passed | -| `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings` | Passed | -| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 754 passed; one existing ignored doc test | -| `lua watchdog/tests/run.lua` | 62 passed | -| `just canonical build-machine-image`, then `cargo run --locked -p canonical-test` | Fresh image; 10 guest scheduler tests passed | -| `bash scripts/ci-c-application-smoke.sh` | External archive, generic host, and independent downstream consumer built and ran their CLI smoke checks | - -The process binaries were rebuilt after the final SDK change. Each of these -scenario filters passed for both the Rust wallet host and the C wallet host: - -- `cold_replica_snapshot_backlog_live_recovery_test` -- `restart_and_replay_test` -- `recovery_after_stale_batches_test` -- `setup_recovery_round_trip_test` - -These eight process runs include canonical watchdog comparison after recovery -and rebuild. Their prerequisites were the checksum-verified rollups-contracts -Anvil fixture, locally built test contracts, and watchdog Lua dependencies. -The first attempted process run lacked the fixture and stopped before startup; -setup supplied it before the successful runs. - -## Discriminating regressions and independent review - -- A matching local batch followed by a foreign accepted batch in the same block - returned HTTP 200 before the finalized-selection guard. The canonical scheduler - advances beyond that local snapshot. With the guard, all three finalized routes - return 503, including a conditional state request, without acquiring a lease. -- With checkpoint clock 900, checkpoint block 901, and stop 899, the unguarded - deterministic rebuild phase reached artifact creation. The new check refuses - with exit 20 before replay or publication. A later resync reaching the checkpoint - does not substitute for the fixed stop; equality and genesis still succeed. -- Without the SDK header timeout, the stalled-header regression exceeded its - outer two-second limit. With the configured header deadline restored, all - 13 SDK tests passed, including a body that outlives that deadline. - -Separate reviewers checked finalized selection and streamed I/O classification, -and the manual-recovery guard and exit classification. No blocking findings -remained. Persistent SQLite errors still reach terminal classification through -the new error wrapper; operational filesystem errors remain nonterminal. - -## Limits and landing work - -A repeated parallel host run reproduced the already registered -`dropped_runtime_scope_keeps_lock_until_detached_worker_stops` failure: final -reacquisition returned `Locked`. Its isolated rerun passed, and the complete -serial suite passed. This does not resolve the concurrency investigation; it -remains in the [review register](register.md#bounded-investigations-and-cleanup). - -The full 49-scenario rollups suite and Sepolia-image scenarios were not rerun. -Private-engine conformance/export and representative deployment capacity remain -in their existing integration plans. The two current recovery TLA+ models were -read but not changed or rerun; neither models these new boundary checks. - -No lower stack branch was rewritten. Reconcile the known C-host test ancestry -once at landing, retaining the evolved coverage at the reviewed tip and the -new independent post-recovery assertions. No push, PR creation, or merge was -performed during this implementation. diff --git a/docs/review/2026-09-19-review-followup-validation.md b/docs/review/2026-09-19-review-followup-validation.md deleted file mode 100644 index 8b1a0063..00000000 --- a/docs/review/2026-09-19-review-followup-validation.md +++ /dev/null @@ -1,82 +0,0 @@ -# Review follow-up validation - -Evidence for the ongoing stack review and landing, covering changes above -`f504c2e88e2b432718493e29e3c56f74fce3ad00` on `codex/stack-review-fixes`. -Local validation below includes the filesystem-identity fix at -`8b6da04374c7a4aef61fba8e7963e71d3cd55122`, CI diagnostics at -`93e6a49ad19c35fe78f4c46d639eb8fd556c58ba`, and the accompanying documentation -clarifications. Those clarifications change no executable behavior. -Retire this record after landing when no ongoing review decision uses it. - -## Changes and discriminating checks - -- Startup compares filesystem device/inode identities after inspecting every - retained reference. Regressions preserve relative/absolute, leading-dot, - symlink, and macOS firmlink aliases, including mixed literal/aliased references, - while removing genuine orphans. The firmlink test fails against the old - canonical-path comparison because it deletes the registered artifact, and - passes with identity matching. A missing reference stops the sweep before any - deletion. Dangling orphan links are removed without following their targets. - All eleven startup hygiene tests pass. The firmlink regression runs on macOS; - no Linux bind-mount scenario was run locally. -- A scheduler/fold regression establishes that `A < B` can coexist with a - pending direct at `A` after a same-block frame and a later empty batch. It - verifies recovery from an eligible earlier checkpoint preserves every direct, - application progress, and the scheduler nonce. The - [checkpoint contract](../recovery/cockroach.md#checkpoint-eligibility) requires - canonical exporters to inspect the pending queue and refuse such candidates. -- The accepted-prefix fixture now places the old future-nonce input both below - and exactly at `C`, before the nonce-0 batch in L1 order. Temporarily changing - the production query from `> C` to `>= C` fails specifically at the equality - case. Restoring `>` passes all four acceptance-projection tests. -- API documentation states the existing address-casing/normalization contract, - WS admission statuses, health-probe semantics, and internal deployment - boundary. Writer ownership and stale source references were corrected. The - submitter's own-sender decode error remains visible under self-trust. -- CI retains harness logs from failed rollups E2E jobs for seven days. This adds - evidence for the unclassified C-host recovery WebSocket reset in the - [register](register.md), without changing test or recovery behavior. - -Separate reviewers examined snapshot cleanup and checkpoint eligibility. Review -caught the dangling-orphan-link case before final validation; its regression is -included. No findings remain within the implemented scope. - -## Validation - -macOS arm64, parent Nix/direnv environment, Rust and Cargo 1.95.0: - -| Check | Result | -|---|---| -| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 760 passed; zero failed; one existing ignored harness doc test | -| `cargo check --locked --workspace --all-targets` | Passed | -| `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings` | Passed | -| `cargo fmt --all -- --check` and `git diff --check` | Passed | -| Relative Markdown links and code fences across repository Markdown files | 409 targets/anchors checked | -| CI workflow YAML parsing | Passed; artifact upload itself requires a failed GitHub job | - -The full host suite includes the new regressions. Guest execution, standalone -rollups E2Es, watchdog Lua tests, and TLA+ model runs were not repeated for this -follow-up. Both current recovery models were read; their modeled transitions -are unchanged. Earlier validation remains in the -[initial closeout record](2026-09-18-stack-review-validation.md). - -## Remaining integration and landing work - -The bundle carries no canonical queue evidence, and no canonical-to-native -exporter is implemented here. The eligibility change is a supported-checkpoint -precondition and executable scheduler regression, not loader enforcement or a -completed CM export drill. The application's exporter must enforce refusal and -demonstrate fallback and eligible pending-queue recovery; the -[integration plan](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) -tracks that work. Canonical acceptance semantics and artifact formats are unchanged. - -Archive concurrency limits remain deferred pending deployment workload needs. -The existing process-lock concurrency investigation remains open; this host run -used the serial suite. The C-host WebSocket reset is a separate unresolved -investigation; one successful run cannot classify its cause. - -The lower-stack ancestry is reconciled: PR #38 is at -`7f3229f2e42585e055f2fabb8e280c4d42dd5a81`, and GitHub reports PR #42 mergeable -at `35697691d7a5ba5a2c868f51b3a45c3dd5b6ee44` (checked 2026-09-19). -Consult CI for the pushed revision before landing; local host checks do not -replace guest, watchdog, or standalone rollups E2E execution. diff --git a/docs/review/2026-10-13-whole-repo-findings.md b/docs/review/2026-10-13-whole-repo-findings.md new file mode 100644 index 00000000..2581c3bb --- /dev/null +++ b/docs/review/2026-10-13-whole-repo-findings.md @@ -0,0 +1,1795 @@ +# Whole-repo review, 2026-10-13: findings appendix + +Generated from the read-only review fleet at `1a9615d`. The plan and triage live in [the review record](2026-10-13-whole-repo-review.md). Delete this file with the record. + +Status: **confirmed** or **partially** (the claim shown is the skeptic's correction) survived an adversarial skeptic; **disputed** means an independent trace refuted a skeptic-confirmed claim; **unverified** and **skeptic-added** are single-source. Line deltas are estimates (negative = removal). BC = behavior change. + + +## A. Application trait, core domain and wire types, C ABI (consumer surface: app integrators) + +Verdict: This area is the integrator surface. It has three parts: the Application trait with its shared execution boundary (about 350 production lines), the wire and domain types (UserOp, SignedUserOp, WireUserOp, Batch, Frame and the history identity types, about 410 lines), and the C ABI bridge (header, adapter and build.rs, about 835 lines, of which about 280 are comment lines in the header). The Rust core is proportionate. The trait is small, the outcome and error types are minimal and covered by tests, and the progress check costs two progress() reads per input while guarding I19 (silent application-history divergence). The problems are at the seam, not in the logic. The header is about 70% prose and repeats an application contract that is already stated in three other places. Two host concerns leak into every engine: rejection reasons only the boundary can produce, and an obligation to fsync a directory the host owns. The stateful drain protocol and the path callback (Track 6) are what a C++ integrator will trip on; they should be decided in one 'seam v1' change with Bart, together with N8, before libdex freezes its header. history.rs's era/generation identity (about 180 lines, plus history_api.rs and broadcast.rs) serves the subscribe-across-recovery egress model that the new direction replaces, so it should be frozen. I found no scheduler/sequencer divergence bug: both sides use validate_and_execute_user_op, build_input_domain and Signature::from_raw over the same canonical 65 signature bytes. Keep the engine-owned progress, the integer-constant enums that refuse unknown values, and fees crossing the ABI as log-space exponents. + + +### A1. Rejection reasons only the boundary can produce leak into the app's and the C ABI's vocabularies + +`medium · confirmed · -40 lines · S · c-abi` +Where: `sequencer-core/src/application/mod.rs:111-131`, `sequencer-core/src/application/mod.rs:169-174`, `sequencer-core/src/application/mod.rs:278-294`, `bindings/c-app-engine/include/application-engine.h:141-148`, `bindings/c-app-engine/include/application-engine.h:172-191` + +- Claim: As stated. Two additions strengthen it. First, AGENTS.md:216 tells integrators that every app must implement NonceExhausted and InvalidMaxFee, which contradicts mod.rs:165-167 and header:300-302. Second, the scheduler's test double RecordingApp (sequencer-core/src/scheduler/mod.rs:503-510) re-implements the max-fee guard and returns InvalidMaxFee, which the boundary at mod.rs:279 makes dead. With AppRejection that becomes a type error. The net line delta is nearer -25 than -40, because the AppRejection enum and its conversion add about 15 lines. +- Fix: Have validate_user_op return Reject(AppRejection), where AppRejection has only InvalidNonce and InsufficientFeeBalance. The boundary wraps it into InvalidReason (an App(..) variant plus InvalidMaxFee and NonceExhausted, or a flat enum with From), and the Display text stays the same. In the header, delete the two codes and ApplicationEngineInvalidMaxFee, marking values 1 and 3 as reserved, and pass the engine only (nonce, data) for validation. With that, the type makes the misuse impossible. +- Decision: C ABI change: agree it with the libdex owner and land it in the same seam-v1 change as the Track 6 items and N8. + +### A2. Every engine must fsync a directory the host owns + +`medium · confirmed · -6 lines · S · BC · application-trait` +Where: `sequencer-core/src/application/mod.rs:234-241`, `bindings/c-app-engine/include/application-engine.h:362-364`, `docs/protocol/application-contract.md:207-211`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:136-148` + +- Claim: As stated. Every production checkpoint write goes through create_dump_dir_with_info: setup/mod.rs:799,844,901, setup/fill.rs:62 and inclusion_lane/snapshot.rs:51. The host fsyncs dump_dir only in write_info (dump_info.rs:170), which runs before app.create_dump creates the `state` entry (dump_info.rs:143). It then fsyncs the dumps directory (dump_info.rs:144-147). The `state` dentry therefore depends on the engine today. The wallet does it at wallet.rs:383-385, under a wrong comment: 'dumps_dir/ contains prefix's dentry' at wallet.rs:382, where the parent is actually the per-checkpoint dump_dir. Once the host fsyncs dump_dir, the wallet can drop those three lines. +- Fix: Add File::open(dump_dir)?.sync_all()? right after app.create_dump in create_dump_dir_with_info. Shrink the engine contract to 'fsync everything you wrote at and beneath prefix', and move the WAL-ordering rationale into dump_info.rs. Update the trait doc, the header and application-contract.md to match. + +### A3. The Application contract is stated in four places, and the header is about 70% prose + +`medium · partially · -180 lines · M · docs-only` +Where: `bindings/c-app-engine/include/application-engine.h:9-68`, `bindings/c-app-engine/include/application-engine.h:107-123`, `bindings/c-app-engine/include/application-engine.h:186-187`, `bindings/c-app-engine/include/application-engine.h:356-395`, `sequencer-core/src/application/mod.rs:124-126` + +- Claim: The cross-cutting duplication is real. The nonce rule appears at AGENTS.md:216-217, header:53-57, contract:96-120 and mod.rs:176-180. The fee conversion appears at header:59-63 and c-application-binding.md:39-46. Dump obligations are restated in mod.rs:225-251, header:356-378, contract:207-234 and README:25-34. The design-rationale lines can go: header:107-109, 115-116, 121-123 and 186-187. But the header cannot shrink to a 10-line preamble. It is the self-contained ABI for bindgen and C++ consumers ('the header carries the compatibility obligation on its own', header:31-34), and c-application-binding.md:37 defers buffer lifetimes to it. The per-entry ownership, lifetime, reentrancy and status rules must stay in the header (header:255-266, 318-320, 339-344, 391-393). README:119-124 is a runnable recipe, not a test inventory, so keep it. batch.rs:7-39 duplicates the batch_policy comment in 0001_schema.sql:557-606, and it describes the CHECK as a linear 'batch_size_target < const_max_batch_bytes' when the actual constraint is log-space (0001_schema.sql:587). A realistic delta is about -100 to -130. +- Fix: Make application-contract.md the only owner. The header keeps a preamble of about 10 lines that links the contract sections, plus a terse @pre/@post/@returns per entry point. The trait docs shrink to one or two lines per method with a section link. The README drops the restatement at 13-34 and the test inventory at 105-129 (link the tests instead). batch.rs keeps two lines and a link to fee.rs. + +### A4. Freeze the era/generation history-identity types; they serve the egress model being replaced + +`low · partially · +0 lines · L · http-api` +Where: `sequencer-core/src/history.rs:45-223`, `sequencer-core/src/history_api.rs`, `sequencer-core/src/broadcast.rs:28-88`, `docs/protocol/application-contract.md:152-155` + +- Claim: Freezing is fine: no uuid-crate swap and no reworked wire codes for EraId (history.rs:45-139) or HistoryPolicyError (history.rs:212-223). But RecoveryGeneration and the generation cuts are I18 machinery for standard recoveries, not cockroach ones. That covers the exact +1 generation trigger, history_generation_cuts and cascade_and_reopen (docs/invariants.md I18). The new direction removes subscribing across a rebuild, which is an era change. Any future sync must still detect a standard-recovery suffix replacement while a client is subscribed (I18 'Breaks: a client silently resumes a replaced suffix'), and the Do-not-simplify list forbids keeping a numeric resume offset without its history identity. Catch-up also pages through the shared claim-based reader (catch_up.rs:53-72). So the ~-350-line deletion does not follow from the direction. At most EraId's wire role shrinks, and generation will probably survive in some form. +- Fix: Freeze these types and do not polish them: no uuid-crate swap, no reworked error codes. Keep ExecutedInputCount (history.rs:10-43), which is application progress, and move it next to ApplicationProgress when history.rs is retired. Retiring the module requires re-homing the catch-up's identity check. +- Decision: Confirm that the new sequencer<->client sync retires era/generation claims; the deletion (about -350 lines) waits for that design. + +### A5. Overridable progress shortcuts on the trait can disagree with progress() + +`nit · partially · -15 lines · S · application-trait` +Where: `sequencer-core/src/application/mod.rs:198-213`, `examples/app-core/src/application/wallet.rs:159-161` + +- Claim: Overridable default methods do exist (mod.rs:203-213) and the wallet shadows one inherently (wallet.rs:159-161). But no implementation overrides them. Rust application code is self-trusted (application-contract.md:6-9, I19 'Scope'), so an override that disagrees with progress() is an app bug inside the trust boundary, not an unverified path. An extension trait would not leave call sites unchanged either: every calling module would need to `use` the new trait. The cheap, worthwhile part is deleting the wallet's inherent shadow and the duplicated doc text. +- Fix: Move the two methods to an extension trait with a blanket impl (impl AppProgressExt for A), so they cannot be overridden and call sites stay unchanged. Drop the duplicated doc text and the wallet's inherent shadow. + +### A6. The adapter turns lifecycle statuses from execute, validate or drain into retryable I/O errors + +`low · confirmed · +4 lines · S · BC` +Where: `bindings/c-app-engine/src/lib.rs:70-85`, `bindings/c-app-engine/include/application-engine.h:47-49`, `docs/protocol/application-contract.md:86-90` + +- Claim: As stated. Note that the reference engine blurs the same line on its own side: its shared report() (c-wallet-engine lib.rs:52-70) maps Io kinds to NOT_FOUND or INVALID_DUMP for validate, execute and drain alike. The lane classifies any AppError::Io from ExecuteUserOp or ExecuteDirectInput as non-terminal (sequencer/src/ingress/inclusion_lane/error.rs:57-59,71-73), so a contract-violating status causes a restart loop instead of a terminal stop. +- Fix: Pass a lifecycle flag to check(), set only by from_dump and create_dump. Everywhere else, map NOT_FOUND and INVALID_DUMP to AppError::Internal with the status in the reason. + +### A7. The frame fee has four names across the integrator surface + +`nit · unverified-nit · +0 lines · S · c-abi` +Where: `sequencer-core/src/application/mod.rs:118-121`, `sequencer-core/src/application/mod.rs:173`, `sequencer-core/src/batch.rs:58`, `sequencer-core/src/l2_tx.rs:18`, `bindings/c-app-engine/include/application-engine.h:156` + +- Claim: The same quantity is called current_fee (trait and validate ABI), fee_price (Frame), fee (ValidUserOp, and the header's ValidUserOp), and base_fee (InvalidReason and ApplicationEngineInvalidMaxFee). 'nonce' likewise has three meanings: user op, batch and L1 wallet (see N14). +- Fix: Use one term (for example frame_fee) in Rust and C. The SSZ field name does not affect the wire format. The header field rename is source-breaking for engines, so do it in the seam-v1 change and in the same naming pass as N14. +- Decision: Choose the term; whether the /tx rejection Display text may change. + +### A8. The scheduler builds the same UserOp twice per op + +`nit · unverified-nit · -2 lines · S` +Where: `sequencer-core/src/scheduler/mod.rs:333-334`, `sequencer-core/src/scheduler/mod.rs:350-356`, `sequencer-core/src/batch.rs:73-79`, `sequencer-core/src/application/mod.rs:296-300` + +- Claim: recover_sender calls to_user_op() (which clones data), the caller calls to_user_op() again, and the boundary then copies data into ValidUserOp: three payload copies per op in the canonical machine. The cost is negligible next to ecrecover; the point is that the wire-to-signed conversion is spelled twice. +- Fix: Build the UserOp once and pass it to recover_sender. Fold this into the N7 change. + +### AS1. AGENTS.md tells every app to implement the two rejections only the boundary may produce + +`low · skeptic-added · +0 lines · S · docs-only` +Where: `AGENTS.md:216`, `sequencer-core/src/application/mod.rs:165-167`, `bindings/c-app-engine/include/application-engine.h:299-302`, `bindings/c-app-engine/src/lib.rs:193-197`, `sequencer/src/ingress/inclusion_lane/error.rs:71-73` + +- Claim: AGENTS.md:216 lists InvalidNonce, NonceExhausted, InvalidMaxFee and InsufficientFeeBalance as 'protocol-level rejection semantics every app must implement'. The trait doc and the header say the opposite: the boundary owns the max-fee and exhausted-nonce guards, and an engine never returns them. A C engine that follows AGENTS.md and reports NONCE_EXHAUSTED for an account at u32::MAX gets AppError::Internal from the adapter. The lane classifies that as terminal (error.rs:71-73; validate errors surface as ExecuteUserOp at inclusion_lane/mod.rs:401). +- Fix: Reword AGENTS.md:216: apps reject only InvalidNonce and InsufficientFeeBalance, and the shared boundary adds InvalidMaxFee and NonceExhausted. This is a one-line doc fix, independent of A1's ABI change, and can land now. + +Keep (load-bearing): + +- Shared execution boundary with a progress assert (sequencer-core/src/application/mod.rs:324-349): two progress() reads per input catch the silent application-history divergence I19 describes, and the overflow preflight runs before any mutation. This is proportionate. +- Engine-owned progress instead of a host mirror or AppWithProgress wrapper (an alternative the jury refuted): cockroach recovery must recover the count and clock from the canonical checkpoint (docs/protocol/application-contract.md:244-249), so they belong in the app's state. +- ApplicationProgress::try_new (mod.rs:62-73): the single coherence check for imported progress (dumps and C engines); it fails loud in the adapter (bindings/c-app-engine/src/lib.rs:250-254). +- ExecutedInputCount newtype with checked arithmetic (sequencer-core/src/history.rs:10-43): stops the app-history offset being confused with SQLite cursors or advanced with wrapping math. +- Separate ValidationOutcome and ExecutionOutcome enums: named outcomes read better than a nested Result> and keep rejection distinct from fatal AppError. +- AppError with exactly Internal and Io: the terminal-versus-retryable split that startup classification relies on (application-contract.md:86-90, 230-234). +- Fees crossing the ABI as log-space exponents, with the linear conversion inside the engine: the same engine code runs natively and canonically, so there is one conversion implementation per engine. Passing a linear amount from the host would add a second implementation that must stay bit-exact. +- Max-fee guard before app validation and the u32::MAX guard after it (mod.rs:278-294): this order gives correct diagnostics, tests pin it, and the sequencer and scheduler share it. +- C enums generated as integer constants that refuse unknown values (bindings/c-app-engine/build.rs:29-31; src/lib.rs:128-132,194-197): drift between engine and host becomes a loud error instead of undefined behavior. +- EngineApp is Send but not Sync or Clone (bindings/c-app-engine/src/lib.rs:87-94): matches the lane's single owner of the engine; an independent fork goes through checkpoint and restore. +- create_dump/from_dump isolation and durability contract (minus the parent fsync, see A2): checkpoint GC (I13) and snapshot leases depend on it. +- SSZ decode-robustness tests on adversarial bytes (sequencer-core/src/batch.rs:163-217): the scheduler decodes L1 input, so this pins totality. +- The scheduler silently dropping ops whose signature cannot be recovered (sequencer-core/src/scheduler/mod.rs:330-333): the fold must stay deterministic and free of side effects. + +## B. Scheduler and protocol spec (consumer surface: scheduler reimplementers) + +Verdict: Area B is the canonical scheduler: `Scheduler` (~420 production lines), the recovery fold `fold_replay` (~150), the off-chain mirror in protocol.rs (~455), the guest harness (~140) and a 302-line spec. The fold itself is small, correct and proportionate. I found no Byzantine path that breaks it. Gate d runs before staleness, so N9 is fixed; the fridge drains in order, the nonce check runs before any state changes, and stale or rejected batches never consume a nonce. The size problems are next to the core, not in it. Only about 60 of protocol.rs's ~455 production lines mirror the scheduler; the rest is sequencer-local danger and wall-clock policy, plus re-validation asserts needed only because the struct fields are public. About half of scheduler-semantics.md is sequencer architecture repeated from I1/I2/I3/I19. Meanwhile the spec leaves out what a C++ port needs most: the SSZ schema, how the EIP-712 domain is built, and the signature acceptance rules. Today those rules are whatever alloy 1.6.1 does (v byte in {0,1,27,28,35..=255}, high-s normalized and accepted). That is the biggest risk before the port. The most valuable code change is to have `scheduler_accepts` mirror gate d. That removes the one documented exception to I1, along with the false positive in I9, and turns a self-bug that today causes silent divergence into a visible stall. `fold_replay` is already shaped like the future standalone recovery tool (pure, takes iterators, trusts the checkpoint nonce). Keep it and its fail-loud asserts, and add the missing `inclusion_block <= C` assert. + + +### B1. Spec omits wire, domain and signature rules a port needs; the signature rules come from alloy, not from the spec + +`medium · partially · +70 lines · S · docs-only` +Where: `docs/protocol/scheduler-semantics.md:63-111`, `sequencer-core/src/scheduler/mod.rs:350-358`, `sequencer-core/src/batch.rs:46-68`, `sequencer-core/src/lib.rs:21-43`, `examples/canonical-app/src/scheduler/mod.rs:33-35` + +- Claim: The spec does leave out the SSZ layout, the EIP-712 domain (lib.rs:23-43), the UserOp type, the MAX_WAIT value (1200, lib.rs:21) and the signature rules. The acceptance set really is alloy 1.6.1's: from_raw requires exactly 65 bytes, normalize_v accepts 0|1|27|28|35.., and both recovery backends call normalized_s() (sig.rs:380, sig.rs:407). The honest-path risk is narrower than claimed, though. Storage re-encodes every signature with Signature::as_bytes (sequencer/src/storage/ingress.rs:723), and as_bytes writes v_byte(), which is 27 or 28. So sequencer-built batches always carry v in {27,28}, and the wider v set only matters for batch bytes this sequencer did not produce. as_bytes copies s as-is (sig.rs:174-179), so user-supplied high-s does reach batches. High-s is the live divergence hazard for a port that enforces low-s. This is a documentation gap that guards a future implementation, not a present bug. +- Fix: Add a normative 'Wire and signature rules' section (~40 lines): the SSZ containers and field order, the domain, the UserOp EIP-712 type, the accepted v set, the high-s rule, how a wrong-length signature is handled, and the constants. Add a pinned test-vector table to the scheduler tests (v = 0, 1, 27, 28, 35, 255; high-s; a 64-byte signature) so both an alloy bump and the C++ port are checked against it. Separately, decide whether to narrow the accepted set (for example v in {27,28}, low-s only) while the port forces a new machine anyway. +- Decision: Should the canonical signature acceptance set be narrowed (a consensus change needing a new machine image) or documented exactly as it is? +- Independent trace: confirmed. The code facts check out: from_raw accepts any s, recovery calls normalized_s() first, and storage re-serializes s exactly as received. + +### B2. scheduler_accepts should mirror gate d; dropping the 'documented omission' simplifies I1 and I9 + +`medium · disputed · -40 lines · S · BC` +Where: `sequencer-core/src/protocol.rs:353-388`, `sequencer-core/src/scheduler/mod.rs:295-317`, `sequencer-core/src/scheduler/mod.rs:1513-1539`, `docs/protocol/scheduler-semantics.md:205-213`, `docs/invariants.md:100-105` + +- Claim: The off-chain predicate skips the structural checks on a self-trust argument, but in the self-bug case it is the predicate that fails silently. Suppose an own batch has a frame with safe_block > inclusion_block or decreasing safe_blocks. The canonical scheduler rejects it without consuming the nonce. `scheduler_accepts` accepts it, and the content-identity check returns Match because these are our own bytes. The frontier advances while every later own batch gets WrongNonce on chain: silent, growing divergence. Conversely, a foreign batch that is structurally invalid triggers a false CanonicalDivergence and forces a manual cockroach recovery. `batch_reject_reason_for_block` never uses `&self`. +- Fix: Make it a free `pub fn batch_reject_reason(inclusion_block, frames) -> Option` in scheduler/mod.rs and call it from `scheduler_accepts` after decoding. Then delete the exception prose in four places (protocol.rs:353-362, spec 205-213, I1 parenthetical, I9 false-positive sentence), turn the special-case block in the duality test into ordinary agree cases, and simplify the l1_submission test. I1 then holds with no exceptions. +- Decision: This reverses a deliberate self-trust omission (needs maintainer sign-off). The added cost is only an O(frames) check. +- Independent trace: refuted. The asymmetry is real and pinned by tests. + +### B3. fold_replay claims it enforces 'everything <= C' fail-loud but never checks replay batches against stop_block + +`low · partially · +6 lines · S · BC` +Where: `sequencer-core/src/scheduler/fold.rs:79-85`, `sequencer-core/src/scheduler/fold.rs:107-133`, `sequencer-core/src/scheduler/fold.rs:140-150` + +- Claim: The comment at fold.rs:79-81 claims 'everything <= C' is enforced fail-loud, but nothing compares replay or seed blocks against stop_block. A replay batch above C is folded in silently. A direct above C also escapes the post-drain fridge assert (fold.rs:144-150) whenever a later batch above C has a frame that drains it. The only production caller bounds its inputs (setup/mod.rs:488 and :494 use safe_inputs_in_block_range(.., stop_block)), so nothing reaches this today. It is a comment overclaim plus a missing cheap assert. +- Fix: Add `assert!(input.inclusion_block <= stop_block, ...)` in the replay loop, and the same for seeds. The post-drain fridge assert then becomes a consequence of these checks; keep it as a cheap backstop. +- Independent trace: confirmed. The code matches the finding. fold.rs:79-81 says the input contract "everything `<= C`" is "enforced fail-loud". + +### B4. protocol.rs is mostly sequencer-local danger policy shipped inside the guest-shared crate + +`low · partially · -10 lines · M · public-rust-api` +Where: `sequencer-core/src/protocol.rs:4-24`, `sequencer-core/src/protocol.rs:30-97`, `sequencer-core/src/protocol.rs:185-324`, `sequencer-core/src/protocol.rs:326-454` + +- Claim: protocol.rs does mix about 60-100 lines that mirror the scheduler (age_exceeds, is_scheduler_stale, scheduler_accepts, SafeInputView, AcceptedBatch) with about 300 lines of sequencer-local timing policy. The struct is a deliberate 'single source of truth' (protocol.rs:7-18) that puts the mirrored max_wait_blocks next to the local fields. Splitting it means changing scheduler_accepts' signature and touching roughly 20 consumer files across sequencer/src and tests/e2e (tests/e2e/src/test_cases.rs:52 reads FRAME_CLOCK_INTERVAL_SAFE_BLOCKS from sequencer_core::protocol). The net delta is about -10 lines and no defect is fixed. +- Fix: Pure move, no behavior change. Move the danger, wall-clock and L1-view timing plus their errors into the sequencer crate (for example sequencer/src/recovery/timing.rs). Move `scheduler_accepts(max_wait, submitter, input, expected)` next to `Scheduler` (scheduler/predicate.rs) so all scheduler-mirroring code sits in one module. Do it in a separate patch from B2 and B6. + +### B5. advance_expected_batch_nonce is described as a gold-frontier scheduler mirror, but it is a submitter heuristic + +`low · confirmed · -25 lines · S · docs-only` +Where: `docs/protocol/scheduler-semantics.md:197`, `docs/protocol/scheduler-semantics.md:222-230`, `docs/invariants.md:110-114`, `sequencer-core/src/protocol.rs:399-435`, `sequencer/src/l1/submitter/worker.rs:99-108` + +- Claim: The spec table row #2 says that `scheduler_accepts` plus `advance_expected_batch_nonce` 'builds the gold frontier'. In fact the frontier uses an inline copy, and the function's only consumer is `decide_submit_start`. That caller folds latest (not safe) events and applies no staleness, so the function does not mirror the scheduler's nonce rule (a stale batch does not consume its nonce on chain). +- Fix: Move the function into submitter/worker.rs as a private helper and delete the spec rows 197 and 222-230 and the I1 bullet at 110-114. The scheduler spec then describes only the scheduler. + +### B6. Public ProtocolTiming fields force re-validation asserts in every accessor, plus five tests for them + +`medium · confirmed · -90 lines · S · public-rust-api` +Where: `sequencer-core/src/protocol.rs:103-106`, `sequencer-core/src/protocol.rs:137-142`, `sequencer-core/src/protocol.rs:191-198`, `sequencer-core/src/protocol.rs:205-212`, `sequencer-core/src/protocol.rs:222-225` + +- Claim: `try_new` validates every field relationship once, but the fields are public 'to keep test fixtures concise'. So five methods re-assert the same invariants at runtime, and five tests (`*_fails_loud_on_*_direct_construction`) cover those asserts. A struct that cannot be built invalid makes all of this unnecessary. +- Fix: Make the fields private, add a `max_wait_blocks()` getter, and switch the ~8 fixtures to `try_new(..).expect(..)`. Delete the five assert blocks, the five fails-loud tests and the two 'fields stay public' comments. + +### B7. Three spellings of one staleness predicate, plus alias re-exports + +`low · confirmed · -30 lines · S · public-rust-api` +Where: `sequencer-core/src/scheduler/mod.rs:17`, `sequencer-core/src/scheduler/mod.rs:409-419`, `sequencer-core/src/scheduler/mod.rs:73-88`, `sequencer-core/src/protocol.rs:391-397` + +- Claim: `has_elapsed_since(start, wait, current)` wraps `age_exceeds(current, start, wait)` with the arguments reversed, and needs a doc comment to warn about the reversal. `age_exceeds` names its second parameter `first_frame_safe_block` even though the backstop passes a direct's inclusion block (mod.rs:387-391). `scheduler::MAX_WAIT_BLOCKS` and `input_domain` are pure aliases of `crate::MAX_WAIT_BLOCKS` and `build_input_domain`. Cross-type `PartialEq` exists only as test sugar. +- Fix: Keep a single `age_exceeds(now, start, window)` with neutral parameter names and delete `has_elapsed_since` and both aliases. Tests should compare `.outcome`. This merges spellings of the same formula only; it does not unify the two staleness references on the Do-not-simplify list. + +### B8. QueuedDirectInput duplicates DirectInput; two near-identical drain loops each clone the payload + +`low · confirmed · -25 lines · S` +Where: `sequencer-core/src/scheduler/mod.rs:115-120`, `sequencer-core/src/scheduler/mod.rs:360-406`, `sequencer-core/src/scheduler/mod.rs:1308` + +- Claim: The fridge stores {sender, payload, inclusion_block} and then rebuilds an identical `DirectInput` on every drain. Each drain loop clones the payload before popping, which momentarily doubles a payload the fridge is already holding (see the register's fridge-memory entry). AppError is fatal everywhere, so the queue state after an error has no meaning, and popping first is safe. +- Fix: Queue `DirectInput` directly and add one `drain_front_while(pred)` helper that pops, then executes, with no clone. Update the 1308 assertion. This is not the refuted retain-over-expect change. + +### B9. Zero-frame batches consume a nonce at any age, and the sequencer never sends one + +`low · confirmed · -10 lines · S · BC · wire` +Where: `sequencer-core/src/scheduler/mod.rs:250-256`, `sequencer-core/src/protocol.rs:373-377`, `docs/protocol/scheduler-semantics.md:72`, `AGENTS.md:86` + +- Claim: Gate c accepts `frames == []` regardless of age and consumes the nonce, so every implementation, the C++ port included, must carry this special case, and the predicate needs `unwrap_or(0)` plus an `!is_empty()` guard. The lane always opens a batch with a frame (I3: batch closure creates the successor's first frame), so the rule serves no honest traffic. Its only effect is a nonce consumer that never expires. +- Fix: If the C++ port forces a new canonical machine anyway, consider making empty `frames` a structural reject in gate d. Staleness then always has a first frame, and gate c disappears. Otherwise leave it and only document it. +- Decision: Changing the empty-batch rule is a canonical acceptance change. Is it worth doing now, alongside the port's new machine? + +### B10. Spec does not say gate d is within-batch only, or that the reject reason is diagnostic + +`low · confirmed · +4 lines · S · docs-only` +Where: `docs/protocol/scheduler-semantics.md:78-85`, `sequencer-core/src/scheduler/mod.rs:305-314`, `sequencer-core/src/application/mod.rs:91-99` + +- Claim: 'non-decreasing across the batch' could be read as a check against previous batches. In fact no cross-batch check exists: a later batch may use a lower safe_block, and user ops then see a non-monotone `safe_block`, while application progress clamps it with max. A port that adds a cross-batch check, or that depends on the per-frame interleaved order of reasons, would diverge or over-specify. +- Fix: Two spec sentences: 'monotonicity is checked within a batch only; no cross-batch state' and 'the reject reason is diagnostic; consensus only distinguishes accept/skip/reject'. Apps must not assume user-op safe_block is monotone across batches. + +### B11. scheduler-semantics.md: about half is sequencer architecture repeated from the invariants + +`medium · partially · -120 lines · M · docs-only` +Where: `docs/protocol/scheduler-semantics.md:122-148`, `docs/protocol/scheduler-semantics.md:189-237`, `docs/protocol/scheduler-semantics.md:255-277`, `docs/protocol/scheduler-semantics.md:281-302` + +- Claim: Two sections are real redundancy and can go: 'Reference-scheduler count discipline' (281-302), which restates the table at 163-171 and I19 and carries process notes, and the test-name inventory (255-277). All 11 test names in that inventory exist today, so 'will go stale' is a forecast, not an observed problem; test vectors would replace the list. The other two sections are not copies of invariants. The frame-clock section (122-148) is the designated prose owner, linked from invariants.md:146, AGENTS.md:265, application-contract.md:129 and protocol.rs:132. 'The three implementations' (189-237) is I1's authoritative prose (invariants.md:96) and is linked from AGENTS.md:79. Those two must be moved into sequencer-side docs with their links updated, not deleted. Realistic net is about -50 lines for the spec, plus relocation. +- Fix: Restructure into a normative porter spec of ~130 lines: inputs and metadata mapping, state (nonce, fridge), wire (B1), gates, drain, outputs, constants, test vectors. Add a five-line 'sequencer mirrors' pointer to I1/I2/I3. Delete the test-name inventory; the test vectors from B1 replace it. + +### B12. N16: MAX_WAIT_BLOCKS is a crate constant hidden behind SchedulerConfig::new + +`low · partially · +10 lines · M · db-schema` +Where: `sequencer-core/src/lib.rs:19-21`, `sequencer-core/src/scheduler/mod.rs:17`, `sequencer-core/src/scheduler/mod.rs:29-39`, `examples/canonical-app/src/bin/canonical-app-devnet.rs:11`, `examples/canonical-app/src/bin/canonical-app-sepolia.rs:11` + +- Claim: MAX_WAIT_BLOCKS is one constant (lib.rs:21) that the guest (through SchedulerConfig::new), run (config.rs:76) and the recovery fold (setup/mod.rs:591) all read, so they agree by construction within a source tree. The proposal would make it an operator-supplied setup input stored in DeploymentIdentity. That is a db-schema change, and it creates a new way for the sequencer's window to drift from the guest's compiled window, the same drift class as the submitter address (see BS1). No observed need for per-network windows was shown. The only cheap, uncontroversial part: test fixtures hardcode 1200 at detector.rs:198, l1_submission.rs:511 and storage/recovery_tests.rs:2771 (the last one is missing from the evidence list) and should reference the constant. +- Fix: Treat max_wait_blocks like sequencer_address. Guest: put per-network constants next to DEVNET/SEPOLIA_SEQUENCER_ADDRESS in app-core, and have the bins build `SchedulerConfig { sequencer_address, max_wait_blocks }` explicitly. Sequencer: setup takes it as a required input, records it in DeploymentIdentity, and run and the recovery fold read it from there (`ProtocolTiming::try_new(identity.max_wait_blocks, ..)`). Delete `crate::MAX_WAIT_BLOCKS`, the alias and `SchedulerConfig::new`. Replace the test literals with a test constant. Optionally, later, the guest's 'state' inspect could also report its config so setup can cross-check it; only add that if a mismatch is observed in practice. +- Decision: Should setup take max_wait_blocks as a required CLI input (no default), and how is the guest value pinned (per-network bin constant vs a rootfs config file)? + +### B13. Guest reports and fold asserts leave out the context needed under error + +`low · partially · +10 lines · S` +Where: `examples/canonical-app/src/scheduler/mod.rs:44-57`, `sequencer-core/src/scheduler/fold.rs:90-93`, `sequencer-core/src/scheduler/fold.rs:117-125` + +- Claim: Confirmed facts: a fixed reason-less report string (canonical-app/src/scheduler/mod.rs:51-57), no report for stale skips, no block or sender in the fatal panic at :46, and no values in the fold assert messages at fold.rs:92 and :119. Correction: reports are not part of the outputs tree, so this is consensus-neutral, but any change to the guest binary produces a new canonical machine image (template hash). It cannot ship to an existing deployment on its own and should ride with the next canonical-machine change (the C++ port or the fridge bound). The fold-assert message changes are host-only and can ship anytime. +- Fix: Use `format!("batch rejected at block {b}: {reason:?}")`, and add a report for `BatchSkippedStale`. Reports are not part of the output Merkle tree, so this is safe. Include inclusion_block and sender in the :46 panic, and prev/current blocks in the fold asserts. + +### B14. Comments that narrate history or repeat the same rationale several times + +`nit · unverified-nit · -40 lines · S · docs-only` +Where: `sequencer-core/src/scheduler/mod.rs:1210-1218`, `sequencer-core/src/scheduler/mod.rs:1544-1548`, `sequencer-core/src/scheduler/mod.rs:1605-1606`, `sequencer-core/src/scheduler/mod.rs:193-195`, `sequencer-core/src/protocol.rs:20-24` + +- Claim: Representative examples. History narration: 'Nothing exercised both until now' (1215), 'Previously tested only alongside an ACCEPTED batch' (1547), 'only the head case was tested' (1605), '`pub` (not `pub(super)`) because ... is now a separate crate' (194-195). The I1 section header at 1210 sits above five unrelated tests (1220-1376). 'The address is identity, not timing' appears four times (protocol.rs:20-24, 99-101, 344-347, safe_accepted_batches.rs:110-112). fold.rs explains its fail-loud rationale three times (19-20, 79-85, 140-143). protocol.rs:33-36 explains why a Result is better than a panic. +- Fix: Delete the history narration, move the I1 header to sit just above the duality helpers (1378), and keep each rationale once at its owning site. + +### B15. Spec ties the scheduler's count to the egress snapshot/WS contract; freeze that part + +`nit · partially · -8 lines · S · docs-only` +Where: `docs/protocol/scheduler-semantics.md:177-185`, `openapi.yaml:375` + +- Claim: Only spec lines 183-185 (3 lines) tie the count to HTTP snapshots and WS claims. Lines 177-181 describe catch-up and recovery, which the recovery tool still needs. The API field is HistoryInfo.baseline.next_batch_nonce (sequencer-core/src/history_api.rs:29,36; openapi.yaml:375), which belongs to the egress/history area, not the scheduler. +- Fix: Don't polish it. When restructuring (B11), replace it with a one-line pointer to application-history.md. Leave the fold API (fold_replay, resume_at, enqueue_direct, drain_covered_at, finish) as it is: it is already the core the standalone tool needs. + +### BS1. Nothing checks that the guest's compiled sequencer_address equals the sequencer's batch_submitter_address, and the spec never says where it comes from + +`medium · skeptic-added · +5 lines · S · docs-only` +Where: `examples/app-core/src/application/wallet.rs:73`, `examples/app-core/src/application/wallet.rs:85`, `examples/canonical-app/src/bin/canonical-app-devnet.rs:4`, `sequencer/src/commands/setup/mod.rs:580`, `sequencer/src/commands/setup/mod.rs:591` + +- Claim: Classification is entirely sender == sequencer_address. The guest takes that address from a per-network constant compiled into app-core (wallet.rs:73,85, used by the canonical-app bins). The sequencer, its frontier predicate and the recovery fold use identity.batch_submitter_address instead (setup/mod.rs:580,591). No production code compares the two: git grep for sequencer_address in sequencer/src finds only a doc comment at setup/mod.rs:473. If they differ, the guest files every sequencer batch as a direct input, so no user op executes canonically. Meanwhile the frontier and the fold, which both use the identity address, agree with each other and report acceptance. The result is total silent divergence that only the watchdog's state compare can catch. The spec states the rule (26-29) but never says the address is deployment data compiled into the machine that must equal the submitter key. A porter or operator has nothing to check against. +- Fix: First (docs-only, S): add one spec sentence saying sequencer_address is deployment data compiled into the canonical machine and must equal the sequencer's batch_submitter_address, and put the same item in the deployment/setup checklist. Optional, only if the cost is acceptable: have the guest's 'state' (or a new 'config') inspect report its SchedulerConfig so that setup can refuse on a mismatch. That is a one-time check at setup and adds no runtime machinery. The guest change would ride with the next canonical-machine rebuild. +- Decision: Is a docs and checklist rule enough, or should setup cross-check against an inspect-reported guest config (a guest change that would ride with the next machine rebuild)? + +Keep (load-bearing): + +- fold_replay's always-on assert! contract checks (sequencer-core/src/scheduler/fold.rs:88-150): run once per recovery, so negligible cost; they turn a dropped direct, which would silently produce a wrong S', into a crash. Extend them (B3), don't remove them. +- fold_replay's pure, iterator-based API, the trusted checkpoint nonce, and the Scheduler fold hooks resume_at/enqueue_direct/drain_covered_at/finish/queued_direct_len (scheduler/mod.rs:122-191): this is the core the planned standalone recovery tool needs, and it drives the canonical source instead of a fourth copy. +- Gate order, with stale and rejected batches never consuming the nonce (scheduler/mod.rs:241-274): cascading invalidation and same-nonce rebuilds depend on it. +- Preflighting nonce exhaustion before any frame mutates state (scheduler/mod.rs:276-278, 289-293): no partially mutated state can exist before the panic. +- Classification by msg_sender, never by decoding (scheduler/mod.rs:214): the payload stays opaque, and a direct can never be misread as a batch. +- The censorship backstop running before classification on every input (scheduler/mod.rs:210-212): the guarantee that the sequencer can't freeze directs. I9's effect-equality argument relies on it being a FIFO prefix of frame 0's drain. +- saturating_sub in age_exceeds (protocol.rs:395-397): keeps the consensus path a total function. Don't turn it into a panic in the guest, even where gate d makes the case unreachable. +- The I1 duality test (scheduler/mod.rs:1401-1540): the only mechanical check of I1. Keep it; B2 just turns its exception block into ordinary agree cases. +- The deliberate inline nonce advance in populate_safe_accepted_batches (sequencer/src/storage/safe_accepted_batches.rs:121-130): it interleaves with the content-identity check and the divergence freeze. +- Graceful inspect errors in the guest (examples/canonical-app/src/scheduler/mod.rs:59-73): a public read-only query must never halt the canonical machine. +- sequencer_address as deployment data rather than a library constant (scheduler/mod.rs:21-25): N16 should give max_wait_blocks the same shape. +- ProtocolTiming::try_new's cross-field validation (protocol.rs:143-183): keep the validation; B6 removes only the re-validation that public fields make necessary. + +## C. Fee and numeric stack + +Verdict: This area has two parts of very different weight. The consensus core is about 90 lines in sequencer-core/src/fee.rs (fee_to_linear, fee_to_linear_fixed, fixed_mul) plus the 119-line build.rs table generator. Both are recomputed by the guest and by Bart's C++ engine, so their exactness is mandatory. The other ~1,200 lines are sequencer-local: the Uniswap TWAP worker, the oracle math, the log-additive batch_policy view and admin knobs. Nothing re-derives any of their outputs. The frame fee is published and the scheduler only consumes it (scheduler/mod.rs:330-341, wallet.rs:244,264). The worker lifecycle is proportionate: it validates the source at setup, builds the runtime source without RPC, retains the last price on transient failures and absorbs contention. The numeric rigor is not proportionate. The TickMath port is already gone (7cfb474, uniswap.rs:247 is f64), but its doctrine remains. An f64 quote goes through checked U256 ceil-division, which adds up to 4x overcharge at sub-gwei L1 gas. A U512 binary search then encodes it, guarded by a 'fail rather than undercharge' error that the view's saturation contradicts downstream. The consensus crate also exports three inverse functions whose only callers are tests, so a porter cannot tell which parts are contract. The biggest open issue is a correctness question: a frame fee above MAX_EXPONENT panics canonical execution instead of being a structural batch rejection, and no C++ port can reproduce that. The batch-fee hoist is not designed anywhere in the repo; AGENTS.md:218 only mentions it. So batch_policy should be frozen, not polished. Keep: the fee_to_linear spec text and fixed_mul truncation argument, frame-fee immutability, the no-RPC runtime source, exact "OLD" revert decoding, and the queries.rs clamp that keeps the sequencer from emitting an out-of-range fee. + + +### C1. Out-of-range frame fee_price panics canonical execution; the behavior is unspecified for ports + +`low · confirmed · +15 lines · S · BC · wire` +Where: `sequencer-core/src/fee.rs:68-71`, `sequencer-core/src/scheduler/mod.rs:100-105`, `sequencer-core/src/scheduler/mod.rs:229-262`, `sequencer-core/src/scheduler/mod.rs:330-341`, `examples/app-core/src/application/wallet.rs:244` + +- Claim: As stated. The trigger needs two things: a frame with fee_price > MAX_EXPONENT (~17.1k), and an included op with a valid nonce and max_fee >= fee_price. Ingress never bounds max_fee, so anyone can sign 65535 and leave such an op waiting. The sequencer's own inclusion path would panic natively first (wallet.rs:244 runs on the host too), so the realistic source is a bug in the batch encoder or persistence, or a compromised key. +- Fix: Decide the rule with Bart before any Track 5 work, since a lookup table has the same bound question. (a) Add one structural rejection (e.g. BatchRejectReason::FeeAboveMax, checked in batch_reject_reason_for_block over all frames, ~10 lines) and document it in scheduler-semantics.md and the C header. Or (b) declare the panic part of the contract (guest halts) and say so in application-engine.h. Ship it as its own behavior patch. +- Decision: Should a sequencer-authored out-of-range frame fee be a graceful structural batch rejection (like DecodeFailed) or an impossible state that halts the guest? Bart's engine must implement the same rule. +- Independent trace: confirmed. The code path reproduces exactly as claimed, but the framing needs three corrections. + +### C2. Oracle price pipeline: an integer ceil in linear units/gas overcharges up to 4x at sub-gwei gas; collapse it to one f64 log expression + +`medium · partially · -120 lines · M · BC` +Where: `sequencer/src/l1/fee_oracle/math.rs:4`, `sequencer/src/l1/fee_oracle/math.rs:9-57`, `sequencer/src/l1/fee_oracle/uniswap.rs:152-176`, `sequencer/src/l1/fee_oracle/uniswap.rs:237-252`, `sequencer/src/l1/fee_oracle/worker.rs:38-39` + +- Claim: The whole-unit ceil at math.rs:33-40 overcharges by at most about 2x, when the true value is just above 1 unit/gas (e.g. 1.25 becomes 2 at 0.5 gwei for USDC at $2500, which is 1.6x). The '4x at 0.1 gwei' example is not caused by the ceil. It comes from the exponent-0 floor: fee_from_linear_ceil returns 0 for anything <= 1 (fee.rs:136-138), and the column has CHECK log_gas_price >= 0 (schema:573). The proposed f64 formula with clamp(...,0,...) keeps that floor, so it would not fix the 4x (see CS1). The 'contradiction' with the view's saturation cannot happen in practice: FatalMath needs about 1e53 units/gas, and Overflow needs gas above about 1e20 wei. That strengthens the case for deleting it, but it is not a live inconsistency. The simplification itself holds (MathError, FatalMath and the U256 quote plumbing go), as long as the 'never undercharge' doc is dropped explicitly: an f64 ceil can land one exponent (0.78%) low near boundaries, which the 10x slack absorbs. +- Fix: Compute log_gas_price = clamp(ceil((ln(base+priority) + directed_tick*ln(1.0001)) / ln(129/128)), 0, MAX_EXPONENT) in f64, with base+priority = 0 mapping to 0. TokenPriceSource returns the mean tick. While there, destructure observe's two cumulatives with a checked pattern that returns Provider, instead of indexing at uniswap.rs:159-164. Delete compute_x_units_per_gas, encode_log_gas_price, MathError, FeeOracleError::FatalMath and the U256 quote plumbing. Keep the tick range check, and log the f64 linear value for operators. Tests switch to tolerance comparisons. If deferred, the stopgap is the 3-line comment fix at math.rs:4. + +### C3. The consensus crate exports sequencer-local inverses whose only callers are tests + +`medium · partially · -250 lines · S · public-rust-api` +Where: `sequencer-core/src/fee.rs:15-16`, `sequencer-core/src/fee.rs:27-31`, `sequencer-core/src/fee.rs:81-127`, `sequencer-core/src/fee.rs:174-219`, `sequencer-core/src/fee.rs:104-115` + +- Claim: The dead code is real. fee_from_linear (fee.rs:81-127) is called only by tests (fee.rs:292,364-386; math.rs:119). log_fee_ratio (fee.rs:174-219) is called only by set_alpha. set_alpha (admin.rs:15-34) is called only by tests (inclusion_lane/tests.rs:1284,1850; admin.rs:115). FEE_BASE_NUM/DENOM are unused, and build.rs:71 hardcodes 129<<57. The same lower-bound binary search is copied three times. The 'porter cannot tell which parts are contract' reason is overstated: fee.rs:10-11 and c-application-binding.md:41-43 already pin the contract to fee_to_linear. Deleting set_alpha also removes the only typed writer listed in the invariants register writer table (docs/invariants.md:90, 'admin'), so update that row and schema:567, and replace the two lane-test calls with raw SQL fixtures. Counted alone, without C2's deletions, the delta is about -200, not -250. +- Fix: Delete fee_from_linear, log_fee_ratio, FEE_BASE_NUM/DENOM and set_alpha, along with their tests. Keep the 296 = log(10) derivation as the schema comment it already is. After C2, delete fee_from_linear_ceil and FeeEncodeError too. Move suggested_signing_max_fee and its slack constant next to FeeQuote (api.rs:130-147). fee.rs is then exactly the consensus core that Track 5 would replace. + +### C4. FEE2: every 12s poll opens a FULL-sync writer and commits an unchanged price + +`nit · partially · -3 lines · S · BC` +Where: `sequencer/src/l1/fee_oracle/worker.rs:133-134`, `sequencer/src/l1/fee_oracle/worker.rs:146-162`, `sequencer/src/storage/fee_oracle.rs:30-41`, `sequencer/src/storage/open.rs:166-173`, `sequencer/src/l1/fee_oracle/worker.rs:386-419` + +- Claim: The mechanics are right: a new writer connection and an unconditional UPDATE on every poll (worker.rs:151-154, storage/fee_oracle.rs:30-41, open.rs:166-173). By the reviewer's own numbers the cost is negligible (under 0.1%), and the proposal pre-empts an open register decision. 'Fee-observation visibility' (register.md:175-181) keeps 'operator SQL inspection suffices' as an option, and that option needs exactly the per-poll 'last successful observation' semantics. A last-change stamp would lose liveness information. invariants.md:91 also records 'stamps on every successful refresh' as the writer contract. This is not a standalone finding; it belongs to the register entry. The one concrete test defect: worker.rs:419 asserts the stamp with >=, which passes even if no second write happened. +- Fix: Write only when the exponent changes. The stamp then means 'last change', or it is dropped once the register entry is decided. Update invariants.md:91 and worker.rs:133-134. If staleness visibility matters, put the retained price's age in the transient warn (C8) rather than in a per-poll write. +- Decision: Should log_gas_price_updated_at_ms become a last-change stamp or be dropped? The register entry 'Fee-observation visibility' owns this. + +### C5. PriceSourceError is classified twice and passes through a tuple and an extra enum on the setup path + +`low · confirmed · -35 lines · S` +Where: `sequencer/src/l1/fee_oracle/uniswap.rs:221-230`, `sequencer/src/l1/fee_oracle/worker.rs:230-243`, `sequencer/src/l1/fee_oracle/bootstrap.rs:15-46`, `sequencer/src/commands/setup/mod.rs:164-178` + +- Claim: As stated. The same transient/misconfig split is coded at uniswap.rs:224-230 and worker.rs:230-243. The setup path goes PriceSourceError, then (bool,String), then UniswapConnectError (bootstrap.rs:17-23,37-44), then BootstrapError (setup/mod.rs:170-178). The single-field struct variant is matched with '..' (setup/mod.rs:171). The net delta is closer to -25 once is_transient() is added. +- Fix: Add PriceSourceError::is_transient(&self) and use it in both places. connect_uniswap returns PriceSourceError, with create_provider failures mapped at the call site, and setup maps the error once. Delete UniswapConnectError, bootstrap_price_source_error and its test. This does not merge BootstrapError variants. + +### C6. Freeze the log-additive batch_policy design until the batch-fee hoist (~250 lines) + +`low · partially · -200 lines · M · BC · db-schema` +Where: `sequencer/src/storage/migrations/0001_schema.sql:535-625`, `sequencer/src/storage/queries.rs:139-177`, `sequencer/src/storage/mod.rs:297-303`, `sequencer/src/storage/mod.rs:375-384`, `sequencer/src/storage/admin.rs:37-135` + +- Claim: Freezing batch_policy is reasonable, but for a different reason than given. coordination-tracks.md:16 records a pending decision that 'may make log-space fees defunct', and AGENTS.md:218 defers the batch-fee hoist. Neither is the stated future direction (a shared cockroach-recovery tool plus a new client sync). This is 'frozen pending a fee decision', not 'superseded-by-direction', and its -200 lines and db-schema change are speculative while no design exists. The claim that the CHECK log_gas_price >= 0 floor is 'a log-space artifact the hoist removes' is wrong. log_gas_price is only summed with about +1356 of constants (schema:617-621), so a negative exponent stays representable. The floor comes from the unsigned u16 oracle output and the CHECK, and it can be fixed now (CS1). +- Fix: Don't polish this now. When the hoist lands: compute a linear batch fee in app code, turn the batch target into a plain byte knob, and drop the view, the clamps, set_alpha's coupling and batch_size_target_bytes. Revisit the /fee quote shape at the same time. +- Decision: The batch-fee hoist design itself, including whether the user signs a linear or a log-space batch fee. It decides C6 and whether Track 5 is ever built. + +### C7. Fee docs: a wrong view comment, a stale name, and the 10x-slack fact repeated in 6+ places + +`low · confirmed · -30 lines · S · docs-only` +Where: `sequencer/src/storage/migrations/0001_schema.sql:618`, `sequencer-core/src/batch.rs:19-21`, `sequencer-core/src/batch.rs:23-38`, `sequencer/src/storage/migrations/0001_schema.sql:537-552`, `sequencer/src/storage/migrations/0001_schema.sql:577-578` + +- Claim: As stated, with two small fixes. The wrong comment is at schema:619, not 618: the sum adds log_user_op_bytes, so the result is a per-op fee, and the wallet charges fee_to_linear(frame_fee) per op. fee.rs:226-227 is not a redundant restatement: it tells the 1.5x signing slack apart from log_slack and should stay. The other slack restatements (schema:545-546,578; batch.rs:34-35; math.rs:20-21; uniswap.rs:234; threat-model:24) are real duplication. +- Fix: Keep the fee derivation in one place, the schema header, and replace batch.rs:23-38 with one line pointing to it. Fix the two errors, drop the TickMath reference, and cut the restating doc lines. + +### C8. Oracle logs say little during outages and may print the RPC URL every 12s + +`nit · partially · +5 lines · S` +Where: `sequencer/src/l1/fee_oracle/worker.rs:164-184`, `sequencer/src/l1/fee_oracle/worker.rs:201-205`, `sequencer/src/l1/fee_oracle/worker.rs:206-216`, `sequencer/src/l1/fee_oracle/uniswap.rs:90-110`, `sequencer/src/l1/fee_oracle/uniswap.rs:333-340` + +- Claim: Two parts hold. A change is logged twice: info at worker.rs:164-173 and debug at worker.rs:202-204. The changed= field is constant in each branch (worker.rs:171,181). The step-label point applies only at setup: at runtime only observe produces PriceSourceError::Provider (uniswap.rs:158), and gas failures carry their own string (eip1559.rs:40,47). Rate limiting and tracking failure counts or age adds state, which a cosmetic problem does not justify: about 300 warns per hour during a pool-only outage is acceptable. URL exposure is plausible, since reqwest transport errors print the URL and estimate_fees stringifies them too. But that is the class already covered by the open register entry 'Startup logs the full RPC URL', whose Next step already says to check diagnostic paths (register.md:133-135). It belongs in that entry's triage, not in a new finding. +- Fix: Log a change once. Add retained-price age and failure count to the warn and rate-limit it. Add a step label to Provider. Route provider errors through whatever endpoint redaction the RPC-URL register entry settles on. + +### C9. Ceremony around the 'OLD' revert classification + +`nit · unverified-nit · -15 lines · S` +Where: `sequencer/src/l1/fee_oracle/uniswap.rs:191-219`, `sequencer/src/l1/fee_oracle/uniswap.rs:296-304` + +- Claim: is_old_observation_reason wraps reason == "OLD" in its own function with its own doc comment, and has a 5-assert test of string equality. The two doc comments above it repeat each other. +- Fix: Inline Some(r) if r.reason == "OLD", keep one doc comment, and drop the equality test. + +### CS1. Oracle floors log_gas_price at exponent 0 (1 smallest unit per gas), overcharging 6-decimal fee tokens by 1/x at low gas + +`medium · skeptic-added · +10 lines · S · BC · db-schema` +Where: `sequencer/src/storage/migrations/0001_schema.sql:573`, `sequencer/src/l1/fee_oracle/math.rs:33-40`, `sequencer-core/src/fee.rs:136-138`, `sequencer/src/storage/fee_oracle.rs:12,30`, `sequencer/src/commands/config.rs:120` + +- Claim: compute_x_units_per_gas rounds any nonzero quote up to at least 1 unit/gas, and fee_from_linear_ceil maps values <= 1 to exponent 0. log_gas_price is a u16 end to end, and the column has CHECK log_gas_price >= 0. With USDC (the wallet prototype's token, schema:543) and ETH at $2500, the true cost is 2.5 x gas_gwei units/gas. Below 0.4 gwei the oracle therefore reports 1 unit/gas, an overcharge of 1/x: 4x at 0.1 gwei, 10x at 0.04 gwei. The 10x safety margin is applied a second time, by accident. The floor is not inherent to log space: log_gas_price is only added to about +1356 of constant exponents in the view, so values down to about -1356 still give a valid non-negative frame fee. +- Fix: Let the oracle produce a signed exponent: i16 in RefreshResult, set_log_gas_price, log_gas_price() and FeeOracleMode::Fixed. Relax the CHECK to a lower bound that keeps the derived fee >= 0, e.g. log_gas_price + log_slack + log_one_plus_alpha + log_delta + log_user_op_bytes >= 0. Do it together with C2: the f64 log formula then clamps at that bound instead of 0. Ship it as its own behavior patch. This is a schema change, so ask first. +- Decision: Should the oracle price below 1 smallest unit per L1 gas (relax the schema CHECK and make the exponent signed), or is the fee token's smallest unit an accepted floor? Either way, record the trade-off in the schema header. + +Keep (load-bearing): + +- fee_to_linear's contract text and the fixed_mul truncation argument (sequencer-core/src/fee.rs:44-59, 248-251): this is the cross-implementation spec Bart's C++ engine ports (c-application-binding.md:39-46), and the doc is the spec until a Track 5 table exists. +- build.rs table generation plus the MAX_EXPONENT assert (sequencer-core/build.rs:42-80, fee.rs:68-71): this is consensus data and its bound fails loud. Track 5 only turns build.rs into a verifier. +- query_batch_policy's asymmetric clamp and panic (sequencer/src/storage/queries.rs:146-176): the clamp is why the sequencer never emits a frame fee above MAX_EXPONENT, which keeps C1 unreachable today. +- Frame fee sampled at frame open and immutable for the frame (AGENTS.md:218, storage/mod.rs:345-372): this is what lets the oracle's math be free. The scheduler only consumes the published exponent. +- Setup-time connect and validate plus no-RPC runtime construction (bootstrap.rs:28-67, uniswap.rs:83-150): this keeps run independent of the pool, and it is what fixed FEE1. +- Exact decoding of the Error(string) 'OLD' revert rather than substring matching (uniswap.rs:202-219): this is correct classification of an availability failure. Only the wrapper ceremony should go (C9). +- is_sqlite_contention absorption and the transient-retention loop (worker.rs:62-72, 195-227): these match the accepted oracle-outage policy in threat-model README:24 and avoid restart storms. +- FeeOracle's process_lock with spawn_blocking_with_lock (worker.rs:92-95, 146-162): it keeps data-directory exclusivity if the awaiting task is cancelled. +- SUGGESTED_MAX_FEE_SLACK as a ceil (53) rather than the nearest value (fee.rs:221-239, test 337-347): nearest rounding would understate the 1.5x margin that lets quotes survive a frame rotation. +- Pinned Alloy estimator constants test (sequencer/src/l1/eip1559.rs:65-76): it guards the external policy that both the poster and the oracle depend on. +- Schema CHECK log_base_gas - log_alpha - log_delta < log_max_batch_bytes (0001_schema.sql:590): it enforces the batch-size bound until the hoist replaces the design. + +## D. Ingress and inclusion lane (hot path) + +Verdict: The ingress/lane area takes a signed user op from POST /tx, checks only size and signature (api.rs:131-156), and hands it to a single blocking lane over a bounded channel. The lane executes bounded chunks, commits each accepted chunk with FULL sync before acking (mod.rs:183-206, 311-335), and on a separate time-gated cadence does L1 reconciliation and batch close with a snapshot. I traced the hot path end to end and found no correctness bug and no per-op clone worth removing. Ops move by value and execute by reference. The real latency sources are the per-chunk fsync, the inline WAL checkpoint, and the occasional batch-close or reconciliation stall, and those are already in the register. Size is mostly proportionate. The lane is about 500 lines of essential concurrency and durability code. storage/ingress.rs (1668 lines) is about 900 lines of tests plus about 770 production lines, but roughly 240 of those are #[cfg(test)] fixtures mixed into production code. Some of those fixtures only test themselves, and a few public helpers are dead. A lean file keeps about 530 lines. The biggest real problem is diagnostic: when the lane dies, the error that gets logged and written to the black-box record drops its cause, so an AppError reason or a rusqlite detail never reaches the operator. After that come one redundant piece of in-memory state (the drain cursor) and a set of stale or untrue comments. The good parts must stay: ack only after a durable commit, bounded fast turns so rejected floods cannot starve reconciliation, WriteHead counter checks before the transaction, the re-derivation and receipt assert at frame rotation, hash-at-seal, file-first snapshots, fixed public 500 messages, and the documented no-nonce-precheck admission policy. N10 is already implemented: frames rotate on the 5-block clock whether or not there are direct inputs. N24 (the listener split) is cheap and should go ahead as a small PR. The egress snapshot-lease machinery in http.rs falls under the future direction and should be frozen, not polished. + + +### D1. Fatal lane errors lose their cause in the exit log and the black-box record + +`medium · partially · +10 lines · S · BC` +Where: `sequencer/src/ingress/inclusion_lane/error.rs:17-44`, `sequencer/src/commands/error.rs:515-516`, `sequencer/src/commands/error.rs:572-573`, `sequencer/src/harness.rs:129`, `sequencer/src/commands/mod.rs:65` + +- Claim: The core claim holds. ExecuteUserOp, ExecuteDirectInput, CatchUp, Snapshot and Gc display fixed strings (inclusion_lane/error.rs:17-21, 29-44). WorkerStop::Source is `#[error("{0}")]` (commands/error.rs:572-573), and every log sink formats Display only, so the AppError reason, the rusqlite detail, and even CatchUpError's own detail (e.g. the SnapshotExecutionCountMismatch numbers) never reach the operator. The sink list is wrong, though. A terminal lane error (e.g. AppError::Internal) never reaches harness.rs:129 or the black-box write at commands/mod.rs:65. Runtime::finish aborts first (commands/run/workers.rs:372-375), and abort_terminal logs `error = %error` and calls abort() (runtime/shutdown.rs:16-18). README.md:158 documents this: terminal runtime faults abort without settlement. Non-terminal lane errors reach harness.rs:129 (Unclassified) or workers.rs:460 during the drain. A sink-side chain renderer would therefore have to cover abort_terminal, harness.rs:129 and workers.rs:460, not commands/mod.rs:65. Inlining `: {source}` in the five #[error] strings, matching CatchUpError (error.rs:105, 110) and LoadFromDump (error.rs:41), covers every sink with no other change and is the smaller fix. +- Fix: Pick one fix. Either inline `{source}` into the five InclusionLaneError #[error] strings, matching CatchUpError, or render the full source chain once at the two top-level sinks (harness.rs:129 and commands/mod.rs:65). The second fixes every worker at once. Either way, the operator sees e.g. 'inclusion lane: user op execution failed: '. +- Decision: Choose between per-variant inlining and a chain renderer at the sink. The renderer covers the other workers too, but touches files outside this area. + +### D2. storage/ingress.rs mixes about 240 lines of test-only fixtures, some self-testing, into production code; two helpers are dead + +`medium · confirmed · -110 lines · M` +Where: `sequencer/src/storage/ingress.rs:93-153`, `sequencer/src/storage/ingress.rs:216-286`, `sequencer/src/storage/ingress.rs:323-405`, `sequencer/src/storage/ingress.rs:440-442`, `sequencer/src/storage/ingress.rs:635-702` + +- Claim: Confirmed as stated. About 237 #[cfg(test)] lines sit in the production half of storage/ingress.rs: 93-114, 115-145, 147-153, 216-286, 323-348, 365-379, 381-405, 635-650 and 679-702. fill_safe_inputs (216-286) is used only by the half-open-interval test at l1_inputs.rs:466-508, so that test largely tests its own fixture. It could use the existing production read safe_inputs_in_block_range (l1_inputs.rs:120) instead of a new SELECT. open_recovery_tip's guard (147-153) is exercised only by ingress.rs:1185-1188. WriteHead::open_frame_has_user_ops (storage/mod.rs:342-344) has no callers anywhere in the repo. Storage::batch_policy (ingress.rs:440-442) has only test callers (admin.rs:56/80/107, fee_oracle.rs:70, ingress.rs:1195+). append_user_ops_chunk has the two test users the reviewer names. The integration_tests module compiles in-crate (lib.rs:33-35), so cfg(test) items are visible to it. +- Fix: Delete fill_safe_inputs and replace the test read-back with a 5-line SELECT. Delete open_recovery_tip and its assertion, since the guard belongs at the external boundary that already exists. Delete open_frame_has_user_ops. Make batch_policy cfg(test) or move it into test_helpers. Port the two physical-only fixture users to the production append. Move the remaining fixtures (initialize_open_state, ensure_open_tip, close_frame_only/close_frame_derived_in, close_frame_and_batch) into storage/test_helpers.rs. The production file drops to about 530 lines and contains only lane writes, the in-transaction helpers, and the two public reads. Do this as a pure refactor patch. + +### D4. Redundant checks and parameters around snapshot restore and batch close + +`low · confirmed · -25 lines · S` +Where: `sequencer/src/ingress/inclusion_lane/mod.rs:102-109`, `sequencer/src/ingress/inclusion_lane/catch_up.rs:16-31`, `sequencer/src/ingress/inclusion_lane/catch_up.rs:47-52`, `sequencer/src/storage/ingress.rs:409-420`, `sequencer/src/ingress/inclusion_lane/snapshot.rs:48-57` + +- Claim: Confirmed with one correction. The count check at mod.rs:102-109 repeats catch_up.rs:47-52 on the same values: checkpoint.executed_input_count is passed straight through run_forever, run_catch_up and catch_up_application as `start`, and nothing touches the app in between. CatchUpSnapshot (catch_up.rs:16-31) only re-wraps latest_snapshot's prefix and count. close_frame_and_batch_with_snapshot's batch_index parameter only feeds an assert, which is tautological in production (snapshot.rs:48, 56; ingress.rs:417-420). The correction is that it has one production caller but six test callers (snapshot_dumps.rs:504, ingress.rs:838, integration_tests/snapshot_endpoints.rs:125/156/197, recovery_compatibility.rs:124, egress/historical/tests.rs:324), which must change with it. Expect about -15 to -20 lines rather than -25. +- Fix: Keep the check in catch_up, which is the place tests exercise, and drop the copy in start(). Return the storage snapshot row from catch_up_snapshot, or inline it. Remove the batch_index parameter and use head.batch_index inside storage. + +### D5. Stale or untrue comments in the hot-path files + +`low · confirmed · -10 lines · S · docs-only` +Where: `sequencer/src/storage/mutations.rs:22-24`, `sequencer/src/storage/ingress.rs:603-608`, `sequencer/src/storage/queries.rs:178-183`, `sequencer/src/clock.rs:6-8`, `sequencer/src/ingress/inclusion_lane/types.rs:33-37` + +- Claim: Items 1, 2, 3 and 5 are confirmed as stated. mutations.rs:22-24 is stale: production passes Some(0) via open_fresh_tip_in_tx (ingress.rs:491) and open_recovery_tip_in_tx (ingress.rs:526). The close_frame_in doc (ingress.rs:603-608) is wrong because close_frame_derived_in does not call close_frame_in. decode_l2_tx_row is #[cfg(test)] (queries.rs:178-184). types.rs:36 is wrong because queue-full is ApiError::overloaded at api.rs:266 and Unavailable maps only to 503 (http.rs:96, 113-114). For item 4 the conclusion is right but the evidence is weak: the cited sites (mod.rs:363, api.rs:261, snapshot.rs:70) need a SystemTime or nanos, not 'now in Unix-ms'. The real contradiction is that production storage has its own duplicate Unix-ms helper, storage/convert.rs:66 now_unix_ms(), used at ingress.rs:452 and lifecycle.rs:126, which bypasses clock::unix_now_ms. No test clock exists anywhere. +- Fix: Correct or delete each one. For clock.rs, say what it is (a saturating Unix-ms helper) and drop the test-clock promise. + +### D6. Verbose comments that restate the code or repeat neighbouring docs + +`nit · unverified-nit · -40 lines · S · docs-only` +Where: `sequencer/src/storage/ingress.rs:116-136`, `sequencer/src/storage/ingress.rs:4-11`, `sequencer/src/ingress/inclusion_lane/mod.rs:69-81`, `sequencer/src/ingress/inclusion_lane/mod.rs:130-135`, `sequencer/src/ingress/inclusion_lane/mod.rs:155-159` + +- Claim: Representative examples: a 21-line doc on an 8-line test-only fn, mostly describing production behavior that lives elsewhere (ingress.rs:116-136); a module doc that says 'live here too' twice (ingress.rs:4-11); InclusionLane::start's 13-line doc mixing three topics (mod.rs:69-81); an inline comment repeating that doc (mod.rs:130-135); a batch-close comment restating snapshot.rs:39-40 and I7 (mod.rs:155-159); a doc explaining that &Transaction coerces to &Connection (mutations.rs:146-151). The module doc also has a single line over 150 columns (mod.rs:14). +- Fix: Cut each to one or two lines that state the non-obvious reason, or link the invariant (I7, I17). Rewrap mod.rs:14. + +### D8. Duplicated shutdown gate and a redundant constructor in api.rs + +`nit · unverified-nit · -15 lines · S` +Where: `sequencer/src/ingress/api.rs:51-80`, `sequencer/src/ingress/api.rs:92-108` + +- Claim: SubmitState and ReadState each carry an identical reject_if_shutting_down. SubmitState also has a 4-argument new() even though all its fields are pub. +- Fix: Make one free fn reject_if_shutting_down(&RuntimeScope). Pick either pub fields or a constructor. Fold this into the N24 split PR, which touches this wiring anyway. + +### DS1. A graceful shutdown racing a frontier check or rotation surfaces as ChannelClosed and exits 1 instead of 0 + +`low · skeptic-added · +4 lines · S · BC` +Where: `sequencer/src/ingress/inclusion_lane/mod.rs:143-150`, `sequencer/src/ingress/inclusion_lane/mod.rs:439-443`, `sequencer/src/http.rs:297-352`, `sequencer/src/commands/run/workers.rs:454-470`, `sequencer/src/commands/error.rs:179-185` + +- Claim: The lane checks the shutdown flag only at the top of its loop (mod.rs:144), then runs maybe_advance_safe_frontier, which can rotate the frame and run GC (mod.rs:254-265), before try_recv. The only lane senders are owned by the HTTP router (SubmitState and HealthState, http.rs:305-315), and they drop as soon as graceful HTTP shutdown completes. If shutdown is requested after the flag check and the server finishes first, try_recv sees Disconnected with attempted == 0 and the lane returns InclusionLaneError::ChannelClosed (mod.rs:440-442) instead of Ok(()). The drain treats that as an error (workers.rs:456-463). It is non-terminal, so it is classified Unclassified (error.rs:179-185) and a clean SIGTERM exits 1 with a 'component errored during runtime drain' warning. +- Fix: In the Disconnected arm, or where run_fast_turn's error is handled, treat ChannelClosed as a graceful stop when self.shutdown.is_shutdown_requested(): reject anything pending and return Ok(()). That is about 3-5 lines and adds no new machinery. + +Refuted: + +- D3. The lane's in-memory drain cursor duplicates a value storage already re-derives and asserts on every frame rotation. The proposal weakens a fail-loud cross-check that protects against silent scheduler divergence (I6 drain completeness, 'impossible states fail loud'). +- D7. Egress snapshot-lease release supervisor lives in the shared http.rs; freeze it and move it with egress at the split. The 'superseded-by-direction' premise fails: the same lease machinery serves the watchdog and operator recovery endpoints, which the direction keeps and probably relies on. + +Keep (load-bearing): + +- Ack only after the FULL-synced chunk commit (mod.rs:311-335, ingress.rs:297-321). The commit is the durability authority behind a 200 (I20). +- Bounded fast turn plus time-gated frontier check, with TurnOutcome's three states (mod.rs:7-18, 337-356, 423-453). This stops a continuously nonempty, all-rejected queue from starving reconciliation. +- WriteHead counters are checked for overflow before the transaction, and the cache is updated only after commit (ingress.rs:305-320; I17). +- Range re-derivation and execution-receipt equality assert at frame rotation (ingress.rs:616-623, mutations.rs:169-173, 210-262). This enforces I6/I19/I20 and runs only once per safe-head tick, not per op. +- Hash-at-seal inside the seal transaction (ingress.rs:582-590). The content-identity check needs the hash atomically with the seal. +- File-first dump, then atomic seal plus snapshot registration (snapshot.rs:39-60, ingress.rs:409-438; I7). +- Unique dump-dir naming with nonce, nanos and a counter (snapshot.rs:62-75). Nonces are reused across recovery cascades. +- is_terminal_invariant classification and its tests (error.rs:52-138, 140-237). It drives the exit codes the supervisor relies on. +- storage_task classifies errors inside the blocking task and aborts on persistent faults (http.rs:213-267). Cancelling the HTTP request must not discard a persistent fault. +- No nonce/max_fee precheck at ingress, with a named revisit trigger (api.rs:18-23). A precheck would add a SQLite read to every honest submit. +- Fixed public error messages: no AppError, serde or rusqlite detail in 4xx/5xx bodies (mod.rs:392-401, api.rs:230-248). +- next_user_nonce's valid_batches filter (ingress.rs:43-47, 73-91). Cascades reuse nonces from invalidated rows. +- The asymmetric clamp/fail-loud in query_batch_policy (queries.rs:146-171). Operator-writable bounds are clamped; contract-impossible ones panic. +- catch_up_application_paged's page-size parameter (catch_up.rs:41-46). It is the test seam for page-boundary coverage. +- SnapshotReleaseDrain (http.rs:150-211). It is correct and load-bearing for lease release on shutdown, but superseded by direction, so freeze it rather than delete it. + +## E. Egress API and client SDK (consumer surface: clients/indexers) + +Verdict: Area E is the internal consumer surface. It has five parts: a WebSocket feed that replays and then follows the tip, keyed by a mandatory (era, generation, next_input) claim; /history metadata with standard-recovery compatibility; a pager for the era's raw L1 prefix (/historical-l1-inputs); health probes; and a Rust SDK. The core is sound and proportionate. One DB-backed cursor re-validates identity on every page inside a single deferred read transaction, refusals are typed and returned as 409 before the upgrade, and the base offset K and the generation cuts are implemented and fail loud on gaps. I found no correctness bug in the replay/subscribe handoff or in the history-identity checks, and N11/N12 are done. The future direction does not justify the current size: about 1,600 lines (code, tests and docs) exist only so a client can rebuild a projection across a cockroach rebuild (/historical-l1-inputs, projection-replay.md, the historical_bootstrap integration test, and half of /history), and the shared recovery tool replaces exactly that. About 1,400 more lines (WS wire format, SDK snapshot-then-subscribe bootstrap) should be frozen until the new sync is designed. The real problems besides supersession: the contract is written out in three places (README, openapi.yaml, protocol docs); the feed re-implements http::storage_task's terminal-fault policy with two near-identical error enums; WS error logs drop their SQLite causes; and the SDK has dead surface. Keep the claim identity (I18), per-page revalidation, cuts with the complete-lineage assert, base_executed_input_count, and the canonical reader. That reader is shared with native restart and should move out of storage/egress before the freeze. + + +### E1. Freeze /historical-l1-inputs and the projection-replay path (~1,600 lines) + +`medium · partially · -1600 lines · M · BC · http-api` +Where: `sequencer/src/egress/api/history.rs:37-46`, `sequencer/src/egress/api/history.rs:101-176`, `sequencer/src/storage/egress/historical.rs:115-260`, `sequencer-core/src/history_api.rs:11-13`, `sequencer-core/src/history_api.rs:56-81` + +- Claim: The direction does supersede the raw-prefix pager. Its only consumer is projection replay across a rebuild (projection-replay.md:12-28; the genesis prefix is empty). Freezing it is right. The removal plan is wrong in two places. (a) historical_bootstrap.rs is also the parent module of recovery_compatibility.rs (historical_bootstrap.rs:7, 403 lines, which uses `super::*` at recovery_compatibility.rs:15). That file tests in-era standard-recovery checkpoint compatibility, which E2 says survives. Deleting historical_bootstrap.rs would drop it together with the helpers it reuses (domain/config/execute, historical_bootstrap.rs:53-204). (b) projection-replay.md:56-93 (checkpoint eligibility: seed directs after A, replay (B,C], terminal drain) and the worked boundary at 95-118 specify exactly the client-state reconstruction the shared tool must perform. The main historical_bootstrap test is a ready-made oracle for it. They should move into the tool's design and tests, not be deleted. Only the HTTP route, raw_page_in/first_input_after_block_in, the HistoricalL1Input* wire types, the SDK method and the README/openapi prose are pure deletions. +- Fix: Freeze now: no new features, no polish, and do not implement the Track 3 'Cockroach reader recovery' follow-up (track3 plan:79,86-92). When the shared tool lands, delete the route, raw_page_in/first_input_after_block_in, HistoricalL1Input*, the SDK method, projection-replay.md and historical_bootstrap.rs in one removal patch. Keep historical_end_in only if /history still reports R. +- Decision: Confirm that the shared tool will produce the client's projection from the client's own checkpoint. If any client must still rebuild a projection from raw L1 after a rebuild, this route survives. + +### E2. /history: baseline and accepted_checkpoint serve cross-era projection prep; history and compatibility survive + +`low · partially · -150 lines · S · BC · http-api` +Where: `sequencer-core/src/history_api.rs:15-54`, `sequencer/src/storage/egress/historical.rs:41-113`, `sequencer/src/storage/egress/historical.rs:28-38`, `sequencer/src/egress/api/history.rs:84-86`, `README.md:318-390` + +- Claim: baseline.{l1_stop_block,l1_end_input_index,next_batch_nonce} and accepted_checkpoint serve only projection replay and manual-recovery checkpoint prep. They appear only in the README, openapi, projection-replay.md, the SDK and tests (git grep), so freezing them is right. The duplicate check is real: api/history.rs:84-86 makes historical.rs:54-58 unreachable through the route, though historical/tests.rs:403 still exercises it. Two corrections. (a) BadRequest-from-storage cannot simply be dropped. The 'from_generation exceeds the current recovery generation' check (historical.rs:59-63) needs the generation read inside the transaction, so after E1 it still needs a storage-side refusal. Only the pager's BadRequests (130-150) and 54-58 go away. (b) The claim that compatibility 'survives the direction' is not established. It exists so a client can reuse a replica checkpoint and resubscribe over WS after standard recovery. That is the claim/feed model E3 freezes, so it should be frozen with E3, not declared a keeper. +- Fix: Freeze. When E1 is removed, reshape /history to {deployment, history, compatibility} and drop the unreachable storage-side era check together with BadRequest-from-storage. No change before then. +- Decision: Is accepted_checkpoint still needed by the watchdog or operators once client recovery goes through the shared tool? The snapshot reviewer owns finalized_dump_in. + +### E3. Freeze the WS wire format and SDK snapshot-to-subscribe bootstrap; record their defects as inputs to the new sync + +`medium · partially · +0 lines · S · wire` +Where: `sequencer/src/egress/api/subscribe.rs:1-140`, `sequencer-core/src/broadcast.rs:12-46`, `sequencer-core/src/api.rs:189`, `sdk/rust-client/src/lib.rs:27-34`, `sdk/rust-client/src/lib.rs:177-228` + +- Claim: The snapshot-then-subscribe replica bootstrap (lib.rs:177-228) is the shared-state assumption the direction rejects, so freezing the WS wire format and the SDK bootstrap is right. All five listed defects are real: String fields at broadcast.rs:15-45 vs typed fields at history_api.rs:62-71; the casing split documented at openapi.yaml:16-20; the alias at api.rs:189; the raw stream at lib.rs:27; no timeout at lib.rs:222-228. Defect (5) should not be frozen, though. subscribe() can hang forever on a stalled server, while every sibling method is bounded (lib.rs:104, 154, 185). The fix is about 3 lines, the same pattern as latest_snapshot at lib.rs:185, and the harness already works around the hang at tests/harness/src/ws.rs:24. If the WS feed stays as the interim sync (the open needs_decision), fix (5) now and put (1)-(4) in the design brief. +- Fix: Do not build a typed SDK stream, WSS, in-band invalidation notices, or a casing fix now. Add these five items to the new-sync design brief so they are designed once. The server-side cursor (canonical_history_page plus claim) probably survives the redesign; see E4. +- Decision: Does the existing WS feed stay as the interim sync for Bart's client until the new design lands, or is it frozen immediately? + +### E4. The canonical application-history reader lives under storage/egress but is shared with native restart + +`low · confirmed · +0 lines · S` +Where: `sequencer/src/storage/egress/canonical.rs:40-143`, `sequencer/src/storage/egress.rs:18-73`, `sequencer/src/ingress/inclusion_lane/catch_up.rs:65`, `sequencer/src/commands/setup/fill.rs:409` + +- Claim: canonical_history_page, history_bounds, ApplicationInputRow, L2TxContext and decode_application_input (storage/egress/canonical.rs:40-143, storage/egress.rs:18-73) are the single application-history reader. Native restart catch-up (ingress/inclusion_lane/catch_up.rs:54,65) and setup fill (commands/setup/fill.rs:409) both use it. Minor correction: history_bounds lives in canonical.rs:41-43/63-71, not next to next_executed_input_count_in. The move target is storage/history.rs, a single file today, or a new history/ directory. The stale module doc l2_tx_feed/mod.rs:4 should be fixed in the same pure move (see ES1). +- Fix: Pure move: put canonical.rs and the row/context/decode types under storage/history/ next to history_bounds and next_executed_input_count_in. storage/egress then holds only historical.rs, which E1/E2 freeze. No behavior change. + +### E5. L2TxFeed duplicates http::storage_task's terminal-fault policy with two near-identical error enums + +`low · confirmed · -90 lines · S` +Where: `sequencer/src/egress/l2_tx_feed/mod.rs:80-109`, `sequencer/src/egress/l2_tx_feed/mod.rs:121-141`, `sequencer/src/egress/l2_tx_feed/error.rs:11-87`, `sequencer/src/http.rs:224-266`, `sequencer/src/egress/l2_tx_feed/mod.rs:47-53` + +- Claim: prepare() (l2_tx_feed/mod.rs:80-109) re-implements storage_task (http.rs:224-249). SubscribeError and SubscriptionError (error.rs:11-83) are four-variant twins of what persistent_storage_error (http.rs:251-266) already classifies. open_error_is_persistent is an alias (error.rs:85-87). task is always Some (mod.rs:145), so the early return in finish (169-171) is dead. runtime_scope() has one caller (subscribe.rs:89), and its select arm duplicates Subscription::recv's shutdown arm (mod.rs:157-161). send_ws_event warns on an infallible serialization of Strings and integers (subscribe.rs:127-133 vs the expect at :63). Also dead: the `Join => source.is_panic()` arms (error.rs:46, 80). Panics abort before any Join is built (mod.rs:95-106, 136-139), and SubscriptionError::Join is built only in finish() (mod.rs:175), after the classification at :132. Moving to storage_task keeps the lock-retention guarantee: it holds a RuntimeScope clone (http.rs:234), which retains the process lock (shutdown.rs:28-33). Persistent-fault detection still works because HistoryReadError::Storage exposes its rusqlite source via #[from] (canonical.rs:36-37). +- Fix: Run prepare through storage_task and, in the handler, downcast HistoryReadError::Policy to the 409 the same way api/history.rs:149-161 does. Reduce SubscriptionError to one small enum checked with persistent_storage_error. Destructure self in finish(). Drop the shutdown arm in run_ws_session, runtime_scope(), and send_ws_message; use expect in send_ws_event. Do this only when the feed is next touched (see E3). +- Decision: Worth doing only if the server-side feed cursor survives the sync redesign; otherwise leave it frozen. + +### E6. WS and history error logs drop their SQLite cause and mislabel what failed + +`low · confirmed · +0 lines · S` +Where: `sequencer/src/egress/api/subscribe.rs:72`, `sequencer/src/egress/api/subscribe.rs:121-123`, `sequencer/src/egress/l2_tx_feed/error.rs:13-22`, `sequencer/src/egress/l2_tx_feed/error.rs:56-66`, `sequencer/src/storage/egress/historical.rs:36-37` + +- Claim: The feed's error Displays drop their SQLite cause. 'cannot open subscription storage' and 'cannot load feed head offset' (error.rs:13-22) and 'cannot load ordered tx page from offset {offset}' (error.rs:61-66) all keep the cause only in #[source]. That loses it in the warn logs (subscribe.rs:72, 122) and, more importantly, in the terminal abort message. For a persistent fault, mod.rs:92 and :133 call abort_terminal with `{error}`, and abort_terminal logs only Display (runtime/shutdown.rs:16-18). The process therefore aborts on, say, a corrupt database with no SQLite reason in the log. By contrast, storage_task paths keep the cause, because StorageOpenError is transparent (storage/mod.rs:267-270) and HistoryReadError::Storage formats {0} (canonical.rs:36). The From impl (error.rs:32-38) actively degrades the more informative HistoryReadError. HistoricalReadError::Storage's 'reading historical L1 inputs' label (historical.rs:36) does include the cause and is only mislabelled for /history. The fix (include {source}, or adopt E5) changes log text only. +- Fix: Include {source} in these Display strings, or log through an error-chain formatter. Rename the end-of-session log to 'tx feed subscription ended with error'. Rename LoadHeadOffset to ReadPage. Use neutral 'reading history' wording. This changes log text only. + +### E7. SDK dead surface and a misnamed transport error + +`low · partially · -35 lines · S · public-rust-api` +Where: `sdk/rust-client/src/errors.rs:12-48`, `sdk/rust-client/src/errors.rs:62-87`, `sdk/rust-client/src/lib.rs:68-75`, `sdk/rust-client/src/lib.rs:81-86`, `sdk/rust-client/src/lib.rs:150-162` + +- Claim: Dead SDK surface is confirmed. SubmitTxError::{TimeoutWrite,TimeoutFlush,IoWrite,IoFlush,Parse} (errors.rs:16-31) are never constructed: map_reqwest_error (lib.rs:238-250) and the body-read maps are the only constructors. SubscribeError::InvalidEndpoint (errors.rs:83-84) is never constructed. host_port (lib.rs:68-75) has no callers. QueryError::Transport wraps SubmitTxError (errors.rs:65-68; lib.rs:150). There are two more mapping defects. Body-read failures, including reqwest per-request timeouts, are mapped to IoRead rather than TimeoutRead (lib.rs:111-114, 159-162), so benchmark breakdown keys (tests/benchmarks/src/rejection.rs:23) mislabel timeouts as io_read. A client build failure is reported as ClientBuildError::InvalidEndpoint (lib.rs:55-56). The with_request_timeout sub-proposal is wrong. The rebuild differs from the constructor's successful build only in the timeout values, which reqwest's build() does not fail on, so the expect at lib.rs:84 encodes an impossible state. Under AGENTS.md that fails loud by design, and returning a Result would add public-API churn for an unreachable error. +- Fix: Rename SubmitTxError to TransportError, drop the dead variants, InvalidEndpoint and host_port, and make with_request_timeout return a Result. Defer unifying the seven error enums into one {route, transport | status+typed ApiError body | decode | history} type until the N24 split creates separate public and internal clients. + +### E8. The 50,001-row 'no total cap' feed test duplicates pure validate() coverage + +`low · confirmed · -50 lines · S` +Where: `sequencer/src/egress/l2_tx_feed/tests.rs:86-134`, `sequencer-core/src/history.rs:414-434`, `sequencer/src/storage/egress/canonical/tests.rs:309` + +- Claim: subscription_accepts_more_than_fifty_thousand_inputs_without_a_total_cap (l2_tx_feed/tests.rs:87-136) builds 50,001 rows and reads 3 events. The feed adds no cap of its own: prepare and run only call canonical_history_page (mod.rs:87, 198). The absence of a cap is already pinned by core history.rs:415-434 (0, 100_001, 0), and deep paging by canonical/tests.rs:309-335, which also uses 50,001 rows. Delete the feed test or shrink it to a few rows. +- Fix: Delete the test, or shrink it to a few rows with page_size 1 if an end-to-end feed check is wanted. + +### E9. The egress contract is written out three times (README, openapi.yaml, protocol docs) + +`medium · confirmed · -250 lines · M · docs-only` +Where: `README.md:271-312`, `README.md:314-440`, `openapi.yaml:236-474`, `docs/protocol/application-history.md:138-177`, `docs/protocol/projection-replay.md:1-125` + +- Claim: The egress contract is duplicated across README.md:271-440, openapi.yaml:236-474 and the protocol docs. Verified examples: the order of refusal checks at README.md:290-293 and openapi.yaml:250-253; 'disconnect without a Close frame' at README.md:304-306, openapi.yaml:260-262 and application-history.md:173-177; the SDK prose at README.md:436-440 and application-history.md:160-164. application-history.md:11-12 and projection-replay.md:6-9 both name the README as owner, so changing the owner to openapi.yaml must update those links too. +- Fix: Make openapi.yaml the owner of wire, limit and refusal detail. Cut the README API section to a route table (public vs internal) plus one paragraph per side with links, and keep only semantics in the protocol docs. Do this together with the E1-E3 freeze: replace the frozen subsections with a link and a 'frozen pending sync redesign' note rather than editing their prose. +- Decision: Confirm that openapi.yaml, not the README, is the owner of the wire contract (the PR #50 review raised ownership). + +### E10. Stale and restating module comments in egress + +`nit · unverified-nit · -8 lines · S · docs-only` +Where: `sequencer/src/egress/mod.rs:4-6`, `sequencer/src/egress/api/subscribe.rs:4-6`, `sequencer/src/egress/api/health.rs:26-28`, `sequencer/src/http.rs:142-143` + +- Claim: egress/mod.rs says 'WS subscribe (today), future read-only endpoints', but /history, /historical-l1-inputs and the snapshot routes already exist. subscribe.rs:5-6 restates RAII ('permit is held for the lifetime of the session and released on disconnect via Drop'). health.rs:26-28 restates the struct's two fields. +- Fix: Reduce egress/mod.rs to one line naming the internal surface. Delete the RAII sentence and the field restatement. Point the http.rs split comment to the register entry instead of 'the api split'. + +### ES1. WS feed module and test still claim to serve native catch-up + +`nit · skeptic-added · +0 lines · S · docs-only` +Where: `sequencer/src/egress/l2_tx_feed/mod.rs:4`, `sequencer/src/egress/l2_tx_feed/tests.rs:139`, `sequencer/src/egress/l2_tx_feed/tests.rs:185`, `sequencer/src/ingress/inclusion_lane/catch_up.rs:65` + +- Claim: The l2_tx_feed module doc says the feed is 'used by WS subscriptions and catch-up replay', and the lock test is named cancelled_catchup_prepare_... and asserts 'detached catch-up preparation must retain process ownership'. Catch-up does not use L2TxFeed: it calls canonical_history_page directly. Anyone freezing or deleting the feed under E3 would wrongly conclude that native restart depends on it. +- Fix: In the E4 pure move, reword mod.rs:4 to 'DB-backed ordered-L2-tx feed for WS subscriptions'. Rename the test to cancelled_subscription_prepare_retains_process_lock... and fix its assert message. + +Keep (load-bearing): + +- Mandatory HistoryClaim (era, generation, next_input) on every WS subscribe (subscribe.rs:30-52; core history.rs:185-209). Do-not-simplify 'Don't retain a numeric resume offset without its history identity' (invariants.md:464-465, I18): recovery reuses suffix offsets. +- Per-page claim revalidation inside one read transaction (l2_tx_feed/mod.rs:196-210, canonical.rs:47-60). This is the actual safety gate; the pre-upgrade prepare() only exists to return a typed 409 before the 101. +- X-History-Error header duplicating the 409 body (subscribe.rs:61-69). WebSocket clients may stop reading after the failed upgrade's headers; the SDK reads the header first (sdk lib.rs:289-302), and a test pins this (lib.rs:409-423). +- Gap, length and i64::MAX+1 guards in canonical_page_in (canonical.rs:82-134): 'impossible states fail loud', on a reader shared with native restart catch-up (catch_up.rs:65). +- Generation cuts and the complete-lineage assert (storage/history.rs:98-148): the in-era standard-recovery mechanism; it survives the direction, and a missing cut must never authorize a partial minimum (I18). +- base_executed_input_count / next_executed_input_count_in (storage/history.rs:85-96): the sequencer's own absolute offsets after a rebuild. This is N12's base C, needed by native replay regardless of the client design. +- Process-lock retention by detached feed blocking work (l2_tx_feed/mod.rs:82, 121-141; test tests.rs:139-205): data-directory ownership must outlive cancelled requests. +- Subscriber cap (64) checked before claim validation (state.rs:42-49, http.rs:145). Each subscriber pins a blocking thread polling every 20 ms; the cap bounds blocking-pool use and SQLite polling (about 3,200 small read transactions per second at the cap, acceptable for internal consumers). +- HistoricalResponseBody tying the admission permit to the body's lifetime (egress/api/history.rs:116-176): bounds in-flight page memory to 8 pages. Keep until E1 removes the route; do not polish. +- Health probes on the internal listener (health.rs:11-12): operators and the kubelet probe from inside the cluster. + +## F. L1 reader, batch submitter, provider + +Verdict: Area F covers the reader, the submitter and the shared provider code. The reader ingests safe InputBox events after a chain-id gate. It checks each batch two ways: index contiguity, and a getNumberOfInputs count pinned at the safe block. The submitter is a stateless tick. It re-derives the unresolved suffix from the safe-accepted frontier plus a Latest scan, raises the wallet-nonce watermark durably before broadcasting (I14), and re-estimates fees on every tick. The content-identity check and the divergence freeze (I9/I15) run in the same transaction as the reader's append. Production code is about 3,300 lines, roughly 23% of them comments. The core mechanisms are proportionate and well chosen. The "large files" are mostly misplaced tests: reader.rs is about 813 production lines plus about 810 lines of tests. l1_submission.rs is only 271 production lines; about 1,000 of its 1,240 test lines exercise check_danger and populate_safe_accepted_batches, which live in other modules. poster.rs is about 546 production lines. The biggest real problems are classification and observability under failure. Self-trust violations and impossible states are routed into the transient "provider/transport" bucket. Provider errors are stringified with no operation, nonce or stage. They are logged at error level on every poll tick and, through reqwest's Display, carry the full RPC URL. The rest is leftover cleanup: a redundant poster-level shutdown select left over from 288ed14, a test-only partition model that duplicates the production bisect, per-nonce pending-age bookkeeping where tracking only the head nonce would do, dead public storage helpers, and rationale prose repeated three or four times. Keep the double input-completeness witness, the write-before-broadcast watermark, the same-transaction divergence freeze, the single encode path shared by seal and submit, and the no-internal-retry stateless tick. + + +### F1. Self-trust violations and impossible states are classified as transient provider errors + +`low · disputed · -15 lines · S · BC` +Where: `sequencer/src/l1/submitter/poster.rs:532-539`, `sequencer/src/l1/submitter/worker.rs:244-266`, `sequencer/src/l1/reader.rs:458-465`, `sequencer/src/l1/reader.rs:740-748` + +- Claim: Three things are true. (a) The impossible usize/overflow paths at reader.rs:458-465 and 740-748 are wrapped as InputReaderError::Provider. The codebase's own pattern for this is expect("...: contract-impossible") (storage/ingress.rs:254, storage/l1_inputs.rs:432). (b) worker.rs:245-265 re-validates the poster's answer and reports the mismatch as a retryable Provider error, which the self-trust section rules out ('no re-validation of a neighbor module's answer', threat-model README.md:34). (c) An undecodable own-key batch maps to Provider (poster.rs:538-539). The claim overstates the impact, though. Nothing here is 'retried silently': worker.rs:193 logs each occurrence at error level. The undecodable-landing stall also clears on its own once the landing passes safe, because the scan starts at safe_block+1 (worker.rs:215-218) and populate already treats undecodable copies as scheduler no-ops (safe_accepted_batches.rs:222-225). None of this has a reachable correctness effect. It is classification and labelling of impossible events. +- Fix: First patch (refactor only): replace the impossible usize/overflow paths with expect() and a reason, and turn the worker's poster-count self-checks into debug_assert! or delete them. Second patch (behavior change): give the own-key undecodable batch its own variant carrying tx hash and InputAdded index. Decide whether it is terminal (the self-trust rule) or skipped in the fold to mirror the scheduler's no-op treatment. Do not label it a provider error either way. +- Decision: For an undecodable batch from our own key: terminal exit (the self-trust rule) or skip it in the nonce fold (mirroring the scheduler's no-op)? +- Independent trace: refuted. The code reads as the reviewer describes. + +### F2. Worker error logs lack operation, nonce and stage, fire at error level every tick, and mislabel SQLite faults + +`low · confirmed · +10 lines · S` +Where: `sequencer/src/l1/reader.rs:318-319`, `sequencer/src/l1/reader.rs:495`, `sequencer/src/l1/reader.rs:798`, `sequencer/src/l1/reader.rs:806`, `sequencer/src/l1/submitter/worker.rs:188-195` + +- Claim: As stated, with one sharper detail. BatchPosterError::Watermark is #[error(transparent)] (poster.rs:70-71), and WalletNonceWatermarkError::Storage/OpenStorage use #[source] with fixed Display text (watermark.rs:25-28). worker.rs:193 logs `error = %source`, so a SQLite BUSY on the watermark writer is logged as 'L1 provider error — will retry' with only 'watermark sink storage write failed' and no rusqlite cause. The divergence marker persists safe_input_index (safe_accepted_batches.rs:362-371), so the tx hash can be recovered with a join. Adding it to the log is a convenience, not a gap. +- Fix: Prefix each Provider string with its operation and key fields (e.g. 'estimateGas nonce=N', 'eth_getBlock(safe)'). Use a small map_err helper per module, not a new error framework. In worker.rs, log Watermark failures under their own message. Add transaction_hash, plus our hash and the landed hash, to the divergence error log. Whether to downgrade the per-tick retry logs to warn after the first occurrence is the maintainer's call. Fix URL leakage through the register entry (see notes_triage), not here. +- Decision: Keep error level on every retry tick, or log error once and then warn at a lower rate? + +### F3. l1_submission.rs is 1,510 lines because about 1,000 test lines belong to other modules + +`low · confirmed · +0 lines · S` +Where: `sequencer/src/storage/l1_submission.rs:509-940`, `sequencer/src/storage/l1_submission.rs:942-1510`, `sequencer/src/storage/recovery.rs:233`, `sequencer/src/storage/recovery.rs:643`, `sequencer/src/storage/safe_accepted_batches.rs:131` + +- Claim: Production code in l1_submission.rs ends at line 271, and tests start at 272. Roughly 850 test lines, not 1,000, belong elsewhere: check_danger tests at 525-940, and populate/divergence/seal-hash tests at 941-1315 and 1390-1510. pending_batches_skips_invalidated_and_respects_min_nonce (1316) and nonce_is_reused_after_torn_cascade (1347) test this module's own pending_batches and should stay. check_danger's tests live in a separate file (`mod tests;` at storage/recovery.rs:643). The local helpers default_test_protocol and unix_now_ms (509-523) must move with the tests. +- Fix: Move the check_danger tests into storage/recovery's tests module, and the populate, divergence and seal-hash tests into safe_accepted_batches.rs tests. Pure move: no code changes and a separate patch. + +### F4. Poster-level shutdown select is a redundant leftover of 288ed14 + +`low · confirmed · -40 lines · S` +Where: `sequencer/src/l1/submitter/poster.rs:68-69`, `sequencer/src/l1/submitter/poster.rs:81`, `sequencer/src/l1/submitter/poster.rs:160-161`, `sequencer/src/l1/submitter/poster.rs:372-381`, `sequencer/src/l1/submitter/poster.rs:442-447` + +- Claim: EthereumBatchPoster races every send against the same RuntimeScope that the submitter's run_forever already biased-selects on. The poster's BatchPosterError::Shutdown arm can never reach a caller in production. +- Fix: Remove the poster's shutdown field, externalize_provider_call, the Shutdown variant and its classification arm, and the dedicated test. Cancellation stays with the worker's run_forever select. Do it as a refactor-only patch. +- Decision: Confirm no planned non-worker caller (e.g. a future one-shot submit command) relies on the poster cancelling itself. + +### F5. The watchdog parity fixture drives a test-only copy of the bisect, not the production code + +`low · partially · -60 lines · S` +Where: `sequencer/src/l1/partition.rs:68-98`, `sequencer/src/l1/partition.rs:108-178`, `sequencer/src/l1/partition.rs:180-184`, `sequencer/src/l1/partition.rs:230-239`, `sequencer/src/l1/partition.rs:212-218` + +- Claim: Core confirmed. partition_vector_simulation_matches_watchdog (partition.rs:321-353) drives the cfg(test) recursive model (116-178), so the production worklist (68-98) is only claimed to match it in prose (64-67). A generic bisect driven by the fixture is the right fix, though the parity test then becomes async. Two parts of the proposal are weaker than stated. sort_logs_by_l1_order is the seam the log-sort parity test calls (partition.rs:359). Deleting it only moves a tautology, because neither version tests production's key closure (219-225). The better fix is to make that key closure the shared function. The reader's missing-block_number branch (reader.rs:684-688) is unreachable through the ordered fetch (partition.rs:213-218), but it is pinned by a unit test (reader.rs:1347). ingest_input_added is documented as a pure function driven by synthetic pairs (reader.rs:673-675). Converting it to expect means deleting that test. That is acceptable under self-trust, but it is not free. +- Fix: Make the worklist generic over its query closure (async fn bisect(from, to, splittable, query)). Use it in production, and drive it in the parity test with a closure that replays the fixture. Delete simulate_partitioned_get_logs and the two one-line wrappers. Change the reader's duplicate check to expect("ordered fetch guarantees block_number"). + +### F6. Per-nonce pending-age map where tracking only the head nonce would do + +`nit · partially · -35 lines · S · BC · docs-only` +Where: `sequencer/src/l1/submitter/poster.rs:117-144`, `sequencer/src/l1/submitter/poster.rs:150-155`, `sequencer/src/l1/submitter/poster.rs:181-189`, `sequencer/src/l1/submitter/poster.rs:201-234`, `sequencer/src/l1/submitter/poster.rs:465-473` + +- Claim: Tracking only the head nonce is not equivalent to the documented contract. docs/l1-fee-policy.md:46-50 promises a per-nonce clock from first attempt that 'Accepted re-broadcasts do not reset'. Under head-only tracking the clock resets whenever Latest advances, so a slowly draining suffix (each nonce mining just under 5 min) never warns, even though its tail batch has been unresolved for k×5 min. The per-nonce map is the minimal structure that keeps first-attempt times for upcoming nonces. The real defect is noise: during a stall every tracked nonce warns independently (poster.rs:203-218), giving k warnings per interval. Warning only for the lowest tracked entry, which holds the oldest clock, fixes that with almost no line change and no doc change. +- Fix: Replace the map with Option<(head_nonce, since, last_warned)>: reset when Latest's nonce changes, warn after PENDING_WAIT_WARN_AFTER, and log the refusal info once per head. Update the fee-policy paragraph to say 'Latest nonce has not advanced for N minutes'. Patch only once the warn semantics are agreed. +- Decision: Is the 'unresolved since first attempt, per nonce' clock worth keeping over 'head stuck for N minutes'? + +### F7. Dead public storage helpers, a test-only error variant and a stale module doc + +`low · confirmed · -25 lines · S` +Where: `sequencer/src/storage/l1_submission.rs:4-13`, `sequencer/src/storage/l1_submission.rs:54-56`, `sequencer/src/storage/l1_submission.rs:108-121`, `sequencer/src/storage/l1_submission.rs:124-129`, `sequencer/src/commands/setup/fill.rs:322` + +- Claim: Confirmed, with two corrections. frames_for_batch_in is production code (l1_submission.rs:243, via load_batch_frames_in); only the Storage::frames_for_batch wrapper (127-129) is test-only. The getter Storage::batch_tree_anchor (l1_submission.rs:59) is also test-only (fill.rs:207/288 in its test module, l1_inputs.rs:671-685); the e2e/harness batch_tree_anchor calls are the harness's own methods. WalletNonceWatermarkError::Other is constructed only at poster.rs:766 and watermark.rs:97, both tests. The test sink can return Storage(SQLITE_BUSY) instead. +- Fix: Mark the three methods cfg(test), or delete them and have the tests call the _in helpers. Fix the module doc. Have the test sink inject a busy rusqlite error instead of keeping the Other variant. + +### F8. Cockroach-recovery seams in the reader and its storage: freeze, don't polish + +`low · partially · +0 lines · S` +Where: `sequencer/src/l1/reader.rs:138-141`, `sequencer/src/l1/reader.rs:225-231`, `sequencer/src/storage/l1_inputs.rs:69-152`, `sequencer/src/storage/l1_inputs.rs:176-183`, `sequencer/src/storage/l1_inputs.rs:266-268` + +- Claim: The freeze recommendation stands, but the inventory needs fixing. open_tip_nonce (l1_submission.rs:110-121) is test-only, per F7, so it is not a live recovery seam: delete it rather than freeze it. The missing-baseline branch in populate (safe_accepted_batches.rs:166-177) also serves plain setup, per its own comment ('Plain setup detects prior activity before installing genesis'), so it is not cockroach-only. The real seams are FrontierMode::DeferUntilAnchorSet and set_frontier_mode (reader.rs:138-141, 225-231; sole caller setup/mod.rs:262), first_batch_submitter_input_after_block (l1_inputs.rs:85; setup/mod.rs:705) and safe_inputs_in_block_range (l1_inputs.rs:120; setup/mod.rs:488/494). Together they come to about 100 production lines. +- Fix: Freeze: no doc or abstraction polish on these items. Carry two constraints into the tool's design. First, it must reuse the reader's verified ingest rather than a fresh log scan (the warning at l1_inputs.rs:74-79 remains true). Second, the batch-tree anchor (I16) survives in any design, because a tool-produced sequencer state still roots at N'. + +### F9. Rationale repeated three or four times, design-history prose, and table cells over 150 words + +`low · partially · -120 lines · M · docs-only` +Where: `sequencer/src/l1/mod.rs:16-29`, `sequencer/src/l1/provider.rs:127-134`, `sequencer/src/l1/submitter/poster.rs:46-50`, `sequencer/src/l1/submitter/poster.rs:394-402`, `sequencer/src/l1/reader.rs:38-44` + +- Claim: The mid-life chain-failover rationale really is stated three times: provider.rs:127-134, poster.rs:46-50 and poster.rs:394-402, the last one tagged '(review)'. The design-history prose at l1/mod.rs:23-29, safe_accepted_batches.rs:122-130 and l1_inputs.rs:41-42 is as described. reader.rs:38-44 is not a repeat, though: it covers in-model operator URL repointing across restarts. The 'Uniswap row' is threat-model README.md:24 (about 208 words); line 25 is the 17-word batch-submitter-key row. The RPC row (:22) is about 175 words. The Uniswap row belongs to the fee-oracle area. The failover rationale itself names a scenario the threat model excludes (see FS1), so trimming it is a content decision, not just a wording one. +- Fix: State each rationale once: chain-id window at create_verified_signer_provider; poster and reader link to it in one line. Delete the design-history sentences. In the threat model, cut each actor row to trust level plus a one-line capability, and move the completeness-witness and oracle-outage arguments into short subsections. Leave the F8 items untouched. + +### F10. SubmitterExit is a one-variant enum that every caller maps away + +`nit · unverified-nit · -10 lines · S` +Where: `sequencer/src/l1/submitter/worker.rs:69-78`, `sequencer/src/commands/run/workers.rs:144`, `sequencer/src/commands/run/workers.rs:353`, `sequencer/src/commands/run/workers.rs:619-621` + +- Claim: SubmitterExit::Shutdown is the only variant, and the supervisor immediately pattern-maps it to (). +- Fix: Return Result<(), BatchSubmitterError> like the reader does. + +### FS1. Chain-id gating is asymmetric: the poster defends against mid-life failover, but the reader checks only once + +`low · skeptic-added · -6 lines · S · BC` +Where: `sequencer/src/l1/reader.rs:146`, `sequencer/src/l1/reader.rs:344-348`, `sequencer/src/l1/reader.rs:488-502`, `sequencer/src/l1/reader.rs:394-401`, `sequencer/src/l1/submitter/poster.rs:46-50` + +- Claim: The poster re-checks eth_chainId before every productive send, specifically to survive a load-balanced RPC that fails over to another chain mid-life. The reader is the component whose ingest writes durable state, yet it verifies the chain id only once per instance (the chain_id_verified flag). Its comment says the check runs 'before reading anything from it'. Either mid-life failover is in scope, in which case the reader is the weaker gate, or it is out of scope ('must be one consistent node'), in which case the poster's gate and its three rationale comments defend an excluded scenario without naming that assumption. +- Fix: Decide the model first. If failover is in scope, drop the chain_id_verified flag and check every tick. That is stateless, about -6 lines and one extra eth_chainId per poll. If it is out of scope, reword poster.rs:46-50/394-402 and provider.rs:131-134 to name the assumption and a revisit trigger, and fix the overstated reader comment. +- Decision: Is a mid-life RPC chain failover inside the supported trust model? + +### FS2. The watermark raise does a blocking SQLite open and durable write on the async submitter and flusher tasks + +`low · skeptic-added · +6 lines · S · BC` +Where: `sequencer/src/l1/submitter/poster.rs:426`, `sequencer/src/recovery/flusher.rs:263`, `sequencer/src/l1/watermark.rs:63-73`, `sequencer/src/storage/open.rs:171`, `sequencer/src/l1/submitter/worker.rs:274` + +- Claim: StorageWatermarkSink::raise_to opens a writer connection with a 5 s busy_timeout and commits a durable MAX upsert synchronously. It is called inline from async submit_batches and from the flusher. The submitter routes every other SQLite access through spawn_blocking_with_lock. Under writer contention with the lane, this ties up a tokio worker thread for up to 5 s, and the biased shutdown select cannot preempt it. +- Fix: Run the sink call through spawn_blocking_with_lock, in the poster or by making the sink own the lock and the blocking hop, and await it before the first send. Write-before-broadcast ordering is preserved, and the MAX upsert is idempotent if a cancelled raise finishes in the background. Do this in the same patch that settles the register's SQLite-contention entry, so both paths get one policy. + +Keep (load-bearing): + +- Double input-completeness witness in the reader (reader.rs:375-405, 438-466): index contiguity proves the right prefix, and getNumberOfInputs pinned at the scanned safe block proves the prefix is complete. It is the only defense against truncated get_logs from a lagging endpoint. The count-first fast path is the same trust base and saves a long paginated crawl. +- Log/payload cross-check in ingest_input_added (reader.rs:678-724): block number and index are both carried by the log and by the EvmAdvance payload, and must agree. It mirrors the watchdog's decode_and_validate_log. +- Three chain-id gates (reader.rs:488-504 once per reader, poster.rs:403-413 per non-empty tick, provider.rs:135-154 for keyed one-shot writers): they protect different windows (ingest, long-lived signing, one-shot signing). Keep the code and trim the repeated prose (F9). +- Write-before-broadcast watermark (watermark.rs:43-74, poster.rs:422-426, l1_submission.rs:85-97): I14. A single MAX upsert committed with synchronous=FULL is the whole zombie-prevention anchor for the flush. +- Content-identity check and divergence marker written in the same transaction as the reader's append (safe_accepted_batches.rs:146-154, 225-245, 296-335): I9/I15. The early return on an existing marker is the frontier freeze. The per-invocation scan cursor (113-120) is deliberate so a recovery batch reusing a nonce is not skipped. +- load_batch_frames_in as the single encode path shared by seal and submitter (l1_submission.rs:231-270): the seal-time hash must come from the same bytes the submitter broadcasts, or I15 raises false mismatches. +- Stateless submitter tick that re-derives everything from the safe frontier plus a Latest scan, with no internal retry (worker.rs:207-269): on the Do-not-simplify list (docs/invariants.md:468-470). The orchestrator respawn is the retry mechanism. +- is_pending_tx_conflict classification (poster.rs:329-350): lets the rest of the suffix go out past a held nonce. An unrecognised wording falls back to the old behavior, so it is safe. +- pad_gas_estimate (eip1559.rs:20-25): an out-of-gas revert would burn a wallet-nonce slot with no InputAdded. +- L1-order sort plus refusal of logs missing a block number, at one point (partition.rs:192-228): both the reader's contiguity witness and the submitter's nonce fold need canonical order. +- Operator-configurable long-range error codes for the bisect (partition.rs:11-26): range-error messages are not stable across providers. The overload with rate-limit codes is documented and accepted. +- App-deployment-block scan floor gated on the v3 InputBox version() witness (reader.rs:171-190, 601-649): turns a potentially enormous genesis crawl into a bounded one, and is sound only under the verified v3 semantics. +- HTTPS enforcement with an explicit insecure opt-in, and SubmitterKey redaction (provider.rs:30-50, l1/mod.rs:46-71): cheap guards against operator foot-guns and key leaks. + +## G. Automatic recovery and cockroach recovery (setup --recovery) + +Verdict: This area does two separate jobs. The first is automatic startup recovery: the detector exits, the orchestrator respawns, startup inspects, syncs, picks one repair under a guarded transaction, inspects again, and runtime admission checks a final time. The second is cockroach rebuild via `setup --recovery`: load a checkpoint, flush, re-sync, fold, publish a baseline. Production code is about 2,600 lines. The large file sizes are mostly tests (flusher.rs is ~405 prod / ~530 test, recovery/mod.rs ~505/~560, fill.rs ~75/~540). The automatic-recovery core is proportionate to its real concurrency and durability complexity: one dispatch table, Immediate transactions that re-inspect before mutating, a flush observation kept only on the stack, and a typed retry/refuse split. I found no high-severity bug. The real defects are a two-call TOCTOU in the flusher's completion observation (a fail-loud backstop exists) and a leftover plain-setup `--checkpoint-block` that turns an early clean refusal into a later, absorbing CanonicalDivergence. The biggest structural problem is that cockroach recovery is woven into `setup` (~270 prod lines in setup/mod.rs, ~450 counting config, errors and tests). That is exactly what the planned shared tool replaces, so it should be frozen. The pieces that survive (Checkpoint load, the seed/replay split, `fold_replay`, `fill_recovery_state`) already form an almost pure seam. N1 is cheap (~45 lines) with a genesis selector. docs/recovery/README.md is accurate and tight, but code comments repeat themselves (the DangerStatus arms are described four times) and carry stale history. + + +### G1. Freeze `setup --recovery` in-band orchestration (~450 lines incl. config/errors/tests) + +`medium · partially · -450 lines · L · config` +Where: `sequencer/src/commands/setup/mod.rs:405-673`, `sequencer/src/commands/setup/checkpoint_tests.rs:1-226`, `sequencer/src/commands/config.rs:378-453`, `sequencer/src/commands/error.rs:387-430`, `sequencer/src/recovery/mod.rs:79-86` + +- Claim: Freezing the in-band `setup --recovery` orchestration is right. The code that only cockroach recovery uses is `Checkpoint`/`load`/`source_fold_inputs`/`recover`/`rebuild_from_checkpoint`/`flush_wallet_nonce`/`to_fold_input` (sequencer/src/commands/setup/mod.rs:405-673), the recovery config and validation (config.rs:378-453), `SetupRecoveryError` (error.rs:387-430), the frontier toggle (setup/mod.rs:255-263), `CheckpointAheadOfStop` (recovery/mod.rs:79-86, raised only at setup/mod.rs:571), and `assert_resync_caught_up` (recovery/mod.rs:136, called only at setup/mod.rs:555). The docs scope is mis-stated, though. cockroach.md:140-160 'Application-specific reader state' (restore the client checkpoint, replay L1 through C, drain, then agree on K) is not removed by the direction. It is the manual form of the client half that the shared tool will automate, and G2 itself relies on it (cockroach.md:142-144), so keep it as that half's spec. docs/protocol/projection-replay.md is mostly the general egress bootstrap: /history, replay from genesis, subscribe at K (projection-replay.md:11-31), plus standard-recovery generation compatibility (:44-56). The new sync supersedes it, but it belongs to the egress area and is not cross-era cockroach machinery. +- Fix: Freeze: fix only safety bugs. Do not polish its error classification (including the re-sync site of the open register entry), load ordering, receipt format, or SetupRecoveryError wording. About 150 of these lines move into the tool library (see G2); the rest is deleted at extraction, not now. +- Decision: Confirm the freeze scope covers the projection-replay cross-era docs as well as the setup code. + +### G2. Extraction seam for the shared recovery tool: keyless, generic over Application, C supplied + +`medium · partially · -250 lines · L · BC · config` +Where: `sequencer/src/commands/setup/mod.rs:560-614`, `sequencer/src/commands/setup/mod.rs:405-499`, `sequencer-core/src/scheduler/fold.rs:4-20`, `sequencer-core/src/scheduler/fold.rs:62-75`, `sequencer/src/commands/setup/fill.rs:36-50` + +- Claim: The pure seam is real. `rebuild_from_checkpoint` only sources `safe_inputs`, folds, and fills (setup/mod.rs:560-614). `fold_replay` is side-effect free and already expects a future standalone tool (sequencer-core/src/scheduler/fold.rs:12-15). The checkpoint loads only after the full sync (setup/mod.rs:534 vs 264), and B<=C is checked only after the signed flush (569 vs 537). The C-sourcing claim is wrong. `flush-mempool` does not return C: the command returns `()` and only logs `safe_block` (commands/flush.rs:23, 78-83). It also cannot run in either cockroach scenario. On a wiped DB it needs a completed setup (lifecycle.rs:250-252). On the surviving diverged DB its preflight refuses on canonical divergence (commands/flush.rs:30 -> lifecycle.rs:141). A keyless tool therefore needs a setup-less flush mode: submitter address plus an optional watermark, emitting C in machine-readable form. Either that, or the tool embeds the flush. The rest of the design is a reasonable plan, but it is a future-tool design, not a refactor of today's code. +- Fix: (1) Library: `recover::(Checkpoint, inputs in (A,C] ascending, C, submitter) -> {app S', N', K, clock}`. Move into it Checkpoint+load (405-465), the seed/replay split (467-499), the A` publishes (S', N', K, C) through `fill_recovery_state`/`complete_baseline_setup`. Genesis becomes the trivial baseline. Setup loses --recovery, key args, the DeferUntilAnchorSet toggle, `flush_wallet_nonce`, `recover`, and SetupRecoveryError. (4) Client contract: the client runs the same library with its own richer `A`, from its own checkpoint (boundaries may differ, cockroach.md:142-144), to the same C. Agreement means equal (K, last_executed_safe_block) per I19 and equal N'. The receipt (era, C, N', K) is what the future sync binds to. Not worth investing in now: streaming fold inputs (setup/mod.rs:487-497 collects into a Vec, though fold accepts iterators), checkpoint.toml/finalized_snapshot receipt polish, setup-recovery error classification. +- Decision: Should C come from `flush-mempool` (keyless tool, recommended) or should the tool embed the flush? Should the library live in sequencer-core or a new crate? Should the tool write the sequencer DB directly or emit a bundle for `setup --baseline`? + +### G3. N1: genesis recovery needs a hand-made bundle; add a genesis checkpoint selector + +`medium · confirmed · +45 lines · S · BC · config` +Where: `sequencer/src/commands/config.rs:425-428`, `sequencer/src/commands/setup/mod.rs:428-464`, `sequencer/src/commands/setup/mod.rs:318-330`, `sequencer/src/egress/api/snapshot.rs:205-218`, `sequencer/src/commands/setup/mod.rs:839-909` + +- Claim: Today `--recovery` requires `--checkpoint-dump-dir` holding info.toml, checkpoint.toml, and app state. checkpoint.toml is produced only by egress `/finalized_snapshot`. Plain setup cannot be used to mint a genesis artifact on a chain with prior batches, because its gate refuses before the genesis app is built. The loader already accepts genesis (`is_genesis`, setup/mod.rs:449-452), and the binary already has the genesis constructor (setup/mod.rs:47-50, used at 329), but recovery never receives it. +- Fix: Add `--checkpoint genesis` (or: --recovery with B=0 and no dump dir) that builds `Checkpoint{app: genesis_app()?, A:0, N:0, B:0}`, reusing the count-zero check from fill.rs:18-28. Then remove the `is_genesis` exception from `Checkpoint::load`, so supplied bundles must strictly satisfy A0` is a leftover knob that defers a clean refusal into absorbing divergence + +`low · disputed · -10 lines · S · BC · config` +Where: `sequencer/src/commands/config.rs:368-377`, `sequencer/src/commands/setup/mod.rs:233-246`, `sequencer/src/commands/setup/mod.rs:318-330`, `sequencer/src/commands/setup/mod.rs:704-705`, `sequencer/src/commands/setup/fill.rs:29-30` + +- Claim: Plain `setup --checkpoint-block B>0` is a dead, misleadingly documented knob (config.rs:368-375 'PR3 ...'), but the failure is not deferred to a later sync. Plain setup syncs with `FrontierMode::Populate` (only recovery defers, setup/mod.rs:261-263), with base 0 before completion (safe_accepted_batches.rs:166-177) and anchor 0. A previously accepted batch therefore fails content identity as Foreign (safe_accepted_batches.rs:301-309) and records the divergence marker during setup's own initial sync. The gate then passes because the batch is <= B, and `register_genesis_baseline` -> `complete_baseline_setup` refuses on that marker (lifecycle.rs:104-111). So setup still refuses before completion, with the same remedy (wipe + cockroach). It just shows a divergence refusal instead of `BatchPastCheckpoint` (exit 40 'setup needs recovery'). If the prior batches were all scheduler-rejected, genesis is correct. The impact is diagnostic only. A simpler fix than adding validation: delete `--checkpoint-block` entirely. In recovery it only cross-checks the receipt the operator copies it from (setup/mod.rs:436-441; cockroach.md:178 '--checkpoint-block '), so B can come from checkpoint.toml. Keep the B-vs-deployment check (setup/mod.rs:239) for recovery. +- Fix: Reject `checkpoint_block != 0` without --recovery in `SetupConfig::validate` (config.rs:434-443). Call the gate with 0, and drop the plain-path B-vs-deployment-block check. Under G2, the flag moves to the tool entirely. +- Independent trace: refuted. The claim says plain setup with B>0 'defers a clean refusal into absorbing divergence' that 'a later sync records'. + +### G6. `TipAlreadyOpen` is documented as impossible yet handled as Retry + +`low · confirmed · -8 lines · S · BC` +Where: `sequencer/src/storage/recovery.rs:115-119`, `sequencer/src/storage/recovery.rs:267-269`, `sequencer/src/recovery/mod.rs:89-90`, `sequencer/src/recovery/mod.rs:497-499` + +- Claim: The variant doc says it is 'unreachable under the process lock, and retryable if it ever fires'. Retrying an impossible state goes against the 'impossible states fail loud' principle: the next boot re-inspects, finds Ready, and admits, hiding the only cause that can produce this state (a lock violation). +- Fix: Classify it as refuse (or `assert!` inside the transaction, which rolls back on unwind), and delete `RecoveryRetryReason::TipAlreadyOpen`. This changes the exit code from 20 to 30 for a state that should never occur. + +### G7. DangerStatus arms restated four times; flush docs narrate each other's callers + +`low · confirmed · -60 lines · S · docs-only` +Where: `sequencer/src/storage/recovery.rs:37-54`, `sequencer/src/storage/recovery.rs:57-83`, `sequencer/src/storage/recovery.rs:173-232`, `sequencer/src/recovery/flusher.rs:124-134`, `sequencer/src/commands/setup/mod.rs:620-628` + +- Claim: The DangerStatus arms are described in the enum-level list (storage/recovery.rs:37-54), the per-variant docs (57-83), the numbered list (173-205), and the trust-hierarchy list (206-228). The enum-level list is also incomplete: it omits `CanonicalDivergence`. Caller narration appears at flusher.rs:128-134 and setup/mod.rs:625-628, and meta commentary at detector.rs:12-16. Keep one ordering paragraph that cites I4, because the arm order is on the Do-not-simplify list (docs/invariants.md:459-461). +- Fix: Keep one-line variant docs plus one ordering paragraph citing I4. Delete the caller narration and meta commentary. + +### G8. Stale code docs: 'shared with the batch submitter', 'PR3', `flush_and_wait` as the entry point + +`low · confirmed · -10 lines · S · docs-only` +Where: `sequencer/src/storage/recovery.rs:10-12`, `sequencer/src/storage/recovery.rs:436`, `sequencer/src/commands/config.rs:372-375`, `sequencer/src/commands/error.rs:394-396`, `sequencer/src/recovery/flusher.rs:4-10` + +- Claim: Confirmed as stated, plus two more stale lines in the same vein. commands/flush.rs:9 says 'the same flush `setup --recovery` will run internally' in the future tense, although it already does (setup/mod.rs:653). The SetupRecoveryError doc (error.rs:388-390) says the post-flush re-sync uses the transient RecoveryError paths, but the re-sync's sync failure maps to `WorkerExit::InputReader` (setup/mod.rs:546-553). Only the caught-up check uses RecoveryError (555). +- Fix: Correct the wording; mark the test-only visibility as such. + +### G9. Log levels contradict the documented semantics (flush timeout, detector exit, per-slot send errors) + +`nit · partially · -5 lines · S` +Where: `sequencer/src/recovery/flusher.rs:380-389`, `sequencer/src/recovery/flusher.rs:236-247`, `sequencer/src/recovery/detector.rs:152-157`, `sequencer/src/recovery/detector.rs:35-37`, `sequencer/src/recovery/flusher.rs:354` + +- Claim: Only the flusher timeout contradiction is clear. flusher.rs:381-389 logs error! with 'should not happen during normal L1 operation', while flusher.rs:236-239 calls the same timeout 'the flush's ordinary rhythm, not a fault'. That deserves debug/warn. The detector point is debatable. detector.rs:35-37 says 'not an error' about the exit classification (a distinct CommandError, exit 10), not about severity. A danger detection stops soft confirmations, and the statuses include `CanonicalDivergence`, so a blanket warn would under-report divergence. If anything, choose the level per status. The per-slot error! (flusher.rs:354) is redundant only for the first three nonces; the aggregate carries just three samples (95-115), so the per-slot lines are the only record of the rest. +- Fix: Log the watch timeout at debug (keep the loop's single warn). Lower the detector to warn. Drop the per-slot error!, since the aggregate already carries three samples. + +### G10. Stringified transport errors can embed credential-bearing RPC URLs + +`low · confirmed · +10 lines · S` +Where: `sequencer/src/recovery/flusher.rs:283`, `sequencer/src/recovery/flusher.rs:324`, `sequencer/src/recovery/flusher.rs:353-354`, `sequencer/src/recovery/flusher.rs:402`, `sequencer/src/l1/provider.rs:146` + +- Claim: Confirmed. Fold it into the open register entry 'Startup logs the full RPC URL' as a scope extension; it is not a separate work item. +- Fix: Extend that register entry's scope to transport error strings and redact once at the provider boundary (strip the URL), rather than fixing each site. Verify with a synthetic credential URL. + +### G11. Flush waits on its own tx hash instead of slot consumption + +`low · confirmed · -25 lines · S · BC` +Where: `sequencer/src/recovery/flusher.rs:367-395`, `sequencer/src/recovery/flusher.rs:117-122`, `sequencer/src/recovery/flusher.rs:265-270`, `sequencer/src/recovery/flusher.rs:62-66` + +- Claim: Accurate, but the race is narrower than implied. No-ops are sent only from `nonce_at(Latest)` upward (flusher.rs:254, 265), so an original tx already mined gets no competing no-op. The 10-block wait (derive_timeouts, 62-66) happens only when the original or a zombie is mined between that read and the broadcast, or outbids the 3x-headroom bumped no-op. Expect about -15 to -20 lines rather than -25. +- Fix: Replace watch_txs with a bounded wait (same timeout, polling every safe_poll_interval) for `nonce_at(Latest) >= flush_end`. Delete map_watch_error, the hash vector, and the PendingTransactionConfig use. This adds no retry loop (the Do-not-simplify rule is respected): errors still propagate and the outer loop is unchanged. Gain: up to 10 blocks only in the race where the original lands, so it is justified as a simplification, not as a performance fix. + +### G12. Three guarded recovery mutations repeat the same transaction preamble + +`nit · partially · -20 lines · S` +Where: `sequencer/src/storage/recovery.rs:248-278`, `sequencer/src/storage/recovery.rs:282-306`, `sequencer/src/storage/recovery.rs:312-338` + +- Claim: The preamble is duplicated three times (storage/recovery.rs:253-260, 288-295, 318-325), but the proposed closure helper hides each method's commit point in safety-critical guarded mutations and nets only about -8 to -12 lines. A plainer helper, `begin_guarded(&mut self, protocol, now_ms) -> Result<(Transaction<'_>, RecoveryInspection), RecoveryMutationError>`, would open the Immediate tx, inspect, refuse divergence, and require the checkpoint. Each method would keep its own guard and an explicit `tx.commit()`. +- Fix: Add a private `guarded(&mut self, protocol, now_ms, body: FnOnce(&Transaction, RecoveryInspection) -> Result)` that commits on Ok, keeping each action-specific guard in its body. Pure refactor. + +### GS1. Cockroach flush throws away a surviving durable watermark + +`low · skeptic-added · +10 lines · S · BC · config` +Where: `sequencer/src/commands/setup/mod.rs:652`, `sequencer/src/commands/flush.rs:30`, `sequencer/src/storage/lifecycle.rs:141`, `sequencer/src/recovery/flusher.rs:160-172`, `docs/recovery/cockroach.md:246-250` + +- Claim: The flusher's only defense against zombie batch txs (dropped from the local pool but alive elsewhere) is the durable watermark anchor `safe >= watermark + 1`. `setup --recovery` reads the watermark from the freshly wiped DB, which is None on a first attempt. `flush-mempool` cannot be pointed at the surviving old data dir either: its preflight refuses on canonical divergence, which is the usual reason for a cockroach recovery. So even when the old DB is intact (every divergence-triggered rebuild), recovery falls back to the provider's pool view, and cockroach.md presents the watermark as unavailable ('The lost database cannot supply'). +- Fix: Do not change frozen setup. In the shared tool (G2), let the flush step take an optional read-only watermark from the old DB (`SELECT watermark FROM wallet_nonce_watermark`) and use the max of that and any fresh watermark. This is a stateless read with no new machinery. Invariant protected: every wallet slot the previous deployment may have broadcast is consumed at safe depth before C. Quantified: it matters only when a zombie exists at rebuild time, which is rare, but the existing fallback is a second full cockroach. Update cockroach.md:246 to say 'when the database is lost'. +- Decision: Should the shared tool's flush accept the old data dir (or its watermark value) as an optional input? + +Refuted: + +- G5. Flusher reads the safe nonce and C in two separate Safe-tag calls. No reachable scenario under the supported trust model. + +Keep (load-bearing): + +- RuntimeAdmission witness plus the final admit_runtime re-inspection (recovery/mod.rs:110-116, 374-386): preparation can outlast the L1 view's freshness; only the final check authorizes launch. +- Provider-failure fallback on the initial sync only, and none after the flush (recovery/mod.rs:267-276 vs 305): the cascade needs a view that has actually caught up to the flush observation. +- Flush observation kept only on the stack, so a retry flushes again (recovery/mod.rs:302-304): a persisted witness could go stale across restarts. +- Guarded Immediate transactions that re-inspect before mutating (storage/recovery.rs:248-338): clock-dependent facts can change between the decision and the mutation; StaleDecision is reachable. +- check_danger arm order and the two find_* helpers (storage/recovery.rs:388-427): on the Do-not-simplify list (I4). +- Unconditional post-flush cascade of the whole non-gold closed suffix (storage/recovery.rs:449-457): a convergence policy; keeping the suffix can re-enter the same danger cycle (pinned by recovery/mod.rs:843-894). +- Generation advance inside cascade_and_reopen (storage/recovery.rs:492-494): I18 offset reuse needs history identity. Freeze it until the new client sync is designed, but do not remove it. +- Watermark anchor `safe >= watermark+1` in the flush completion predicate (flusher.rs:203-208): I14, it covers zombie txs the local pool forgot. +- No internal retry loop in the flusher (flusher.rs:298-303): Do-not-simplify; respawn is the retry. +- RecoveryL1 trait (recovery/mod.rs:212-217): the one seam that lets procedure tests run real SQLite inspections and repairs. +- Detector as its own worker holding a ProcessLock clone (detector.rs:73-80, 165-178): blocking checks keep data-directory exclusivity after shutdown races. +- Checkpoint receipt agreement checks (setup/mod.rs:436-446) and fold.rs's always-on ordering assertions: cheap fail-loud guards on trusted inputs. They move to the tool rather than being deleted. +- Exhaustive retry/refuse classifiers with polarity tests (recovery/mod.rs:388-504, 536-641): they pin exit-code semantics for the supervisor. + +## H. Commands, config, errors, runtime, storage foundation, logging (cross-cutting) + +Verdict: This area holds the operator surface: clap config for setup, run and flush-mempool, the exit-code taxonomy (10/20/30/40/1 plus SIGABRT 134), the run bracket and worker supervisor, the process lock and shutdown scope, the storage open/lifecycle foundation, and the baseline schema. Its size is mostly proportionate. About 4,200 Rust prod lines plus 660 SQL lines sit in scope. error.rs is about 727 prod and 600 test lines. config.rs is about 607 prod and 490 test lines. In both, the bulk is doc comments (31% and 26% of prod lines) and the verdict table test, not machinery. The exit-code projection is the essential part: one exhaustive CommandFailureVerdict feeds both the black box and the exit code, and it should stay. The biggest problems are diagnostic, not structural. (1) Root causes get lost: the error types mix two Display conventions, and every sink prints only Display. (2) io::Error values at data-dir and lock boundaries reach the operator with no path. (3) Expected exits (10/20) and per-poll retryable provider errors log at ERROR, which breaks ERROR-based alerting. The one real config hazard is N16: MAX_WAIT_BLOCKS is a binary constant that the deployment identity does not pin. Freeze the setup --recovery CLI and error surface, since the shared recovery tool supersedes it. The schema's triggers and CHECKs mostly earn their keep because they enforce registered invariants at the engine layer. + + +### H1. Root causes vanish from exit, abort and black-box lines (two Display conventions, Display-only sinks) + +`medium · confirmed · +12 lines · S · BC` +Where: `sequencer/src/ingress/inclusion_lane/error.rs:17-44`, `sequencer/src/l1/watermark.rs:25-28`, `sequencer/src/egress/l2_tx_feed/error.rs:15-20`, `sequencer/src/egress/l2_tx_feed/error.rs:58`, `sequencer/src/harness.rs:129` + +- Claim: The claim holds. InclusionLaneError (error.rs:17-44), CatchUpError (error.rs:94-103), WalletNonceWatermarkError (watermark.rs:25-28) and SubscribeError/SubscriptionError (l2_tx_feed/error.rs:13-21, 57-67) use #[source] with a Display that omits the cause. Every sink prints only Display. Two details are missing. (a) WorkerExit (commands/error.rs:513-526) and WorkerStop::Source (error.rs:572-573) embed `{0}` without #[source], so source() stops at WorkerStop, and a chain-walker would also need those types changed. That makes embed-in-Display the consistent choice. (b) SubscribeError::OpenStorage also reaches abort_terminal through l2_tx_feed/mod.rs:92,105, which prints `preparing tx-feed subscription: cannot open subscription storage`. SubscribeError::Join embeds `{source}` while also marking it #[source] (l2_tx_feed/error.rs:23-27), so it would print twice under a walker. +- Fix: Use one convention. The cheap option is to append `: {source}` to the ~10 format strings that omit it, which matches the convention CommandError and BootstrapError already use. Add one test that a nested lane error's Display contains its io-level root. Example: before `ERROR terminal runtime failure; aborting process error=inclusion lane: snapshot at batch close failed`; after `... error=inclusion lane: snapshot at batch close failed: : No space left on device (os error 28)`. A chain-walking reporter is the alternative, but only if the embedded `{0}` is removed everywhere, otherwise causes print twice. +- Decision: Choose embed-in-Display (small) or source()+chain-walker (repo-wide). Recommend embed-in-Display. + +### H2. Data-dir, lock and sweep io::Errors reach the operator with no path + +`low · partially · +25 lines · S · BC` +Where: `sequencer/src/commands/error.rs:51-52`, `sequencer/src/commands/error.rs:633`, `sequencer/src/runtime/process_lock.rs:66-72`, `sequencer/src/commands/run/mod.rs:51`, `sequencer/src/commands/flush.rs:24` + +- Claim: Confirmed in code: CommandError::Io (error.rs:51-52) and ProcessLockError::Io (process_lock.rs:40-41) are transparent. create_dir_all (run/mod.rs:51, flush.rs:24), the lock open (process_lock.rs:66-72), try_exists (commands/mod.rs:42) and read_dir (startup_hygiene.rs:85-86) therefore surface bare OS messages and exit 1. The diagnostic loss is smaller than claimed. These are the first filesystem actions of a single-data-dir command, and the operator supplied the path through --data-dir or the env, so the path can almost always be inferred. Carrying the path is a cheap papercut fix, not a medium defect. +- Fix: Carry the path at these ~6 sites. Make ProcessLockError::Io a struct variant `{ path, source }`. Add a `DataDir { path, source }` CommandError variant for create_dir_all, try_exists and read_dir, classified Unclassified exactly like Io today. Example: before `ERROR sequencer exiting error=Permission denied (os error 13) exit_code=1`; after `ERROR sequencer exiting error=data-directory lock /var/lib/seq/sequencer.lock: Permission denied (os error 13) exit_code=1`. Add a row to the error.rs unclassified() table so the verdict stays pinned. + +### H3. Expected exits (10/20/40) and per-poll retryable provider errors log at ERROR + +`low · partially · +12 lines · S · BC` +Where: `sequencer/src/harness.rs:125-131`, `sequencer/src/l1/submitter/worker.rs:193`, `sequencer/src/l1/reader.rs:312-319`, `sequencer/src/runtime/shutdown.rs:17` + +- Claim: error! is used for every CommandError exit (harness.rs:129) and for each retried provider failure in the submitter (worker.rs:193, once per idle tick) and in the reader (reader.rs:312-319). The documented alerting contract, however, is the exit code (error.rs:61-73 projection header; docs/watchdog/operator-deployment.md), not log level, and the repo states no log-level policy. Exit 10 (danger detected) is an operationally significant event, not routine noise. The strongest part of the finding is the per-tick ERROR during an L1 outage, one line every 2-5 s. Downgrading retry logs to warn and adding a verdict field on the exit line is a reasonable low-priority cleanup. +- Fix: Pick the level from the verdict: Terminal and Unclassified log at error; ExpectedRecovery, Retryable and SetupRecoveryRequired log at warn. Add a `verdict` field that names the supervisor action. Downgrade the per-tick retry logs to warn. Add `exit = "SIGABRT(134): terminal, do not restart"` to the abort line. Example: before `ERROR sequencer exiting error=worker exited: danger detector: danger detected (TipInDanger(41)) — stopping for startup recovery exit_code=10`; after `WARN sequencer exiting for supervised restart verdict=ExpectedRecovery exit_code=10 error=...`. + +### H4. MAX_WAIT_BLOCKS is a binary constant the deployment identity does not pin (N16) + +`low · disputed · +60 lines · M · BC · db-schema` +Where: `sequencer-core/src/lib.rs:21`, `sequencer/src/commands/config.rs:74-81`, `sequencer/src/storage/mod.rs:239-246`, `sequencer-core/src/scheduler/mod.rs:33-38`, `examples/canonical-app/src/bin/canonical-app-devnet.rs:11` + +- Claim: MAX_WAIT_BLOCKS (sequencer-core/src/lib.rs:21) is compiled into both the guest (scheduler/mod.rs:33-38) and the sequencer (config.rs:74-81), and DeploymentIdentity does not pin it (storage/mod.rs:239-246). The hazard is real but generic. AGENTS.md:74 makes 'the same source compiled into the on-chain machine' the model, and any build skew in sequencer-core scheduler rules (staleness formula, fee math, DOMAIN_NAME) diverges the same way. Pinning one constant covers one vector of many. The constant has not changed since it was introduced (git log -S: only 2bd2e62). The proposed `setup --max-wait-blocks` flag is worse than the status quo. It turns a single compile-time source of truth into an operator input that the sequencer cannot check against the guest's per-binary constant, which adds a new divergence path at setup. If build-skew detection is wanted, pin a compiled protocol fingerprint (MAX_WAIT_BLOCKS, DOMAIN_NAME, a protocol version) at setup and refuse a mismatch at run, with no operator knob. +- Fix: Do not add a free runtime knob. Give it the same shape as the sequencer address: a per-deployment value. (1) Add a `SchedulerConfig::new(addr, max_wait_blocks)` constant per guest binary. (2) Add `setup --max-wait-blocks` (env CARTESI_SEQUENCER_MAX_WAIT_BLOCKS, default 1200), pin it in deployment_identity and include it in mismatch_fields. (3) In run, build ProtocolTiming after load_setup_identity from the pinned value, which moves run/mod.rs:57 below :64. Tests keep using the literal. +- Decision: Schema and identity change (ask-first area). Confirm guest images take it per-binary like the sequencer address. +- Independent trace: refuted. Every claim in the evidence holds: the constant is unpinned and the guest and the sequencer share it. + +### H5. Freeze the setup --recovery CLI, error and lifecycle surface; the shared recovery tool replaces it + +`medium · partially · +0 lines · S` +Where: `sequencer/src/commands/config.rs:286-331`, `sequencer/src/commands/config.rs:378-453`, `sequencer/src/commands/error.rs:388-434`, `sequencer/src/storage/lifecycle.rs:36`, `sequencer/src/storage/lifecycle.rs:112-119` + +- Claim: Freezing is right for the CLI and validation surface: OptionalKeyArgs (config.rs:286-331), the recovery fields and validate()/resolve_recovery_key (config.rs:378-453), and SetupRecoveryError. Those leave setup when the shared tool lands. The claim overreaches on the storage side. The tool must still produce the sequencer's startup DB, and that is exactly what LifecycleCommand::Rebuild plus complete_baseline_setup(recovery=true) do (initialize_for_command, open_recovery_tip_in_tx; lifecycle.rs:100-130). That layer probably moves behind the tool rather than being deleted, so 'resolves Overlapping admission policies by removal' is not assured. Plain setup's checkpoint_block detection scan and the exit-40 refusal also stay meaningful, because they point the operator at the tool. +- Fix: Do not polish any of it: no KeyArgs/OptionalKeyArgs unification and no SetupRecoveryError cleanup. When the tool lands, delete it from setup. That also resolves the 'Overlapping admission policies' register entry (Rebuild arms) by removal. Size in this area is about 200 prod lines plus about 150 test lines. + +### H6. L1/data-dir args are copied across three configs with identical docs, despite the module's own shared-group rule + +`low · partially · -35 lines · S` +Where: `sequencer/src/commands/config.rs:15-16`, `sequencer/src/commands/config.rs:339-356`, `sequencer/src/commands/config.rs:462-481`, `sequencer/src/commands/config.rs:547-567` + +- Claim: Confirmed: eth_rpc_url and allow_insecure_rpc, with an identical 5-line doc, are repeated in SetupConfig (config.rs:339-351), RunConfig (:464-476) and FlushConfig (:549-555), against the module's own shared-group rule (config.rs:15-16). The line delta is overstated. The duplication is about 13 lines × 3 = 39. A flattened L1RpcArgs plus three flatten sites costs about 23, for a net of roughly -15 to -20, not -35. FlushConfig's separate seconds_per_block is intentional, because flush does not take the rest of TimingArgs. +- Fix: Add `#[derive(Args)] struct L1RpcArgs { eth_rpc_url, allow_insecure_rpc }` and flatten it into all three configs. Leave data_dir in place, or move it into the same group. Flag names and env vars stay the same. + +### H7. Test-only code in production modules: set_alpha and complete_setup + +`low · partially · -40 lines · S` +Where: `sequencer/src/storage/admin.rs:15-34`, `sequencer/src/storage/migrations/0001_schema.sql:567`, `sequencer/src/storage/lifecycle.rs:145-171` + +- Claim: complete_setup is #[cfg(test)] (lifecycle.rs:150) and has only two test callers (:376, :438), so it can go. Only the divergence assertion (:375-378) should be repointed to complete_baseline_setup. The test at :436-440 checks complete_setup's own precondition, which is meaningless for complete_baseline_setup because that function creates the baseline itself, so delete that test rather than repointing it. set_alpha has no in-repo non-test caller, but it is a `pub fn` on Storage in `pub mod storage` (sequencer/src/lib.rs:30), so moving it into test_helpers is a public-rust-api change for downstream app crates, not a free move. Fixing the schema comment (0001_schema.sql:567) is uncontroversial. Net delta is about -30. +- Fix: Move set_alpha into storage/test_helpers and drop admin.rs. Reword the schema comment to 'constant; tests override'. Delete complete_setup and point its two tests at complete_baseline_setup. + +### H8. run/flush create a mistyped data dir, then refuse with a message that names no path + +`low · confirmed · +5 lines · S · BC` +Where: `sequencer/src/commands/run/mod.rs:51-55`, `sequencer/src/commands/flush.rs:24-27`, `sequencer/src/commands/mod.rs:42-44`, `sequencer/src/commands/error.rs:343-344`, `sequencer/src/storage/open.rs:51-58` + +- Claim: run (run/mod.rs:51-55) and flush (flush.rs:24-27) create the data dir and sequencer.lock before preflight_lifecycle_command finds no DB (commands/mod.rs:42-43). They then exit 30 with SetupNotComplete, whose Display names no path (error.rs:343-344). The open.rs rule cited (open.rs:51-58) concerns DB files, so this is a spirit-level inconsistency, not a violation. The proposed existence check before the lock is a read ahead of the lock, which the run/mod.rs:52-54 comment says to avoid. It is benign, though: absence refuses, and presence proceeds to take the lock. +- Fix: Only setup creates the data dir. In run and flush, a missing dir maps to SetupNotComplete before the lock is taken. Give SetupNotComplete a `data_dir` field shown in Display. Exit code stays 30. + +### H9. The terminal_faults black box records only causes that re-log anyway + +`low · partially · -150 lines · M · BC · db-schema` +Where: `sequencer/src/storage/migrations/0001_schema.sql:447-482`, `sequencer/src/storage/lifecycle.rs:70-76`, `sequencer/src/storage/lifecycle.rs:173-231`, `sequencer/src/commands/mod.rs:50-113`, `sequencer/src/commands/setup/mod.rs:380-402` + +- Claim: The black box captures only terminal verdicts returned through a command bracket (commands/mod.rs:57-79). In run, terminal worker faults abort instead (workers.rs:375, 458), so most recorded causes are startup-phase and are re-derived on the next boot. This is ADR mechanism 2, registered as policy in docs/invariants.md:32-43, and the ADR itself states the limitation (2026-08-authority-boundary-adr.md:88-91). The finding adds no new evidence beyond the ADR's accepted trade. The 'set looks empty' claim is also overstated. Config-derived terminal verdicts (ChainIdMismatch or FeeOracleMisconfig in setup, SignerMisconfig at workers.rs:653) are not re-derived once the operator changes the config. The box also has a live consumer, the boot echo (run/mod.rs:62). Leave it to the maintainer, and note HS1, a concrete defect in that consumer. +- Fix: Either keep it as an accepted cost (it is ADR-recorded) or delete it and let the exit code plus logs carry the cause, which H1-H3 make reliable. Quantify first: count how many terminal verdicts are not re-derived on the next boot. Today that set looks empty. +- Decision: ADR mechanism 2. Maintainer decides keep vs delete; do not change it silently. + +### H10. Startup event omits the timing knobs; orphan-sweep deletions are not audited + +`low · confirmed · +8 lines · S · BC` +Where: `sequencer/src/commands/run/mod.rs:138-151`, `sequencer/src/commands/run/startup_hygiene.rs:21-25`, `sequencer/src/commands/run/startup_hygiene.rs:98-99` + +- Claim: The startup event (run/mod.rs:138-151) omits max_batch_open_seconds, l1_read_stale_after_blocks, seconds_per_block and confirmation depth. The orphan sweep logs successful deletions only as a debug count (startup_hygiene.rs:20-24, 98-99). Per-path info logging matters most for the sweep, because of its known path-spelling hazard (startup_hygiene.rs:64-65: 'setup and run may spell the same directory differently'). The eth_rpc_url removal belongs to the open register entry, not to this finding. +- Fix: Add those fields to the startup event and drop eth_rpc_url (see the register entry). Log each orphan deletion at info with its path: one line per destructive action. + +### H11. batch_submitter_idle_poll_interval_ms accepts 0 and turns the idle submitter into an RPC hot loop + +`low · confirmed · +1 lines · S · BC · config` +Where: `sequencer/src/commands/config.rs:485-491`, `sequencer/src/l1/submitter/worker.rs:198-202`, `sequencer/src/l1/submitter/worker.rs:219-222` + +- Claim: batch_submitter_idle_poll_interval_ms (config.rs:485-491) has no range parser, unlike poll_interval_ms (config.rs:112) and max_batch_open_seconds (:509). With 0, the idle and transient arm (submitter/worker.rs:198-201) sleeps for zero and re-runs load_frontier plus observed_submitted_batch_nonces RPCs back-to-back. +- Fix: Add `value_parser = clap::value_parser!(u64).range(1..)`. + +### H12. Danger-exit terminality is defined twice (abort decision vs exit verdict) + +`nit · unverified-nit · -4 lines · S` +Where: `sequencer/src/commands/error.rs:201-225`, `sequencer/src/commands/error.rs:556-561` + +- Claim: WorkerExit::is_terminal hard-codes `CanonicalDivergence | Safe` for DangerDetected, and danger_failure_verdict separately maps the same statuses to Terminal. The abort decision (workers.rs:373) and the exit code could drift apart. +- Fix: Write `WorkerExit::DangerDetected { status } => danger_failure_verdict(status).is_terminal()`. + +### H13. Comments restate code, repeat each other, or carry change history + +`low · partially · -120 lines · S · docs-only` +Where: `sequencer/src/commands/error.rs:74-90`, `sequencer/src/commands/error.rs:489-495`, `sequencer/src/commands/config.rs:368-377`, `sequencer/src/storage/migrations/0001_schema.sql:21-26`, `sequencer/src/storage/migrations/0001_schema.sql:117-120` + +- Claim: The comment-density figures are correct: 228 of 727 prod lines in error.rs, 159 of 607 in config.rs. The EXIT_SETUP_NEEDS_RECOVERY doc (error.rs:81-89) and the 6-line clippy rationale (error.rs:489-494) are trimmable. Two examples are mischaracterized. The config.rs:369-376 'PR3' sentence is history-flavored wording, but not stale, since it points at `setup --recovery` as the non-genesis path. The schema's 'a CHECK on it once wedged batch close' (0001_schema.sql:22-24) is the rationale for a deliberate absence, a do-not-add guard. Keep the reason and drop only the anecdote. Do the --recovery-adjacent text last or skip it, given the H5 freeze. +- Fix: Trim to the invariant and the why: one line for the Box rationale, one line per exit constant, delete the PR3 sentence, and leave history to git. Do one docs-only pass after H1-H3 so the trimmed text describes the final behavior. + +### H14. Schema seed uses INSERT OR IGNORE against the fail-loud check policy + +`nit · unverified-nit · +0 lines · S · db-schema` +Where: `sequencer/src/storage/migrations/0001_schema.sql:601`, `docs/invariants.md:29-31` + +- Claim: The check policy forbids `INSERT OR IGNORE`. The batch_policy seed uses it inside a migration that rusqlite_migration runs exactly once, so OR IGNORE can only mask a duplicate-seed bug. +- Fix: Use a plain INSERT. + +### HS1. Boot-time black-box echo warns forever about a fault the operator already fixed + +`low · skeptic-added · +6 lines · S · BC` +Where: `sequencer/src/commands/run/mod.rs:62`, `sequencer/src/commands/mod.rs:84-113`, `sequencer/src/storage/lifecycle.rs:196-229`, `sequencer/src/storage/migrations/0001_schema.sql:470-482`, `sequencer/src/commands/setup/mod.rs:359-376` + +- Claim: Every `run` boot logs a WARN for the newest terminal_faults row. The table is append-only and nothing marks a fault as resolved. A terminal setup failure on a fresh data dir (for example, a typo'd --chain-id) is therefore echoed as a WARN on every healthy `run` boot for the life of the database, even after a corrected setup retry completes. +- Fix: Have the black box's one consumer say only what is still news. Skip, or log at debug, any fault whose recorded_at_ms predates setup_complete.completed_at_ms, because a later completed setup proves a setup-era fault was resolved. Include the fault's age in the remaining WARN. If H9 deletes the black box, this goes with it. +- Decision: Tied to H9: fix only if the black box is kept. + +Keep (load-bearing): + +- CommandFailureVerdict as the single classification behind both the exit code and the black box (commands/error.rs:95-195): one exhaustive match, so a new error shape cannot get one class in the logs and another at the supervisor. +- The exit-code table tests (commands/error.rs:806-1292): about 480 lines, but each row pins one supervisor-facing integer per failure shape. This is the wire contract for orchestrators, and the distinct-codes test makes code pins equal verdict pins. +- Per-worker is_terminal_invariant beside each error enum plus the generic WorkerStop (commands/error.rs:529-609): a new variant fails to compile in its own module instead of falling through a distant wildcard. +- is_persistent_storage_error failing open toward operational (storage/convert.rs:121-175): a wrong 'terminal' pages for a self-healing condition, while a wrong 'operational' only restarts. +- SYNCHRONOUS_PRAGMA = FULL and its comment (storage/open.rs:16-32): it protects acknowledged chunk commits, the write-before-broadcast watermark and setup completion against power loss. The comment names exactly what relaxing it would break. +- Storage::open refusing a missing DB file (storage/open.rs:49-72): database absence means uninitialized, which prevents an ownerless schema on a mistyped path. H8 extends the same rule to the directory. +- The kernel process lock as Arc clones plus spawn_blocking_with_lock (runtime/process_lock.rs): exclusivity outlives cancelled async futures that detach blocking DB work, and a persisted row cannot provide that. +- abort_terminal and the panicking ShutdownOnDrop abort (runtime/shutdown.rs:16-19; commands/run/workers.rs:102-111): a terminal runtime fault exits without draining over corrupt state. +- refuse_divergence_on_clean_exit (commands/run/run/mod.rs:96-102 path: sequencer/src/commands/run/mod.rs:96-102): it closes the exit-0 hole where a clean drain outruns divergence re-detection. +- The Workers exhaustive destructures and concurrent drain (commands/run/workers.rs:323-473), already upheld by the jury: they make a forgotten worker a compile error and keep a hung component from hiding a terminal exit. +- Schema invariant triggers and typeof guards (0001_schema.sql:27-33, 110-190, 360-370, 394-445, 526-532, 653-660): they enforce I15 (divergence freeze), I16 (single anchored root), write-once stamps and history immutability at the engine layer. That is cheap, multi-writer-safe defense for a store where divergence equals theft. +- Secret handling at the clap edge (commands/config.rs:249-254, 588-592; hide_env_values on key and RPC args): the key never reaches Display or a log line, even on read failure. +- The TTY-gated ANSI init in examples/wallet-sequencer/src/main.rs:10-18: it keeps escape codes out of journald and file logs (the earlier CI-red cause). + +## I. Snapshots, dumps, checkpoint metadata + +Verdict: This area turns each batch close into an immutable app-state artifact. The order is: the file is made durable, then one SQLite transaction seals the batch and registers the artifact. Selection is derived from acceptance: restart uses the newest valid batch, the watchdog and recovery use the newest accepted batch. GC deletes rows before files, and a lease lets HTTP serve the watchdog's comparison file and the operator's recovery archive. The core is correct and fails loud where it should. Selection never falls back to an older snapshot, GC keeps every valid snapshot beyond the accepted frontier, the divergence check shares one transaction with selection and lease, and the archive never reports a truncated success. I found no correctness bug. The size is about 1,330 production lines across the five in-scope files, plus a 65-line release supervisor in http.rs. That is roughly 30-40% more than the essential complexity. The excess sits in three places: 1. **Leases:** a lease is a process-local reference count, but it is stored durably in SQLite. That needs egress writer connections, a release supervisor task, a persistent-failure reporter and a startup reset (about 120 lines). 2. **`info.toml`:** the on-disk file repeats what SQLite already knows, and the export receipt repeats it again. It costs one extra file and 2 extra fsyncs per batch close (about 70 lines). 3. **Snapshot-plus-subscribe bootstrap:** `/latest_snapshot` and the history-claim headers serve a client bootstrap that the future direction replaces. They should be frozen, not polished (about 140 production and 250 test lines). Per-batch snapshots (N23) should stay. The cost is one lane stall per batch, and batches are paced by L1 posting. The real risk is a large native state on a filesystem without reflinks, which is a deployment requirement and needs no new policy. Sparse snapshots would trade that cost for a liveness failure in startup, recovery and the watchdog. For N3, the sequencer needs nothing from machine-emulator PR #398: the SQLite row is a stronger publish point than an atomic rename, and the watchdog has already adopted `rename_stored` and `remove_stored`. + + +### I1. On-disk info.toml duplicates SQLite and the export receipt; drop it and generate it inside archives + +`medium · confirmed · -70 lines · S · BC · docs-only` +Where: `sequencer/src/ingress/inclusion_lane/dump_info.rs:38-60`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:93-101`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:136-149`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:156-205`, `sequencer/src/ingress/inclusion_lane/snapshot.rs:51` + +- Claim: info.toml holds only `format_version` and `next_batch_nonce`. Every server path already takes the nonce from SQLite, and the recovery receipt is a strict superset of the file. The file adds an unpublished-directory temp+rename and 2 fsyncs to every batch close, and it versions only itself, not the app bytes. +- Fix: 1. Make the dump dir just mkdir + `app.create_dump(dir/state)` + parent fsync. 2. In `write_archive`, generate `info.toml` from the SQLite nonce it already receives, the same way it generates `checkpoint.toml` (dump_info.rs:108-117). The archive bytes and the recovery loader stay unchanged. 3. Replace the startup parse with a stat of `state`, or drop it. 4. Leave merging info.toml into a single receipt to the shared recovery-tool design, which will own the export format. If the file is kept instead, at least drop the temp+rename and the directory fsync in `write_info`. +- Decision: Should startup keep a cheap stat-based presence check of the rollback artifact, or rely on catch-up's terminal `from_dump` classification? Archive format stays unchanged either way. + +### I2. Download leases are stored durably in SQLite but are process-local; replace with an in-memory registry + +`medium · confirmed · -120 lines · M · db-schema` +Where: `sequencer/src/storage/snapshot_dumps.rs:46-97`, `sequencer/src/storage/snapshot_dumps.rs:114-131`, `sequencer/src/storage/snapshot_dumps.rs:182-244`, `sequencer/src/storage/snapshot_dumps.rs:431-451`, `sequencer/src/http.rs:160-206` + +- Claim: A lease only means something while this process serves a body, and startup zeroes every count. Even so, each download does 2 SQLite write transactions from egress. Release goes through an injected scheduler, a supervisor task with a JoinSet, and a persistent/transient error classifier that aborts the process. All of that exists to decrement a counter. +- Fix: Share `DumpLeases(Arc>>)` between egress and the lane's GC. - **Acquire:** lock, run the read-only selection transaction (divergence check included), increment, unlock. - **Release:** the guard's Drop locks and decrements. It is infallible and does no I/O. - **GC:** lock, select candidates inside its write transaction, skip leased ids, delete, unlock. - **Lock order:** registry before SQLite. Egress never takes the SQLite writer lock, so this cannot deadlock. - **Invariant protected:** a leased artifact's row and files are never deleted. - **Assumption:** only the serving `run` process collects. Revisit if a second process ever serves or collects dumps. This removes the column, the startup reset, the release supervisor and the reporter wiring, and egress becomes SQLite read-only. +- Decision: Approve dropping `dumps.lease_count` and adopting the single-serving-process assumption. + +### I3. /latest_snapshot and the history-claim headers serve the snapshot-plus-subscribe bootstrap the new sync replaces: freeze + +`medium · confirmed · -140 lines · S · BC · http-api` +Where: `sequencer/src/egress/api/snapshot.rs:195-202`, `sequencer/src/egress/api/snapshot.rs:392-410`, `sequencer/src/egress/api/snapshot.rs:183-187`, `sequencer/src/egress/api/snapshot.rs:268-276`, `sequencer/src/storage/snapshot_dumps.rs:99-106` + +- Claim: `/latest_snapshot` and the `X-History-Era`, `X-Recovery-Generation` and `X-Executed-Input-Count` headers exist so a client can restore the app dump and subscribe with a history claim. The future direction drops that assumption: clients hold their own SQLite state and get startup state from the shared recovery tool. +- Fix: Freeze until the new client sync lands, then delete: - the route - `acquire_latest_snapshot_lease` and `snapshot_next_nonce_in` - `LeasedDump.history_version` and the history headers on all snapshot routes - the SDK method and the openapi entries Keep `Storage::latest_snapshot()`, which catch-up uses (inclusion_lane/catch_up.rs:22-31). Don't polish in the meantime (e.g. the redundant `snapshot_next_nonce_in` re-query). +- Decision: Confirm `/latest_snapshot` and the history headers are frozen until the replacement sync exists. + +### I4. ETag/If-None-Match 304 on /finalized_state has no consumer + +`low · confirmed · -10 lines · S · BC · http-api` +Where: `sequencer/src/egress/api/snapshot.rs:150-169`, `sequencer/src/egress/api/snapshot.rs:436-441`, `openapi.yaml:531-560`, `README.md:456` + +- Claim: The watchdog idle-polls `/finalized_state/inclusion_block` and compares `/digest`. It never sends `If-None-Match`, so the conditional-GET path and its contract surface are unused. +- Fix: Remove the ETag header, `if_none_match` and the 304 branch, plus the matching openapi/README text and the integration test. +- Decision: Drop the conditional-GET feature from the operator API? + +### I5. The per-batch snapshot stalls the lane without any timing recorded; no revisit trigger exists for large native state + +`low · partially · +5 lines · S` +Where: `sequencer/src/ingress/inclusion_lane/snapshot.rs:41-60`, `sequencer/src/ingress/inclusion_lane/mod.rs:152-167`, `sequencer/src/storage/snapshot_dumps.rs:133-160` + +- Claim: `create_dump` runs synchronously and untimed on the lane at every close (snapshot.rs:41-60; mod.rs:152-167). That part is true. But closes are not paced by L1 posting. They fire when the fee-policy byte target is reached (`batch_size_target` → `max_batch_user_op_bytes`, storage/mod.rs:354, 371-377; lane mod.rs:435) or after `max_batch_open_seconds` (config.rs:501-511). Under load, several closes can happen per L1 block, so the stall scales with throughput ÷ target size. The register already tracks this: 'representative deployment latency, including checkpoint creation' sits under Track 3 follow-ups (register.md:267-269). Add the elapsed-time field and hang the reflink requirement and revisit trigger on that item instead of adding a duplicate entry. +- Fix: - Log `create_dump` elapsed time at batch close (one debug field). - Add a register 'Known optimizations' entry: the stall is per batch, mitigated by a reflink-capable filesystem (XFS reflink=1, btrfs, APFS) for large-state engines. - Revisit trigger: measured `create_dump` p99 above about 250 ms, or retained-dump disk over budget on deployment storage. This needs no new mechanism. +- Decision: Accept 'reflink-capable filesystem for large-state native engines' as a deployment requirement. + +### I6. Snapshot stream failures after the 200 are never logged; error messages omit the conflicting values and the path + +`low · partially · +10 lines · S` +Where: `sequencer/src/egress/api/snapshot.rs:253-263`, `sequencer/src/egress/api/snapshot.rs:316-332`, `sequencer/src/egress/api/snapshot.rs:443-446`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:76-80`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:94-98` + +- Claim: Non-terminal mid-stream errors are not logged: the producer only forwards them through `done_tx` (snapshot.rs:253-263), and `ComparisonReader` returns them (snapshot.rs:327-330). tower-http 0.6.11's `TraceLayer::new_for_http` does not log body errors for HTTP classification, because `classify_eos` is None (tower-http trace/future.rs:74-88, body.rs:82-88). But a warn in the producer's non-terminal branch would fire on every client disconnect: once the body is dropped, the duplex writer fails with BrokenPipe (snapshot.rs:240-263). So skip broken-pipe/disconnect errors or log them at debug. The missing path (snapshot.rs:191, 443-446) and the missing values (dump_info.rs:76-80, 94-98) are accurate, although the nonce-mismatch error is InvalidData, so it is logged via `abort_terminal` with the prefix (snapshot.rs:254-258). It only lacks the 2 nonces. +- Fix: - Log a warn with the prefix and error in the producer's non-terminal branch and in `ComparisonReader`'s error map. - Pass the path into `internal_error` for open failures. - Include the expected and found values in the 2 messages. + +### I7. Three AsyncRead wrappers where two suffice + +`low · partially · -20 lines · S` +Where: `sequencer/src/egress/api/snapshot.rs:281-287`, `sequencer/src/egress/api/snapshot.rs:297-302`, `sequencer/src/egress/api/snapshot.rs:412-432` + +- Claim: `GuardedReader` is pure delegation (snapshot.rs:417-432), and the `Arc` in `stream_body` is unneeded (snapshot.rs:300). For archives, though, the body-side guard is redundant, not just the `GuardedReader`. All source-file reads happen inside `write_archive` on the producer thread, which holds its own guard (snapshot.rs:244-252). After it returns, the remaining bytes sit only in the 64 KiB duplex buffer. Simpler shape: the producer owns the `LeaseGuard` by value, `ArchiveReader` carries no guard, `ComparisonReader` owns its `LeaseGuard`. Delete `GuardedReader` and both `Arc`s, and update lifecycle.md:129-131. +- Fix: Add `_guard` fields to `ComparisonReader` (`LeaseGuard`) and `ArchiveReader` (`Arc`), then delete `GuardedReader`. This composes with I2. + +### I8. Snapshot docs and comments restate code, narrate history, or contradict the code + +`low · partially · -40 lines · S · docs-only` +Where: `docs/snapshots/lifecycle.md:38`, `docs/snapshots/lifecycle.md:133-148`, `docs/snapshots/README.md:3-13`, `docs/snapshots/format.md:62-65`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:174-183` + +- Claim: These verbosity points hold: - lifecycle.md:38 says "Initially". - lifecycle.md:139-145 restates the inode sweep that the code comments already own (startup_hygiene.rs:66-69, 112-115). - format.md:62-65 narrates a test. - dump_info.rs:174-183 restates `deny_unknown_fields` and the version gate. - dump_info.rs:505-507 narrates "the old line parser". - snapshot.rs:428 explains Unpin soundness the compiler already enforces. The `make_dump_dir` 'contradiction' is overstated. 'Opaque' there means the code never parses the name (snapshot.rs:62-67), and the nonce is a human aid. Reword the comment, which must change anyway if I1 lands because it cites info.toml. Renaming dirs by batch_index is churn without benefit. Folding README.md into lifecycle.md is optional: README is the directory index, and its paragraph at lines 8-13 is not duplicated elsewhere. +- Fix: - Cut the device/inode paragraph to one sentence that links to the code. - Replace "Initially" with "Every". - Fold README.md into lifecycle.md's opening. - Drop the narrated test and history comments. - Name dumps by `batch_index`, the real snapshot identity, plus nanos, or drop the "opaque" claim. + +### I9. The wallet nests its single file as state/state; the reference engine should write its file at the prefix + +`nit · partially · -8 lines · S · BC · http-api` +Where: `examples/app-core/src/application/wallet.rs:362-391`, `docs/snapshots/format.md:12-22` + +- Claim: The wallet does nest one file as `state/state` (wallet.rs:367-391; format.md:21-22), and the contract permits a file at the prefix (application/mod.rs:253-254; application-contract.md:217-218). The gain is 1 mkdir and 1 directory fsync per close, plus an exemplar. The contract text and the N3 recipe already give libdex that pattern. The change breaks existing wallet bundles, touches the genesis/`--state-file` wiring (tests/harness/src/sequencer.rs:1398, 1448), the C wallet engine that delegates to `WalletApp` (examples/c-wallet-engine/src/lib.rs:407), and the wallet test at wallet.rs:1058. Do it only if the toy wallet is meant to be the reference engine pattern. +- Fix: Write the SSZ file at `prefix` and have `state_file_in_dump` return the prefix itself. Update format.md. Ship it as its own patch, because existing wallet recovery bundles carry the nested layout. +- Decision: Change the toy wallet's dump layout? This breaks existing wallet recovery bundles. + +Keep (load-bearing): + +- File created and fsynced before the SQLite row, row deleted before the files, and the split between storage/snapshot_dumps.rs (SQLite only) and the filesystem halves. The module boundary is the crash-ordering guarantee (I13, do-not-simplify). +- Exact-snapshot selection with no fallback to an older artifact (snapshot_dumps.rs:162-163, 349-356). A missing artifact is corruption and fails loud (I7). +- GC keeps every valid snapshot beyond the accepted frontier (snapshot_dumps.rs:133-160). An intermediate batch can become the next required checkpoint (do-not-simplify). +- The canonical-divergence check runs in the same transaction as selection and lease (snapshot_dumps.rs:371-375). A matched batch that precedes a divergent one in the same block must not be advertised. +- Only genesis baselines are comparable (snapshot_dumps.rs:385-400). A rebuilt baseline may contain the fold's speculative drain. +- Leases as a concept. Acceptance advancing mid-download lets GC delete the finalized artifact, and NotFound is classified terminal, so the lease prevents a benign race from aborting the process. Only the storage of the lease is over-built (I2). +- `referenced_artifact_io_is_terminal` (dump_info.rs:27-36) and `comparison_io`. They separate a corrupt referenced artifact (terminal) from an operational I/O error (retry). +- `ArchiveReader` turning a producer failure into a stream error (snapshot.rs:334-369). Without it a truncated archive would pass as a successful 200. +- `catch_unwind` around the foreign `state_file_in_dump` callback (snapshot.rs:291-295). Request-task panics would otherwise be swallowed. +- The receipt is generated at export, never stamped into the immutable dump (dump_info.rs:85-117). Artifacts stay immutable. +- The device/inode identity in the orphan sweep (startup_hygiene.rs:72-115). It fixed a path-alias data-loss bug. +- `diagnose_missing_dump`'s watchdog-layout hint (dump_info.rs:207-252). It is cheap and aimed at the most likely operator mistake under recovery pressure. +- Unconditional per-batch snapshots (N23). They are paced by L1 posting, and sparse snapshots would turn a cost into a liveness failure. + +## J. Example apps, canonical app, binaries + +Verdict: This area holds the placeholder wallet (app-core), the two Rust wallet binaries, the canonical RISC-V guest shell, the C-seam host (c-app-sequencer), the reference C-ABI engine (c-wallet-engine, which wraps WalletApp in unsafe FFI and adds a genesis tool), and the guest test suite (canonical-test). It is mostly proportionate. The 1,080-line wallet.rs is about 400 lines of app and 680 of tests. The app part is fine for a placeholder. The tests are where the excess sits: some duplicate sequencer-core guard tests, some are trivia, and one keeps a "legacy" SSZ type alive just to test an unknown selector. The Rust wallet and the C engine share no business logic: the engine delegates every call to WalletApp, and a native-vs-bridge conformance test checks the two agree. Keep both. The biggest problems are about what an app author would copy: - Anvil test private keys and their parser live in the app crate and are part of its public API (J1). - App state uses HashMap, so canonical ordering depends on remembering to sort in the encoder (J5). - There are two different genesis models. Rust picks a config per binary; C uses a genesis tool plus a `--state-file` flag that actually names a directory (J2, J3). - Tracing setup is copied three times, and the harness doc example calls an `init_tracing()` that does not exist (J4). The genesis plumbing should be frozen, not polished: the planned shared recovery/startup tool replaces it. Things to keep: the engine-delegates-to-WalletApp shape, the fsync'd create_dump, the typed error on incoherent snapshot progress, the golden snapshot vector shared with Lua, and the guest-side fee-arithmetic test. + + +### J1. Anvil test-key list and its validating parser live in the app crate's public API + +`medium · partially · -120 lines · S` +Where: `examples/app-core/src/application/anvil_accounts.rs:1-87`, `examples/app-core/anvil_accounts.txt:1-100`, `examples/app-core/src/application/mod.rs:9`, `tests/harness/src/rollups.rs:380`, `tests/harness/src/rollups.rs:449` + +- Claim: app-core publicly re-exports `default_private_keys` (examples/app-core/src/application/mod.rs:9). Its only callers are tests/harness (rollups.rs:380, :449, :495; sequencer.rs:1368; wallet.rs:46) and tests/benchmarks (workload.rs:69, :187, :311, :369). parse_accounts validates the address column and then discards it (anvil_accounts.rs:49-51), and 32 test lines (88-119) cover a parser of a fixed, checked-in file. The keys can move to tests/harness: benchmarks already depends on rollups-harness (tests/benchmarks/Cargo.toml:13). The mnemonic option is not free, though. The harness enables only alloy's `signer-local` feature (tests/harness/Cargo.toml:14), so it would need `signer-mnemonic` added. The cheapest move is to keep the .txt file under tests/harness and take column 2 with a three-line split. Also update the doc pointer at wallet.rs:83-84. +- Fix: Move the keys to tests/harness as a `const DEFAULT_PRIVATE_KEYS: &[&str]`, or derive them from Anvil's well-known mnemonic with alloy's mnemonic builder, which the harness already depends on. Delete the parser and its tests, and drop the re-export from app-core. + +### J2. Two genesis models (per-binary preset vs genesis tool + --state-file): freeze, the shared startup-state tool replaces both + +`low · partially · -150 lines · M · BC · config` +Where: `examples/wallet-sequencer/src/main.rs:1-24`, `examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs:1-24`, `examples/c-wallet-engine/src/bin/c-wallet-genesis.rs:1-38`, `examples/c-wallet-engine/tests/genesis_cli.rs:1-17`, `examples/c-wallet-engine/src/lib.rs:420-427` + +- Claim: There are three per-network genesis selections. The two wallet-sequencer binaries differ only at line 23 (`default()` vs `devnet()`). The guest has its own pair, canonical-app-devnet.rs and canonical-app-sepolia.rs. The C path uses c-wallet-genesis (21-29) plus --state-file (c-app-sequencer lib.rs:50-58). Freezing them is right: do not add a --preset flag. The deletion plan overreaches, though. The future tool consumes a *supplied* checkpoint, and something app-specific still has to produce the genesis checkpoint. c-wallet-genesis.rs:4-5 and c-wallet-engine lib.rs:422-427 state that the seam has no create path, so every app ships a genesis tool. The conformance fixture also uses write_genesis (conformance.rs:21). So c-wallet-genesis and write_genesis survive any version of the direction. At most, the Rust devnet binary and the run_main/run_command genesis-closure plumbing (about 30-40 lines) would go, and only if setup becomes dump-only. The guest pair cannot go, because the machine image bakes its config in at build time. +- Fix: Freeze. Do not add a `--preset` flag to wallet-sequencer or polish the genesis tool. When the shared tool lands, make setup consume a tool-produced startup dump for both Rust and C apps. That deletes wallet-sequencer-devnet, c-wallet-genesis plus its CLI test, and the genesis closure plumbing. +- Decision: Should genesis become just 'a checkpoint produced by the shared tool' (one startup-state input for setup), and should that tool bind the app's deployment preset to the chain id setup pins? + +### J3. `--state-file` / CARTESI_SEQUENCER_STATE_FILE names a dump prefix (a directory), not the state file + +`low · confirmed · +0 lines · S · config` +Where: `bindings/c-app-sequencer/src/lib.rs:27-30`, `bindings/c-app-sequencer/src/lib.rs:57`, `tests/harness/src/sequencer.rs:1398`, `tests/harness/src/sequencer.rs:1448`, `bindings/c-app-engine/README.md:43-44` + +- Claim: `--state-file` / CARTESI_SEQUENCER_STATE_FILE (c-app-sequencer lib.rs:27-30) is passed to `EngineApp::from_dump` (lib.rs:57), so it names a dump prefix, while `state_file_in_dump` means the canonical file inside a dump (sequencer-core application/mod.rs:252-256; wallet.rs:389-391). The harness ends up with /state/state (sequencer.rs:1398, 1448), and the README repeats the usage (bindings/c-app-engine/README.md:43-44). Because the genesis tool survives the future direction (see J2), the rename does not need to wait for that tool. +- Fix: Rename to `--genesis-dump` / CARTESI_SEQUENCER_GENESIS_DUMP in the same patch that lands J2's tool. Do not rename it on its own before then. +- Decision: Rename now, or fold into the J2 startup-tool change? + +### J4. Tracing init copied into every app binary; harness doc example calls a nonexistent init_tracing() + +`low · partially · -22 lines · S · public-rust-api` +Where: `examples/wallet-sequencer/src/main.rs:10-19`, `examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs:10-19`, `bindings/c-app-sequencer/src/lib.rs:43-48`, `sequencer/src/harness.rs:7-18` + +- Claim: The ANSI-on-TTY tracing block appears three times: wallet-sequencer main.rs:14-19, the devnet bin at 14-19, and c-app-sequencer lib.rs:43-48. The comment appears twice. The harness.rs:12-17 example calls an `init_tracing()` that exists nowhere; git grep finds only the doc line, and the block is `ignore`d, so it never compiles. The proposal is more costly than stated. The sequencer library has no tracing-subscriber dependency (only examples/wallet-sequencer/Cargo.toml:17 and bindings/c-app-sequencer/Cargo.toml:21 have one), so exporting init_tracing from the library adds a dependency and a process-global subscriber policy. Calling it inside run_command would double-initialize for c-app-sequencer, which already calls `.init()`, and `.init()` panics when a global subscriber is already set. Smallest fix: make the doc example show the real block, and, if wanted, add `harness::init_tracing` only under run_main's binary-facing path. Net about -10 lines, not -22. +- Fix: Export `sequencer::harness::init_tracing()`, or call it inside run_main/run_command after parsing (as c-app-sequencer already orders it). The Rust app's main then shrinks to one line, and the doc example becomes true. + +### J5. Deterministic app state uses HashMap; canonical order depends on remembering to sort in the encoder + +`low · partially · -45 lines · S` +Where: `examples/app-core/src/application/wallet.rs:57-63`, `examples/app-core/src/wallet_snapshot.rs:46-63`, `examples/app-core/src/wallet_snapshot.rs:83-102`, `examples/app-core/src/application/wallet.rs:996-1036` + +- Claim: WalletApp keeps HashMaps (wallet.rs:60-61), and canonical bytes depend on the sorts at wallet_snapshot.rs:54 and :63. A BTreeMap swap preserves the bytes: Address is FixedBytes<[u8;20]>, whose derived Ord is the same byte-lexicographic order as the current sort key. It also removes the sorts and both duplicate checks. Three corrections. (1) The golden vector (wallet_snapshot.rs:139-143) encodes the EMPTY wallet, so it does not guard ordering; byte equality has to be argued, or pinned with a non-empty vector. (2) The scrambled-order test (wallet.rs:996-1036) is vacuous today. It dumps the same unmutated HashMap twice, and an unmutated HashMap iterates in the same order both times, so the test would pass with the sort removed. That strengthens the case for deleting it. (3) Requiring strictly increasing addresses in decode tightens what decode accepts: unique but unsorted entries would be rejected. No encoder writes such bytes, but it is still an acceptance change for corrupt inputs. The 'RandomState seeds from randomness in the guest' point does not matter, because no unsorted iteration reaches output. +- Fix: Switch to BTreeMap. Address's Ord is byte-lexicographic, which matches the current sort key, so the snapshot bytes do not change (the golden vector at wallet_snapshot.rs:139-143 guards this). Drop the sorts. In decode, require strictly increasing addresses, which replaces both duplicate checks and makes decode∘encode a bijection. Delete the scrambled-order test. + +### J6. wallet.rs test module duplicates sequencer-core guard tests and carries trivia, while the register's business-failure gap stays open + +`medium · confirmed · -170 lines · S` +Where: `examples/app-core/src/application/wallet.rs:420-479`, `examples/app-core/src/application/wallet.rs:541-549`, `examples/app-core/src/application/wallet.rs:551-609`, `examples/app-core/src/application/wallet.rs:812-823`, `examples/app-core/src/application/wallet.rs:856-881` + +- Claim: sequencer-core already pins the max-fee and exhausted-nonce guards generically (sequencer-core/src/application/mod.rs:512-523, 526-547), so wallet.rs:420-479 duplicates them. The Legacy* SSZ types (551-609) exist only to test an unknown union selector; Method has two variants (method.rs:11-13), and conformance.rs:94 already runs `vec![0xff]`. A one-line unknown-selector payload in an existing wallet test keeps the balance-unchanged assertion that conformance (which only compares native with bridge) lacks. The trivia tests are 541-549, 812-823 and 1057-1065. 856-881 nearly duplicates 826-854. No test overdraws after the fee: conformance's mixed history is fully funded (conformance.rs:53-94). Removing ~174 lines and adding a ~25-line business-failure test nets about -150, not -170. +- Fix: Delete the listed tests. Replace the legacy block with a one-line malformed-method case. Add one test that pins an insufficient transfer and an insufficient withdrawal: fee, nonce and progress advance, and there are no outputs and no other balance change. + +### J8. WalletConfig doc says the fee recipient defaults to Address::ZERO; Default is Sepolia's sequencer address + +`low · confirmed · -1 lines · S · docs-only` +Where: `examples/app-core/src/application/wallet.rs:29-30`, `examples/app-core/src/application/wallet.rs:34-39`, `examples/app-core/src/application/wallet.rs:51-55`, `examples/app-core/src/application/wallet.rs:856-881` + +- Claim: wallet.rs:29 says the fee recipient 'Defaults to Address::ZERO (burned)'. But Default is sepolia() (51-55), whose sequencer_address is SEPOLIA_SEQUENCER_ADDRESS (38, 73-74). The test 'fee_sent_to_zero_address_when_default' (856-881) sets ZERO explicitly. See JS1 for the root cause. +- Fix: Change the doc to 'Address credited with fee revenue.' Delete or rename the test (covered under J6). + +### J9. Engine status mapping drops PermissionDenied, so the host's PermissionDenied-terminal arm can never fire for C apps + +`low · confirmed · -1 lines · S · BC` +Where: `examples/c-wallet-engine/src/lib.rs:51-70`, `bindings/c-app-engine/src/lib.rs:78-84`, `sequencer/src/commands/error.rs:159-173`, `sequencer/src/ingress/inclusion_lane/dump_info.rs:27-35` + +- Claim: On AppBootstrap, the host classifies Io(PermissionDenied) as terminal (sequencer/src/commands/error.rs:159-172). Over the C seam, the reference engine maps it to IO_ERROR (c-wallet-engine lib.rs:52-63), and the shim turns that into ErrorKind::Other (c-app-engine lib.rs:78-79), which is classified Unclassified (error.rs:186), giving exit 1. The arm is also reachable natively beyond genesis reads: write_baseline_dump maps any create_dump AppError into AppBootstrap (setup/fill.rs:70-72), so an unwritable dumps dir exits 30 natively and 1 over C. error.rs:163 also contradicts the rationale for referenced artifacts (dump_info.rs:24-34), which keeps permission failures operational because they 'may clear'. Dropping PermissionDenied from error.rs:163 makes both paths and both policies consistent. +- Fix: Frequency is low (operator misconfiguration only) and the fallback is safe (a retried failure that keeps failing), so add no new status. Drop PermissionDenied from error.rs:163 so native and C agree it is operational, or document the asymmetry in the header. Point the engine's mapping comment at referenced_artifact_io_is_terminal. +- Decision: Should PermissionDenied on the genesis dump be terminal (exit 30) or operational (exit 1)? + +### J10. Dead public surface and a handled-impossible branch in the example crates + +`nit · unverified-nit · -25 lines · S · public-rust-api` +Where: `examples/app-core/src/application/mod.rs:14-16`, `examples/app-core/src/wallet_snapshot.rs:22-43`, `examples/app-core/src/application/wallet.rs:159-161`, `examples/app-core/src/application/wallet.rs:204-213`, `examples/c-wallet-engine/src/lib.rs:408-417` + +- Claim: The re-export aliases `encode_wallet_snapshot`/`decode_wallet_snapshot` have no users. The Snapshot* structs are `pub` but used only inside their module. The inherent `last_executed_safe_block` shadows the trait's default method. The interior-NUL branch in `state_file_in_dump` cannot be reached, because the prefix came from a CStr and `join("state")` adds no NUL. AGENTS.md says impossible states fail loud rather than being handled. +- Fix: Delete the aliases and the inherent method, make the Snapshot* structs private, inline decode_portal_erc20_deposit, and replace the NUL branch with `.expect("prefix came from a CStr")`. + +### J11. ERC-20 deposit encoding and EIP-712 user-op signing hand-rolled in many test files + +`low · confirmed · -70 lines · S` +Where: `examples/app-core/src/application/wallet.rs:611-617`, `examples/canonical-test/src/main.rs:430-436`, `examples/c-wallet-engine/tests/conformance.rs:32-42`, `examples/canonical-test/src/main.rs:334-368`, `sdk/guest/rollups-types/src/lib.rs:220-236` + +- Claim: rollups-types has decode only (sdk/guest/rollups-types/src/lib.rs:220-235). The three deposit encoders are at wallet.rs:611-617, canonical-test main.rs:430-436 and conformance.rs:32-42. Nine parity-search signing loops exist, all in test code (canonical-test:352, sequencer-core scheduler/mod.rs:657, api.rs:520/:567 inside its tests module at :271, e2e_sequencer.rs:74/124/166/1595, harness wallet.rs:335). A cross-crate shared helper needs a feature-gated home. The simpler route is to switch each site to k256 `sign_prehash_recoverable` (or alloy's signer) in place, with no new shared module. Most sites are outside area J. +- Fix: Add `Erc20Deposit::encode` next to decode, which also gives decode a round-trip test. Add one shared test helper `sign_user_op(key, op, domain)` that uses k256 `sign_prehash_recoverable` to get the parity directly. Migrate the in-area copies first. + +### J12. canonical-app justfile duplicates every devnet/sepolia recipe and shares their intermediate outputs + +`low · partially · -30 lines · S` +Where: `examples/canonical-app/justfile:17-23`, `examples/canonical-app/justfile:41-51`, `examples/canonical-app/justfile:55-71`, `examples/canonical-app/justfile:82-104` + +- Claim: The build-dapp-*, build-rootfs-* and build-machine-image* recipe pairs (justfile:41-51, 55-71, 82-104) differ only in network name, so parameterizing them saves about 25 lines. Sharing the intermediates (dapp, rootfs tar/ext2, lines 17-20) causes no stale mix: every image recipe re-runs its own dapp and rootfs dependencies in order (82→55→41, 94→64→47). Per-net intermediates are therefore not needed. The recipe names are referenced by the root justfile (68-71, 89-93), .github/workflows/release.yml:148-151 and the watchdog docs, so keep them as aliases. +- Fix: Parameterise with `net := devnet|sepolia` (for example `build-machine-image net="devnet"`), use per-net intermediate paths, and keep the two image output names. + +### J13. Error messages without context, Debug formatting, and a user-triggerable warn repeated on every re-execution + +`nit · partially · +0 lines · S` +Where: `examples/app-core/src/application/wallet.rs:266-269`, `examples/c-wallet-engine/src/lib.rs:218-220`, `examples/c-wallet-engine/src/lib.rs:66`, `examples/c-wallet-engine/src/bin/c-wallet-genesis.rs:33`, `examples/app-core/src/application/wallet.rs:327-334` + +- Claim: These are minor message cleanups. wallet.rs:267-268 is an impossible-state error (the fee was validated first; conformance.rs:236 calls it an invariant fault), so adding context there is fine. engine lib.rs:218-219 is documented as unreachable (lib.rs:211-213). `{other:?}` at engine lib.rs:66 and `{err:?}` at c-wallet-genesis.rs:33 could use AppError's Display (sequencer-core application/mod.rs:13-20). Do NOT downgrade the unsupported-token warn at wallet.rs:328. Such a deposit moves the tokens to the app contract and credits nobody, so it records user funds stuck with no refund path. Repetition is bounded by attacker L1 gas per deposit, plus replays after a restart or rebuild. +- Fix: Add sender, fee and balance to the Internal reason, name the reason in the engine message, use `{err}`, and downgrade the unsupported-token log to debug, or info at most. + +### J14. Comments that restate the trait contract or carry cross-module lore already enforced elsewhere + +`nit · unverified-nit · -15 lines · S · docs-only` +Where: `examples/app-core/src/application/wallet.rs:373-378`, `examples/app-core/src/application/wallet.rs:75-84`, `examples/app-core/src/application/wallet.rs:241-242`, `examples/c-wallet-engine/src/lib.rs:229-232` + +- Claim: Several comments restate the trait's durability doc, carry sync obligations the harness already asserts, or are hard to parse. +- Fix: Replace each with a one-line pointer: 'durability: see Application::create_dump', and 'anvil #9, distinct from the deployer (setup's nonce gate); harness asserts sync'. Delete 241-242 and reword 229-232 as 'refuse while outputs are queued: overwriting would drop them'. + +### JS1. `Default` on WalletConfig/WalletApp silently selects Sepolia; the release binary and the genesis tool pick a network implicitly + +`low · skeptic-added · -8 lines · S · public-rust-api` +Where: `examples/app-core/src/application/wallet.rs:51-55`, `examples/app-core/src/application/wallet.rs:216-220`, `examples/wallet-sequencer/src/main.rs:23`, `examples/c-wallet-engine/src/bin/c-wallet-genesis.rs:24`, `examples/app-core/src/application/wallet.rs:29` + +- Claim: A deployment-specific config implements Default as `sepolia()`. As a result, the release wallet-sequencer's network (main.rs:23) and c-wallet-genesis's no-argument fallback (c-wallet-genesis.rs:24) are chosen implicitly, and the field doc wrongly says the default fee recipient is ZERO (J8). An operator or app author has to read the Default impl to learn which network a binary serves. A sequencer/guest config mismatch shows up as canonical divergence (operator-deployment.md:315 pairs the release binary with the Sepolia image). +- Fix: Delete both Default impls. Call `WalletConfig::sepolia()` explicitly in main.rs:23 and in tests, and make c-wallet-genesis require the preset argument. The golden vector bytes do not change, because the vector is sepolia(). +- Decision: Should c-wallet-genesis require the preset (a CLI change) or keep Sepolia as an explicit, documented fallback? + +Refuted: + +- J7. Dump-contract tests are hand-written per app; no reusable check for an app author's Application impl. The duplication evidence is inaccurate, the -60 estimate ignores the helper's own size, and this conflicts with 'quantify before machinery' and Track 6 ownership. + +Keep (load-bearing): + +- c-wallet-engine delegates every entry point to WalletApp (examples/c-wallet-engine/src/lib.rs:139, 188, 276, 300, 391) rather than reimplementing the rules. That gives zero business-logic duplication between the Rust and C wallets, and the native-vs-bridge conformance suite (conformance.rs:44-161) is the seam's only end-to-end proof. +- WalletApp::create_dump's create_dir (not create_dir_all) plus fsync of the file, the prefix and its parent (wallet.rs:367-385). This is the I13 obligation stated at sequencer-core/src/application/mod.rs:234-241; shorten the comment but keep the code. +- from_snapshot_parts returns a typed AppError on incoherent progress instead of panicking (wallet.rs:97-118). Dump bytes are untrusted input. +- The golden empty-snapshot vector test (wallet_snapshot.rs:139-143) is shared with the Lua watchdog (watchdog/tests/e2e.lua:163). It also guards J5's byte-identity. +- `use c_wallet_engine as _;` in c-wallet-sequencer main.rs:9-11 is a load-bearing link import, as its comment says. +- c-app-sequencer build.rs uses a cfg rather than a cargo feature (build.rs:6-13), so `--all-features` cannot produce a binary with no engine to link. +- The engine's execute() refuses to run while outputs are still queued (c-wallet-engine lib.rs:240-243). Otherwise chain-bound outputs would be silently dropped. +- The canonical-test fee-arithmetic test (canonical-test main.rs:224-302) is the only guest-side test with a nonzero fee, and it pins guest gas == host fee_to_linear from both sides. +- The scheduler shell's inspect path reports and keeps serving instead of panicking (canonical-app scheduler/mod.rs:59-73). Inspect is a public read-only endpoint. +- DEVNET_SEQUENCER_ADDRESS being Anvil account 9, distinct from the deployer (wallet.rs:85-86). A past drift between account 0 and account 9 caused CI reds; keep the fact even after J14 shortens the comment. + +## K. Tests, harness, benchmarks + +Verdict: Area K holds three test layers. Crate-internal unit and integration tests (sibling test files about 7.3k lines, src/integration_tests about 4.7k lines, plus inline modules) sit in front of three process-level crates under tests/: harness about 3.7k, e2e about 5.2k with 49 libtest-mimic scenarios, and benchmarks about 4.1k. Coverage depth fits a security-critical sequencer. The size does not: fixtures are copied rather than shared. There are 10 hand-written Application stubs, two identical TempDb helpers in three differently named helper modules, four bespoke in-process server bootstraps, two batch-tree invariant checkers, and around 140 repeated open/append/timing boilerplate blocks in recovery_tests.rs alone. Together that is roughly 1.3k removable lines with no loss of discrimination. Correctness problems in the tests themselves: - 43 integration tests silently pass when the listener cannot bind. - The 429 test relies on a 50 ms sleep and can hang instead of failing. - Two e2e scenarios sleep a fixed 7 s instead of polling. - e2e hand-copies sequencer defaults, citing a file that does not exist. The benchmark crate is heavier than the questions it answers. Its target-verdict code can only ever print NOT_EVALUATED. compare_latest re-parses CSV and re-implements the capacity summary the library already computes. The sweep bin has two copy-pasted branches. Roughly 3.5k test lines protect cross-rebuild client continuity and cockroach rebuild. The new direction (a shared recovery tool that produces both startup states) replaces that machinery, so freeze those tests rather than polish them. Worth keeping: the outage matrix (TcpProxy and faketime), the post-scenario schema check against the real binary's DB, the schema-trigger tests, the boundary tests for each recovery entry point, the C-host scenario variants, and kill_on_drop teardown. + + +### K1. 43 in-crate integration tests silently pass when the test listener cannot bind + +`medium · confirmed · -170 lines · S` +Where: `sequencer/src/integration_tests/e2e_sequencer.rs:1183-1192`, `sequencer/src/integration_tests/e2e_sequencer.rs:1290-1296`, `sequencer/src/integration_tests/ws_broadcaster.rs:394-403`, `sequencer/src/integration_tests/snapshot_endpoints.rs:47-55`, `sequencer/src/integration_tests/e2e_sequencer.rs:207-209` + +- Claim: Confirmed as stated. Four helpers map bind PermissionDenied to eprintln + None: e2e_sequencer.rs:1184-1192 and :1290-1296, ws_broadcaster.rs:394-403, snapshot_endpoints.rs:49-55. There are 43 `else { return; }` sites under src/integration_tests. The risk is specific to this repo: agents run `cargo test` inside sandboxes that can deny bind, so a green suite there can hide 43 tests that never ran. Fix: fail on bind error. If sandboxes must stay supported, require an explicit env opt-in for the skip; do not keep the silent pass. Realistic delta is about -150. +- Fix: Panic on any bind error (or `expect`), make the helpers return the runtime directly, delete the let-else at all 43 sites. If a sandbox genuinely cannot bind, gate the module with #[ignore] and an explicit env flag instead of a silent pass. +- Independent trace: confirmed. The code matches the claim exactly. Only PermissionDenied takes the silent path; + +### K2. Three fixture modules under three names, duplicated TempDb, four bespoke in-process server bootstraps + +`medium · partially · -250 lines · M` +Where: `sequencer/src/integration_tests/common.rs:8-23`, `sequencer/src/storage/test_helpers.rs:28-43`, `sequencer/src/commands/test_support.rs:1-89`, `sequencer/src/integration_tests/e2e_sequencer.rs:1179-1282`, `sequencer/src/integration_tests/e2e_sequencer.rs:1284-1337` + +- Claim: Real duplication: - TestDb/temp_db is byte-identical in integration_tests/common.rs:8-23 and storage/test_helpers.rs:28-43. The integration tests are in-crate (lib.rs:33-35) and could import crate::storage::test_helpers (storage/mod.rs:48 is pub(crate)), so common.rs can be deleted. - wait_until is duplicated at inclusion_lane/tests.rs:1000 and setup/fill.rs:593. Overstated: only start_full_server_with_max_body does genesis-dump registration and ensure_open_tip (e2e_sequencer.rs:1201-1225). The other three bootstraps share only bind, L2TxFeed::new and start_on_listener with SnapshotState, about 25 lines each. They differ on purpose: a parked queue (:1299-1336), ws_max_subscribers (ws_broadcaster.rs:418-424), and a dummy state layout (ws_broadcaster.rs:382-384). A single TestServer builder saves less than claimed: realistic total is about -100 to -130, not -250. +- Fix: One #[cfg(test)] crate::test_support module (submodules if needed) holding temp_db, wait_until, pin_test_deployment_identity, the stub app (K3) and a single TestServer builder (with_lane / parked_queue(cap) / max_body). Delete common.rs. + +### K3. Ten hand-written Application stubs repeat the same dump/from_dump/state_file plumbing + +`medium · partially · -350 lines · M` +Where: `sequencer/src/ingress/inclusion_lane/tests.rs:55-128`, `sequencer/src/ingress/inclusion_lane/tests.rs:130-186`, `sequencer/src/ingress/inclusion_lane/tests.rs:201-356`, `sequencer/src/ingress/inclusion_lane/tests.rs:1770`, `sequencer/src/commands/test_support.rs:22-74` + +- Claim: Confirmed: - There are 10 test `impl Application`. One, dump_info.rs:271 PrefixDumpApp, is missing from the locations. - TestApp.nonces is write-only (tests.rs:57, 91-93). - The SweepTestApp const hack is at test_support.rs:27-31. - SharedCountingApp, UserOpCounterApp and CountedSweepTestApp are the same progress-persisting stub. fill.rs:137-151 even re-implements decode_progress inline. The proposed single StubApp with instance knobs does not fit the trait. from_dump is static, and the lane reloads through it, so instance knobs survive only on directly constructed instances (tests.rs:533, :1978). - TestApp, InternalUserOpApp, SweepTestApp and StartupProbeApp reset to default on reload (tests.rs:113-115, workers.rs:722-726). - The counting stubs decode the persisted progress. - ReplayForbiddenApp and PrefixDumpApp (const-generic layout) must stay bespoke. Making every stub persist progress would change reload semantics for the reset-style stubs. Realistic shape: one persisting counter stub, one reset stub with knobs, plus the two bespoke ones. That is about -200, not -350. +- Fix: One StubApp in test_support. It persists progress via encode/decode_progress and has opt-in knobs (reject_after, fail_dump, fail_validation, fail_apply, record: Option>). Keep a bespoke impl only where the static from_dump must behave differently (ReplayForbiddenApp). Drop the write-only nonce map. + +### K4. recovery_tests.rs: ~140 repeated open/append/timing blocks, plus 680 lines of schema tests in the wrong file + +`medium · confirmed · -500 lines · M` +Where: `sequencer/src/storage/recovery_tests.rs:26-56`, `sequencer/src/storage/recovery_tests.rs:1913-1938`, `sequencer/src/storage/recovery_tests.rs:2136-2818`, `sequencer/src/storage/recovery.rs:642-643` + +- Claim: Confirmed, with corrected counts: - `Storage::open(db.path.as_str())` appears 68 times. - `&default_protocol_timing()` appears 57 times, not 65. - `StoredSafeInput {` appears 38 times. schema_invariants spans recovery_tests.rs:2136-2818 (682 lines). recovery.rs:641-643 is the only #[path] module in sequencer/src. The schema module's own comment (recovery_tests.rs:2141-2148) says it lives there only to share fixtures. Those fixtures are already in test_helpers, so moving it is a pure relocation. Realistic delta is about -350 to -450. The appends vary in sender, payload and count, so not all of them collapse to one line. +- Fix: Extend the fixture trait in test_helpers with land(block, &[(sender, payload)]) and advance_safe_head(block), using SENDER_A and the default timing. Add a fixture fn that returns an opened Storage. Move schema_invariants next to the migration code (storage/tests.rs or open.rs tests). Keep every scenario. + +### K5. About 3.5k test lines guard cross-rebuild client continuity and the in-process cockroach rebuild: freeze them + +`medium · partially · +0 lines · S` +Where: `sequencer/src/integration_tests/historical_bootstrap.rs:1-561`, `sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs:1-403`, `sequencer/src/storage/history/generation_tests.rs:1-170`, `tests/e2e/src/cold_replica.rs:1-366`, `sequencer/src/commands/setup/checkpoint_tests.rs:1-226` + +- Claim: The superseded set is much smaller than 3.5k lines. Recovery generations and cuts are advanced by the automatic stale-batch cascade (recovery.rs:487-493, cascade_and_reopen), not only by cockroach rebuild. These tests therefore guard client continuity across routine automatic recovery, which the recovery tool does not replace: - generation_tests.rs (170) - canonical/tests.rs (429) - cold_replica.rs (366). Its "process-boundary recovery" is advance_wall_and_mine past MAX_WAIT plus respawn (cold_replica.rs:184-190), not a rebuild. - recovery_compatibility.rs ('survives missed recoveries') At most they are frozen pending the later sync redesign, not slated for deletion when the tool lands. The snapshot download tests are not superseded. /finalized_state, /inclusion_block and /digest are consumed by the watchdog (watchdog/sequencer.lua:48-64). The LeaseReadError scaffolding (snapshot_endpoints.rs:276-355) backs the /finalized_state lease tests (:357-638). checkpoint_tests.rs pins rebuild_from_checkpoint refusal safety. That logic moves into the tool rather than dying, so freeze it and migrate it. Genuinely superseded by the tool: the cross-rebuild client path, i.e. historical_bootstrap.rs (561, 'crosses the rebuilt baseline', LifecycleCommand::Rebuild at :270) and the era/raw-history egress tests in historical/tests.rs (497). Total is about 1.1k lines. +- Fix: Freeze: do not apply K2-K4 consolidation here beyond mechanical path fixes. Delete or move these tests together with the machinery when the recovery tool lands. Its tests should become checkpoint-to-(sequencer DB, client DB) golden tests. The snapshot lease-read retry scaffolding (LeaseReadError, :276-345) goes with it. +- Decision: Confirm snapshot download (snapshot_endpoints.rs) is in the superseded set, not kept for the watchdog or operator tooling. + +### K6. Benchmark target-verdict code can only ever say NOT_EVALUATED; the spec lists harnesses that do not exist + +`medium · confirmed · -280 lines · S` +Where: `tests/benchmarks/src/evaluation.rs:13-214`, `tests/benchmarks/src/bin/round_trip_latency.rs:72,199-206`, `tests/benchmarks/src/bin/sweep.rs:93,391`, `tests/benchmarks/BENCHMARK_SPEC.md:83-139`, `tests/benchmarks/README.md:63` + +- Claim: Confirmed. NetworkProfile::same_host_baseline() is the only constructor used (round_trip_latency.rs:72, sweep.rs:93). CanonicalNetworkAware is never built, so every verdict is NOT_EVALUATED (evaluation.rs:39, 171-187). Section 7 of the spec maps metrics to `ack_latency` and `unit_hot_path`, but src/bin holds only compare_latest, report, round_trip_latency and sweep. When deleting, keep a one-line network-profile string in the report, because spec section 6.5 requires it. +- Fix: Delete evaluation.rs, the --evaluate flags and the verdict JSON field. Shrink the spec to the two p99 targets and the sample-size rule, and fix section 7. Re-add a verdict only when a network shaper exists. + +### K7. Benchmark tooling duplicates itself: CSV re-parsing, a re-implemented capacity summary, copy-pasted sweep branches, a redundant bin + +`medium · partially · -700 lines · M` +Where: `tests/benchmarks/src/bin/compare_latest.rs:35-57,283-390,431-534`, `tests/benchmarks/src/sweep.rs:110-145`, `tests/benchmarks/src/bin/sweep.rs:236-330`, `tests/benchmarks/src/bin/sweep.rs:340-450`, `tests/benchmarks/src/bin/round_trip_latency.rs:1-238` + +- Claim: Confirmed: - compute_sweep_summary (compare_latest.rs:509-533) re-implements compute_capacity_summary (sweep.rs:110-145) field for field, over hand-parsed CSV rows (compare_latest.rs:431-500, 351-390). - The sweep bin writes JSON in both branches (:353, :481). - report.rs already loads that JSON (report.rs:26-36, 258-297). - The ack and round-trip branches in bin/sweep.rs share their structure. Not verified: that round_trip_latency is exactly `sweep --round-trip` at one concurrency. It owns the --evaluate soft-confirm path (round_trip_latency.rs:199-206), which disappears with K6, and possibly other flags. Realistic delta is about -400 to -500, not -700. +- Fix: Make JSON the only machine-read artifact (CSV becomes print-only or goes away). Use one row type with optional round-trip columns, one sweep code path that takes a Mode, and drop round_trip_latency (just keeps the recipe name). compare_latest then loads two JSONs and diffs the library summaries. Goal: about 4.1k down to about 3.3k lines. Do this before K8 so the new metrics land once. +- Decision: Whether the ack-only sweep (no WS clients) must stay a separate mode for capacity runs. Recommended: keep it as a flag, not a separate code path. + +### K8. N17: add DB size and post-load batch drain time to the benchmark report + +`low · partially · +120 lines · S` +Where: `tests/benchmarks/src/runtime.rs:107-217`, `tests/benchmarks/src/bin/round_trip_latency.rs:172-193`, `tests/harness/src/sequencer.rs:571-591`, `tests/harness/src/sequencer.rs:700-731`, `tests/harness/src/sequencer.rs:876` + +- Claim: The gap is real. MemoryReport (runtime.rs:107-116) is the only resource report, and the harness has the building blocks: data_dir (:876), count_safe_accepted_batches (:571-591) and mine_live_l1_blocks (:938-940). StorageReport (DB plus WAL bytes) is cheap and well defined. drain_secs is less meaningful than claimed. In self-contained mode it is shaped by: - harness-driven mining at 1 s per block (rollups.rs:95-102) - the Anvil safe-head lag (--slots-in-an-epoch, rollups.rs:206) - the submitter idle poll of 5 s by default (test_cases.rs:3074-3076) So it measures poll cadence times batch count, not submitter throughput. Report it as an environment-shaped profiling number, or ship only StorageReport first. This is a missing metric, not a defect, so severity is low. +- Fix: (a) StorageReport { db_bytes, wal_bytes, bytes_per_1k_accepted } from a stat of data_dir/sequencer.db and -wal when the run ends, recorded next to MemoryReport (self-contained only). (b) When the last load worker returns, record t0 and the highest sealed-batch nonce (add a MAX(nonce) accessor beside count_safe_accepted_batches). Then loop mine_live_l1_blocks(1) and poll MAX(nonce) in safe_accepted_batches until it reaches that nonce, with a deadline. Report drain_secs and batches_drained. Exclude the open tip so the metric measures submitter throughput, not max_batch_open. About 120 lines, no machinery. +- Decision: Whether 'after all workers stop' means the load generators stop (recommended; measures the submitter backlog) or the sequencer's own workers shut down (that would measure flush-mempool instead). + +### K9. Timing-based tests: the 50 ms race in the 429 test can hang; fixed 7 s submitter sleeps in e2e + +`low · confirmed · -10 lines · S` +Where: `sequencer/src/integration_tests/e2e_sequencer.rs:464-472`, `sequencer/src/integration_tests/e2e_sequencer.rs:1301,1331-1337`, `sequencer/src/integration_tests/e2e_sequencer.rs:1546-1567`, `tests/e2e/src/test_cases.rs:3077,3120`, `tests/e2e/src/test_cases.rs:3909,3944` + +- Claim: Confirmed. /tx does try_send and then awaits a oneshot (ingress/api.rs:257-268). With the receiver parked (e2e_sequencer.rs:1336) and no timeout in post_raw_json (:1546-1567), the 429 test hangs if the main request wins the single slot. Both e2e scenarios sleep a fixed 7 s (test_cases.rs:3077/3120, 3909/3944). Better fix for the 429 case: add a deterministic unit test in ingress/api.rs with a pre-filled channel(1), modelled on closed_lane_is_service_unavailable (api.rs:289), and drop the racy integration test. +- Fix: Fill the parked queue directly (keep a tx clone and try_send a dummy PendingUserOp) and drop the sleep and the spawned first request. In e2e, poll Anvil's pending tx count (txpool) with a deadline instead of the 7 s sleep, and hoist the shared constant. + +### K10. Two hand-written batch-tree invariant checkers that disagree on the anchor + +`low · confirmed · -90 lines · S` +Where: `sequencer/src/storage/recovery_tests.rs:2835-2924`, `tests/harness/src/sequencer.rs:752-858`, `tests/e2e/src/main.rs:45-49`, `sequencer/src/storage/migrations/0001_schema.sql:110` + +- Claim: Confirmed. The crate copy asserts that a NULL parent has nonce 0 (recovery_tests.rs:2853-2856). The harness copy is anchor-aware (harness sequencer.rs:760-767) and also checks the single valid root. Both run the same LEFT JOIN query. +- Fix: Keep the harness check: it is the only check against the real binary's DB after each scenario. Drop the crate copy and have tree_invariants_hold_across_mixed_workload rely on the trigger tests, or move one anchor-aware checker into test_support and call it from both. + +### K11. e2e hand-copies sequencer defaults and cites a file that does not exist + +`low · confirmed · +0 lines · S` +Where: `tests/e2e/src/test_cases.rs:72-80`, `sequencer/src/commands/config.rs:51-52` + +- Claim: Confirmed. test_cases.rs:72-80 cites runtime/config.rs, which does not exist (sequencer/src/runtime holds only mod.rs, process_lock.rs and shutdown.rs). The real default is commands/config.rs:49-53 (default_value = "300"). ManagedSequencerConfig would need a new field or env passthrough so the harness can set these values explicitly. +- Fix: Have the harness spawn config set CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS and SECONDS_PER_BLOCK explicitly from these constants, so the test owns the values instead of copying them. Drop the sync comment. + +### K12. e2e scenario configuration is keyed by name strings + +`nit · confirmed · +0 lines · S` +Where: `tests/e2e/src/main.rs:19-34` + +- Claim: Confirmed: main.rs:19-33 selects the binary by the c_host_ prefix, disables faketime for three literal names, and sets the fee oracle for one name. This is a style issue, and it is low-risk because names are rarely renamed. +- Fix: Have test_cases() return (name, run, configure: fn(&mut ManagedSequencerConfig)) so each scenario declares its own config next to its body. + +### K13. Harness errors leave out the context needed to diagnose flakes; skip logs hide vacuous passes + +`nit · partially · +5 lines · S` +Where: `tests/harness/src/ws.rs:31-34`, `tests/harness/src/util.rs:57-60`, `tests/harness/src/sequencer.rs:1616-1621`, `sequencer/src/integration_tests/snapshot_endpoints.rs:52` + +- Claim: Real but small: - The WS timeout error 'wait for websocket frame' (ws.rs:31-34) omits the 10 s timeout and the expectation. - 'sequencer exited before readiness' (util.rs:57-60) omits the log path. The link to the register entry is overstated. The recorded failure was a 'Connection reset', which already reaches the error text via `receive websocket frame: {err}` (ws.rs:35). The register asks for server-side logs, which CI now uploads (register.md:147-158). +- Fix: Include the timeout and the expectation in the WS error (expect_user_op_from can pass the sender), and append log_path() to readiness and exit errors. The skip logs go away with K1. + +### K14. Test comments that restate code, describe their callers, or have gone stale + +`nit · unverified-nit · -30 lines · S` +Where: `tests/harness/src/sequencer.rs:565-570`, `sequencer/src/integration_tests/e2e_sequencer.rs:1306-1311`, `sequencer/src/ingress/inclusion_lane/tests.rs:107-111,416-418`, `sequencer/src/ingress/inclusion_lane/tests.rs:206-211`, `tests/harness/src/sequencer.rs:745-752` + +- Claim: count_safe_accepted_batches' doc describes the one test that calls it. The 'Sentinel submitter: pass an address no fixture seeds' comment sits on an L2TxFeedConfig literal that no longer has an address field. The comments around drop(app) and from_dump restate the code. SharedCountingApp claims it 'counts direct-input executions' but only persists progress. assert_schema_invariants is documented as panicking while it returns HarnessResult (mixed error channels). +- Fix: Delete or trim them in the K2/K3 sweep. Per AGENTS.md, keep only comments that state a non-obvious why. + +### K15. `extern crate self as sequencer` lets in-crate tests spell the same path two ways + +`nit · unverified-nit · -2 lines · S` +Where: `sequencer/src/lib.rs:32-35`, `sequencer/src/integration_tests/e2e_sequencer.rs:7,16-22` + +- Claim: Left over from moving the tests into src (bc8ffe8). Integration files mix `crate::storage::...` and `sequencer::storage::...`; there are 35 `sequencer::` sites across five files. +- Fix: Rewrite sequencer:: to crate:: in integration_tests and drop the alias. A mechanical change. + +### KS1. Harness repeats the read-only DB open block in ten accessors + +`low · skeptic-added · -50 lines · S` +Where: `tests/harness/src/sequencer.rs:387`, `tests/harness/src/sequencer.rs:446`, `tests/harness/src/sequencer.rs:468`, `tests/harness/src/sequencer.rs:553`, `tests/harness/src/sequencer.rs:572` + +- Claim: Each harness DB accessor repeats the same 6-line block: join("sequencer.db"), open_with_flags(READ_ONLY) and the same map_err. The DB filename is spelled out 11 times. +- Fix: Add one private `fn open_db_read_only(&self) -> HarnessResult` and call it from every accessor. This also gives a single home for the DB filename that K8's StorageReport needs. + +### KS2. Stale 'Sentinel submitter' comments describe an argument L2TxFeed::new no longer takes + +`nit · skeptic-added · -5 lines · S` +Where: `sequencer/src/integration_tests/e2e_sequencer.rs:1253-1254`, `sequencer/src/integration_tests/e2e_sequencer.rs:1309-1310`, `sequencer/src/integration_tests/snapshot_endpoints.rs:60` + +- Claim: These comments say the fixture passes a sentinel submitter address. L2TxFeed::new takes only (db_path, shutdown, config) (egress/l2_tx_feed/mod.rs:71), and the L2TxFeedConfig literals the comments sit in have no such field. The real sentinel is NO_OWN_BATCHES, pinned through the deployment identity in l2_tx_feed/tests.rs:322-339. +- Fix: Delete the three stale comments. + +Keep (load-bearing): + +- src/integration_tests inside the crate (integration_tests/mod.rs:1-2): keeps the raw worker-launch APIs crate-private; moving them to tests/ would force public surface. +- tests/harness TcpProxy (harness/src/proxy.rs, about 440 lines) together with faketime: the only way to drive the provider-outage / both-down / hiccup scenario matrix. +- Harness assert_schema_invariants run after every scenario (e2e/src/main.rs:45-49): the only structural check against the real binary's DB. Keep this anchor-aware copy (see K10). +- The schema_* trigger tests (recovery_tests.rs:2152-2801): the schema is the enforcement layer for the batch-tree invariants. Move them (N4), do not cut them. +- Boundary pairs that look redundant but cover different entry points: recover_aging_tip (recovery_tests.rs:985-1022) vs recover_post_flush (:1913-1966). +- The reject-flood frontier-starvation test (inclusion_lane/tests.rs:505-585): its sleep-based stub is the only pin on bounded turns versus divergence detection. +- The c_host_ scenario variants (e2e main.rs:20, test_cases.rs:160-171): they run the same scenarios through the C ABI bridge. +- Zone-math derived constants and compile-time asserts in test_cases.rs:57-100: they tie the outage scenarios to MAX_WAIT_BLOCKS (fix the copied defaults per K11, keep the derivation). +- Serial e2e runner (main.rs:12, test_threads=1): scenarios share Anvil, faketime and the process lock. +- kill_on_drop on spawned children (harness/sequencer.rs:1397,1565): a panicking assertion still tears down Anvil and the sequencer. +- MemorySampler and the accepted-tx-to-WS-event sanity assertion (BENCHMARK_SPEC section 3.5): the benchmark's real correctness guard and memory profile. + +## L. Build, CI, supply chain, cross-compilation + +Verdict: Build/CI/supply-chain is about 2,250 lines: 21 manifests, two workflows, three composite actions, three justfiles, six scripts, two Dockerfiles and one build.rs. Its essential complexity is real. A host workspace that links the emulator (testsi), a RISC-V guest cross-build whose template hash is consensus-relevant, a C-engine seam and a Lua/Docker watchdog all need pinned, hash-verified inputs. The core is sound and should be kept. Every downloaded artifact is SHA-pinned. The cross image is pinned by digest. CI runs with --locked. default-members keeps a plain `cargo build` free of the CM/libclang requirement. Clippy already covers all 21 members, guest crates included, through libcmt's host mock. The biggest problems sit around that core, not in it. (1) The release path's supply-chain posture is weaker than CI's: third-party actions are on mutable tags with workflow-wide write permissions, cross/just are unpinned, and a dispatch builds the dispatched branch while labelling it with an arbitrary tag. (2) CI runs every PR push twice with no concurrency cancel, and builds the devnet guest image in two jobs. (3) The pin plumbing (verify/load scripts, a Rust version pinned in three places) is about 150 lines that mostly compare the pins file with itself, while the real drift points go unchecked: Foundry version ×3, contracts version ×2, guest-tools version. (4) The lockfile looks heavier than the build: 537 entries, ark-ff×4, three syn majors. Most of that comes from weak `dep?/feat` optional entries that never compile. The compiled duplicates the manifests actually cause are four (reqwest, tokio-tungstenite, thiserror, toml), plus one unused direct dep. Measure with `cargo build --timings` / `cargo tree -d -e normal,build` before any wider dependency surgery. + + +### L1. Release path: mutable action tags, workflow-wide write token, unpinned cross/just, caches in release + +`medium · confirmed · +15 lines · S` +Where: `.github/workflows/release.yml:23-25`, `.github/workflows/release.yml:65-71`, `.github/workflows/release.yml:191-229`, `.github/workflows/release.yml:270`, `.github/actions/setup-guest-toolchain/action.yml:89-118` + +- Claim: Confirmed as stated. One refinement: DOCKERHUB_TOKEN reaches only build-watchdog-image (release.yml:180-208). The broader exposure is the workflow-wide GITHUB_TOKEN with contents:write and packages:write (release.yml:23-25). Every tag-referenced action in every job can use that token, including taiki-e/install-action@v2 (:69), Swatinem/rust-cache@v2 (:66) and dtolnay/rust-toolchain@stable (:60). +- Fix: Pin every non-GitHub-owned action to a commit SHA with a `# vX.Y` comment. Move `permissions` to the job level: contents:write only on `publish`, packages:write only on `build-watchdog-image`, and `contents: read` at the top of both workflows. Pin tool versions (`tool: cross@,just@`) and record them in toolchain-pins.env. Drop rust-cache/gha cache from release.yml, since a release build should not restore a cache written by other refs. Add .github/dependabot.yml for `github-actions` so the SHA pins stay current. +- Decision: Whether release builds may keep caches (speed) or must build cold (provenance). + +### L2. workflow_dispatch release builds the dispatched branch but labels/tags it with inputs.tag + +`medium · confirmed · +6 lines · S · BC` +Where: `.github/workflows/release.yml:11-21`, `.github/workflows/release.yml:41-42`, `.github/workflows/release.yml:127-128`, `.github/workflows/release.yml:174-175`, `.github/workflows/release.yml:269-273` + +- Claim: Confirmed. It matters more than the reviewer says: generate-release-manifest.sh:85-91 writes an explicit `alignment.rule`, 'All artifacts sharing release_tag were built from git_commit'. A dispatch from main naming a new tag breaks that rule, because softprops creates the tag at the default-branch HEAD while git_commit is GITHUB_SHA. A dispatch naming an existing tag is also wrong: it uploads assets built from the dispatched branch into that tag's release. +- Fix: Pass `ref: ${{ inputs.tag || github.ref }}` to every checkout in release.yml, and fail early if `inputs.tag` does not resolve to an existing tag. Alternatively, drop the free-form `tag` input and let dispatch only re-run an existing tag. +- Decision: Should dispatch be allowed to create a new tag at all, or only rebuild an existing one? +- Independent trace: confirmed. Code reading confirms the claim. There are four checkouts and none passes `ref:`. + +### L3. CI runs every PR push twice and never cancels superseded runs + +`low · partially · +5 lines · S` +Where: `.github/workflows/ci.yml:3-5`, `.github/workflows/ci.yml:102-105`, `.github/workflows/ci.yml:131-134`, `.github/workflows/ci.yml:182-186` + +- Claim: The double run and the missing concurrency group are real: ci.yml:3-5 has no filters and the file has no `concurrency:` key. Two parts are overstated. First, watchdog-docker's `needs: rust` (ci.yml:185) is not 'for no reason'. It is the same fail-fast gate canonical-guest (:104) and rollups-e2e (:133) use, trading latency for wasted minutes on red pushes. Second, `push: branches: [main]` also stops CI on v* tag pushes (today the unfiltered push trigger runs it). release.yml has no CI gate, so a tag commit that never reached main or a PR would then ship with no CI run at all. Treat this as a cost and latency cleanup, not a defect. +- Fix: Use `push: branches: [main]` plus `pull_request:`, and add `concurrency: { group: ${{ github.workflow }}-${{ github.ref }}, cancel-in-progress: true }`. Drop `needs: rust` from watchdog-docker. This roughly halves runner minutes per PR iteration, and the safety net is unchanged because the PR run covers the same tree. +- Decision: The double run has incidentally surfaced flakes (register:150-152). Confirm that losing that is acceptable; a nightly re-run is the explicit alternative. + +### L4. Four compiled duplicate stacks are caused by our own version picks, plus one unused direct dep + +`low · partially · -2 lines · S · public-rust-api` +Where: `Cargo.toml:62`, `Cargo.toml:71`, `sdk/guest/testsi/Cargo.toml:23`, `sequencer/Cargo.toml:19`, `sequencer/Cargo.toml:28` + +- Claim: The four duplicate stacks exist as described, but none of reqwest 0.12, tokio-tungstenite 0.28 or thiserror 1 is in the shipped wallet-sequencer graph. sequencer pulls reqwest and tokio-tungstenite only as dev-dependencies (sequencer/Cargo.toml:45-46). The other users are the SDK, harness and benchmarks, and thiserror 1 comes only from testsi (sdk/guest/testsi/Cargo.toml:23). That makes this test and CI build time only. alloy-network-primitives is confirmed unused: sequencer/Cargo.toml:28, with zero `alloy_network_primitives` uses in any .rs file. The tokio-tungstenite part is superseded-by-direction. axum's `ws` feature (sequencer/Cargo.toml:14) serves only `/ws/subscribe` (egress/api/mod.rs:36), and the SDK's tungstenite is that endpoint's client (sdk/rust-client/src/lib.rs:25). Both are the subscribe path the new sync replaces, so freeze them rather than dedupe. Keep the reqwest, thiserror, toml and unused-dep parts. The reqwest bump changes the SDK's public API (`pub response: reqwest::Response`, sdk/rust-client/src/lib.rs:33), so coordinate it with the open register entry 'Support HTTPS and WSS in the Rust SDK', which needs a TLS feature choice anyway. +- Fix: Bump `reqwest = { version = "0.13", default-features = false }` and `tokio-tungstenite = "0.29"` in [workspace.dependencies]. Set testsi to `thiserror = { workspace = true }`. Delete `alloy-network-primitives` from sequencer/Cargo.toml:28. Optionally replace `alloy-transport` with `alloy::transports::layers::RetryBackoffLayer` and bump toml to the current major. Check the result with `cargo tree -d -e normal,build --workspace`. +- Decision: sequencer-rust-client is an SDK. If its public API exposes tungstenite/reqwest types, the bump is a semver-visible change. The toml bump changes how checkpoint info.toml is rendered (dump_info.rs:109,159); confirm no digest covers those bytes before bumping. + +### L5. Pin plumbing (~150 lines) mostly checks the pins file against itself and misses the real drift points + +`medium · partially · -90 lines · S` +Where: `scripts/verify-toolchain-pins.sh:43-74`, `scripts/load-toolchain-pins.sh:1-49`, `rust-toolchain.toml:2`, `toolchain-pins.env:5`, `toolchain-pins.env:21-22` + +- Claim: The core is confirmed. verify-toolchain-pins.sh:47-55 re-reads toolchain-pins.env and compares it with values sourced from the same file. Its only justfile check (:57) greps for `pins := "../../toolchain-pins.env"`, and that variable is dead: examples/canonical-app/justfile:3 defines it, but lines 4-6 hard-code the path. The real drift points go unchecked. Foundry 1.5.1 appears at ci.yml:57, action.yml:30 and justfile:16. The contracts version appears at Cargo.toml:49 and justfile:15, plus a third copy the reviewer missed: tests/harness/src/paths.rs:6-7 hard-codes both 3.0.0-alpha.10 and anvil-1.5.1. Guest-tools 0.18.0 appears at canonical Dockerfile:7 and watchdog/test-guest/Dockerfile:8, and must match libcmt-sys. Two corrections to the proposal. RUST_TOOLCHAIN cannot simply be deleted, because generate-release-manifest.sh:79 reads os.environ["RUST_TOOLCHAIN"] into RELEASE.json; the script must parse rust-toolchain.toml instead. The cited toolchain-pins.env:21-22 is the lua-cURL comment and is irrelevant. A realistic net delta is about -60 to -70 lines. +- Fix: Make rust-toolchain.toml the only Rust pin: delete RUST_TOOLCHAIN, the `rust-toolchain` action input and the dtolnay steps, and use `rustup toolchain install` (adding `targets` where needed). Add FOUNDRY_VERSION, ROLLUPS_CONTRACTS_VERSION and MACHINE_GUEST_TOOLS_VERSION to toolchain-pins.env, and have ci.yml, the justfile and the Dockerfile ARG consume them. Replace verify lines 43-74 with three real cross-file greps (Cargo.toml `=${ROLLUPS_CONTRACTS_VERSION}`, justfile dump name, libcmt-sys description/README). Inline the loader as `grep -E '^[A-Z_]+=' toolchain-pins.env >> "$GITHUB_ENV"`. + +### L6. Consensus guest build lacks --locked, uses an unpinned cross binary, and does not remap paths + +`low · partially · +2 lines · S` +Where: `examples/canonical-app/justfile:43`, `examples/canonical-app/justfile:49`, `examples/canonical-app/Cross.toml:3`, `.github/actions/setup-guest-toolchain/action.yml:115-118`, `.github/workflows/release.yml:147-151` + +- Claim: The facts are right. The cross builds have no --locked (canonical justfile:43,49), and neither does `cargo run -p canonical-test` (:107). The cross binary is unpinned (action.yml:115-118). There is no .cargo/config.toml or RUSTFLAGS remap anywhere. The impact is smaller than claimed. In CI, canonical-guest and rollups-e2e `needs: rust`, and rust builds with --locked (ci.yml:60), so a stale lock fails first. A release tag on a commit that already passed CI cannot re-resolve. The checkout-path concern is probably moot: cargo passes workspace-relative paths for workspace members, and no path dependencies sit outside the workspace. Adding --locked is still a free one-line fix. Leave the remap concern pending the reviewer's two-path experiment. +- Fix: Add `--locked` to both cross builds (and to `cargo run -p canonical-test`). Pin the cross version (covered by L1). Try a cheap experiment first: run `just canonical print-build-hashes` from two different checkout paths. Add `RUSTFLAGS=--remap-path-prefix=$PWD=/src` only if the hashes differ. +- Decision: Whether independent rebuilds of the template hash from any checkout path are a goal. The remap part is low-confidence until the two-path experiment runs. + +### L7. The devnet guest image is built twice per CI run (canonical-guest and rollups-e2e jobs) + +`low · confirmed · -25 lines · S` +Where: `.github/workflows/ci.yml:102-129`, `.github/workflows/ci.yml:131-171`, `examples/canonical-app/justfile:82`, `examples/canonical-app/justfile:106-107`, `justfile:62-71` + +- Claim: Confirmed. canonical-guest runs test-guest, which depends on build-machine-image (canonical justfile:106,82). rollups-e2e builds the same devnet image through ensure-machine-image (root justfile:62,67-68), each job after a full setup-guest-toolchain. The e2e path runs lua-deps three times (ci.yml:163, justfile:63, justfile:84) and `apt-get update` a second time (ci.yml:157 after action.yml:46). The ~-25 delta is plausible because it mostly removes ci.yml:102-129. +- Fix: Fold the canonical-guest job into rollups-e2e by running `just canonical-test-guest` first. That builds the image once, and ensure-machine-image then reuses it. Drop the redundant lua-deps invocation and the second `apt-get update` (ci.yml:155-163). The trade-off is one less parallel job and roughly 10 minutes more wall-clock on the e2e path. +- Decision: Choose between parallel wall-clock and the runner minutes saved by building the guest image once. + +### L8. canonical-app justfile copy-pastes devnet and sepolia recipes three times + +`low · confirmed · -35 lines · S · config` +Where: `examples/canonical-app/justfile:39-104`, `justfile:89-93`, `.github/workflows/release.yml:147-151` + +- Claim: Confirmed. The pairs at examples/canonical-app/justfile:41-51, 55-71 and 82-104 differ only in the bin name or the --store path. dapp_binary, rootfs_tar and rootfs_ext2 (:17-20) are shared across chains. Today this is hygiene, not a bug: each machine-image recipe rebuilds its own rootfs before storing it, so the sequential builds in release.yml:147-151 and justfile:62 come out correct. A realistic net is about -20 to -25 lines once the aliases are added back. +- Fix: Use one `build-machine-image chain="devnet"` recipe with per-chain intermediates under out//. Keep the existing output dir names and thin aliases so the harness, watchdog and release consumers do not change. + +### L9. Release builds wallet-sequencer through cross even for the native x86_64 target and installs unused apt packages + +`low · confirmed · -10 lines · S · BC` +Where: `.github/workflows/release.yml:28-76`, `.github/workflows/release.yml:47-57` + +- Claim: Confirmed. release.yml:76 builds both matrix targets with cross. There is no root Cross.toml, so the default tag-based images are used. lua5.4, liblua5.4-dev, libslirp-dev and libssl-dev are installed (:47-57), but Cargo.lock has no openssl, native-tls or hyper-tls. Cross's older-glibc images are a deliberate portability benefit, which the needs_decision already captures. +- Fix: Either build natively on `ubuntu-24.04` and `ubuntu-24.04-arm` runners, which drops cross and docker from this job, or pin cross and add a root Cross.toml with image digests. Drop the unused apt packages either way. +- Decision: Cross images give an old-glibc, portable binary, while native 24.04 runners raise the glibc floor to 2.39. Pick the supported glibc floor, or move to a static musl target. + +### L10. Lint gaps: the production bash launcher and scripts are not shellchecked, one Lua fixture is unlinted, there is no workspace lint table, and `just ci` has drifted from CI + +`low · confirmed · +12 lines · S` +Where: `.github/workflows/ci.yml:65-69`, `watchdog/sequencer-watchdog:1-5`, `scripts/watchdog-lua-deps.sh:84`, `scripts/verify-toolchain-pins.sh:13`, `Cargo.toml:1-73` + +- Claim: Confirmed, with one count corrected: there are 8 `allow` attributes, not 6. The two missed are libcmt-sys/src/lib.rs:1-3 (three bindgen allows) and sequencer/src/commands/test_support.rs:29; all remain justified. Nothing runs shellcheck, even though five `shellcheck disable` directives exist (scripts/*.sh) and watchdog/sequencer-watchdog is 67 lines. luacheck at ci.yml:69 misses watchdog/tests/fixtures/input_added_evm_advance.lua. No manifest has a `[lints]` table. `just ci` (justfile:120-124) omits clippy, luacheck, the Lua tests and the C smoke. +- Fix: Add a CI step `shellcheck scripts/*.sh watchdog/sequencer-watchdog` and lint `tests/**/*.lua`. Add `[workspace.lints.clippy] undocumented_unsafe_blocks = "warn"`, opted in first by bindings/c-app-engine (the seam reader). Either delete `just ci` or make CI invoke it so there is one list. +- Decision: Whether to require SAFETY comments on the FFI seam crates (adds about 40 short comments). + +### L11. No advisory or source gate on the crate graph + +`low · confirmed · +35 lines · S` +Where: `Cargo.toml:41-73`, `sdk/guest/testsi/Cargo.toml:19`, `.github/workflows/ci.yml:8-81` + +- Claim: Confirmed. There is no deny.toml or audit.toml, and no advisory or source step in ci.yml. The only git source is the dave rev (sdk/guest/testsi/Cargo.toml:19). A non-blocking job plus a schedule is proportionate. +- Fix: Add a small deny.toml covering `advisories` and `sources` (allow-git = the dave repo only), with `bans` left at warn. Run `cargo deny check advisories sources` as a separate CI job that does not block PRs, plus a weekly schedule. The invariant it protects is that no known-vulnerable crate ships in the sequencer or watchdog release. +- Decision: Blocking or advisory-only on PRs. + +### L12. sequencer-core build.rs generates 15 constants; a checked-in table plus a recompute test is simpler and visible in review + +`low · confirmed · -50 lines · S` +Where: `sequencer-core/build.rs:1-119`, `sequencer-core/Cargo.toml:25-26`, `sequencer-core/src/fee.rs:20-24` + +- Claim: Confirmed. build.rs:66-118 generates TABLE and MAX_EXPONENT into OUT_DIR, fee.rs:21-23 include!s them, and ruint is the only build-dependency (sequencer-core/Cargo.toml:25-26). A checked-in literal plus a recompute test gives the same guarantee. It also serves fee.rs:15-16 ('Independent implementations must reproduce the conversion bit-for-bit') better, because the values become reviewable. The fee.rs doc references to build.rs (:14,:44) must be updated with it. +- Fix: Check in the generated `TABLE`/`MAX_EXPONENT` as a literal const in fee.rs. Move the repeated-squaring algorithm into a #[test] that recomputes the table with alloy U256 and asserts equality. That gives the same guarantee, enforced in CI, and the table becomes reviewable. +- Decision: Whether the maintainer prefers derived-at-build constants on principle over a test-pinned literal. + +### L13. No profile tuning: full debuginfo for roughly 500 deps on every dev/test build and in the CI cache + +`low · confirmed · +4 lines · S` +Where: `Cargo.toml:1-73` + +- Claim: Confirmed: no manifest has a `[profile.*]` section. Add one caution: restrict the change to `[profile.dev]`. The guest is built with `cross build --release` (canonical justfile:43,49), so any `[profile.release]` edit changes the template hash and is consensus-relevant. +- Fix: Add `[profile.dev] debug = "line-tables-only"` (backtraces keep file:line) and optionally `[profile.dev.package."*"] debug = false`. Measure before and after with `cargo build --timings` and `du -sh target`. +- Decision: Accept losing full variable-level debuginfo in dependencies. + +### L14. Manifest hygiene: wrong inherited readme path, unhoisted shared deps, guest crates outside workspace.package + +`nit · unverified-nit · +0 lines · S` +Where: `Cargo.toml:34`, `sequencer/Cargo.toml:13`, `sequencer/Cargo.toml:19-20`, `tests/harness/Cargo.toml`, `tests/e2e/Cargo.toml` + +- Claim: Inherited `readme = "../README.md"` resolves relative to the workspace root, so it points outside the repo. Shared deps are declared per member: tar 0.4 ×3, toml 0.8 ×2, libtest-mimic 0.6/0.6.1, and the `sequencer-core` path in 8 manifests. rollups-types uses `alloy-* = "1"` instead of the workspace entries. The five guest crates hard-code version, edition 2021 and license instead of inheriting. Only the guest crates set publish=false. +- Fix: Set `readme = "README.md"` and `publish = false` in [workspace.package]. Hoist tar, toml, libtest-mimic and the internal path crates into [workspace.dependencies]. Point rollups-types at workspace alloy. Migrate the guest crates to edition 2024 and `*.workspace = true` in a separate refactor patch (bindgen output may need `unsafe extern`). + +### L15. Comments that restate code or describe things that do not exist (representative) + +`nit · unverified-nit · -12 lines · S · docs-only` +Where: `sequencer/Cargo.toml:50-52`, `sequencer-core/build.rs:24-25`, `sequencer-core/build.rs:73`, `sequencer-core/build.rs:82`, `sequencer-core/build.rs:84-86` + +- Claim: Examples: (1) sequencer/Cargo.toml:50-52 spends three lines explaining what [dev-dependencies] means; "# TcpProxy for provider-disconnect tests" would do. (2) build.rs repeats 'exact integer, no floating point' three times (:11, :86, :100) and narrates steps (`// Build the table by repeated squaring.`, `// Generate Rust source.`). (3) toolchain-pins.env:21-22 documents a pin that does not exist, duplicating watchdog-lua-deps.sh:6-9. +- Fix: Trim to the non-obvious: keep the alloy 2.x/1.x constraint note (Cargo.toml:51-53) and the Cross.toml digest rationale; cut the rest. This is moot for build.rs if L12 lands. + +### L16. Checksum-failure messages hide the actual hash (representative) + +`nit · unverified-nit · +2 lines · S` +Where: `justfile:77`, `examples/canonical-app/justfile:28`, `examples/canonical-app/justfile:36` + +- Claim: When a pinned download is replaced upstream, the operator sees 'SHA256 mismatch ... delete it and re-run' on stdout, with shasum's FAILED line discarded by `>/dev/null`. There is no expected-vs-actual pair to tell a corrupted download from a re-published asset. The kernel check (canonical justfile:28) likewise discards shasum's stdout and prints nothing that names the pinned release. +- Fix: On failure, print `expected= actual=$(shasum -a 256 file)` and the URL to stderr. + +### LS1. The guest kernel, a consensus-relevant input, is not hash-pinned in-tree; its checksum is downloaded next to it + +`medium · skeptic-added · -3 lines · S · BC · config` +Where: `examples/canonical-app/justfile:13-16`, `examples/canonical-app/justfile:25-37`, `toolchain-pins.env:17-19`, `watchdog/test-guest/justfile:41`, `justfile:21-24` + +- Claim: download-deps fetches both the kernel and its .sha512 from the same machine-linux-image release (justfile:31-32) and checks one against the other. That catches corruption in transit, but not a replaced release asset. Every other download is pinned in-tree: the CM and xgenext2fs .debs (toolchain-pins.env:8-15), guest-tools (canonical Dockerfile:8) and the anvil dump (root justfile:24, whose comment states the 'replaced/corrupted release asset' TOFU model). The kernel goes into the machine image's RAM (canonical justfile:87), so it determines the template hash. The area verdict's 'every downloaded artifact is SHA-pinned' is therefore false for the one input that matters most. +- Fix: Add CARTESI_LINUX_KERNEL_SHA512 (or SHA256) to toolchain-pins.env next to CARTESI_LINUX_KERNEL_FILENAME. Verify the downloaded kernel against the pin in download-deps and in watchdog/test-guest, and drop the second wget and the sed rewrite. This protects the invariant that the template hash is a function of in-tree pins and source only, under the same TOFU model the repo already uses. + +### LS2. Tag-push releases are never marked prerelease, so every alpha is published as a full release + +`low · skeptic-added · +0 lines · S · BC` +Where: `.github/workflows/release.yml:273`, `.github/workflows/release.yml:17-21` + +- Claim: `prerelease: ${{ github.event_name == 'workflow_dispatch' && inputs.prerelease }}` is false on the normal path, a v* tag push. Every tag so far is v0.1.0-alpha.1..10 (git tag -l), so each tag push publishes a non-prerelease, which GitHub's default make_latest then marks as Latest. Only manual dispatches honour the prerelease flag. +- Fix: `prerelease: ${{ github.event_name == 'workflow_dispatch' && inputs.prerelease || github.event_name == 'push' && contains(github.ref_name, '-') }}`, which marks SemVer pre-release tags automatically. +- Decision: Whether alpha tags should be marked prerelease (likely yes). + +Refuted: + +- L17. Snapshot-egress-only deps: freeze, don't polish. The reviewer tied sha2 and tokio-util to download-then-subscribe without reading which routes use them. + +Keep (load-bearing): + +- toolchain-pins.env plus SHA verification of every downloaded input: the CM .deb (install-cartesi-machine/action.yml:38-39), xgenext2fs (setup-guest-toolchain/action.yml:77-79), kernel sha512 (canonical justfile:27-37), guest tools sha512 (canonical Dockerfile:8-13) and the TOFU anvil dump hash (justfile:21-24). These are what make the canonical image and the e2e chain state trustworthy. +- examples/canonical-app/Cross.toml:3 pins the guest cross image by digest; the consensus binary's toolchain must be bit-stable. +- SOURCE_DATE_EPOCH, `xgenext2fs -f` (zeroed timestamps), `--assert-version` and `--assert-rolling-template --final-hash` in the canonical justfile (:43, :62, :84-92). These make the template hash reproducible and catch a wrong emulator early. +- `--locked` on every CI and release cargo call (ci.yml:60,66,78; release.yml:76) prevents silent dependency drift in tested and shipped artifacts. +- default-members = [sequencer, wallet-sequencer] (Cargo.toml:25) keeps plain `cargo build` free of the CM/libclang/libslirp requirement that canonical-test and testsi impose. +- scripts/ci-c-application-smoke.sh builds an out-of-workspace consumer against the C seam, which catches workspace-only assumptions in c-app-sequencer that in-workspace tests cannot. +- scripts/ci-watchdog-docker-smoke.sh exercises the real production entrypoint, the flock contention and the uninitialized-state path inside the shipped image, not just that the image builds. +- c-app-engine generates its FFI bindings with bindgen from the header at build time (bindings/c-app-engine/Cargo.toml build-deps comment): the header is the ABI source of truth, and checking in generated bindings would let them drift. +- libcmt-sys builds a host mock (sdk/guest/README.md), which is why clippy and tests cover guest crates on the host with no RISC-V lint job. +- Workspace alloy feature lists are member-specific and `node-bindings` is dev-only (sequencer/Cargo.toml:41-43). This keeps the Anvil-spawning surface out of production builds. +- E2E failure-log upload (ci.yml:173-180) is the instrumentation the open flake register entry depends on. +- Release SHA256SUMS + release-manifest.json (release.yml:251-267) give operators one artifact that ties the sequencer, the watchdog image and the canonical images to one commit and one set of pins. + +## M. Watchdog (Lua) and guest SDK provenance + +Verdict: The watchdog is a one-shot Lua CLI of about 2,240 production lines plus a 67-line bash wrapper. It replays InputBox inputs on the canonical Cartesi Machine in-process, proves the L1 view complete (contiguous indices plus getNumberOfInputs pinned at B), compares a flat SHA-256 against the sequencer's accepted checkpoint, latches a divergence before collecting evidence, and offers `replay` for recovery. The size fits the essential complexity. No module passes 320 lines, each concept has one home, there is no daemon and no in-tick retry, there are only three failure classes, and the README's "Why it is built this way" names the invariant behind each hardening mechanism. The v2 rewrite (c0bebe8 and later) overtook both backlog notes: AI3 is fixed, and AI2 is down to a small residual in idempotent init. The real problems are small and local: (1) the credential-bearing RPC URL leaks into stderr, last_tick.json and `status` output (http.lua:73), the same shape as an open register entry; (2) config.json mixes deployment identity with endpoint and provider tuning that operators rotate; (3) init's no-config branch deletes an agreed head instead of failing loud; (4) module headers restate the README. Nothing here is superseded by the recovery and client-sync direction. `replay` is the canonical-side input the future shared recovery tool will consume, so keep it, and do not grow application-specific export inside the watchdog. sdk/guest is clearly vendored: libcmt is pinned by tag, commit and tarball sha256 and has update steps, and the Rust crates are an acknowledged in-tree fork. The one gap is that the guest-tools/libcmt version pairing is held only by prose in two files. + + +### M1. L1 RPC URL (often carrying an API key) is embedded in failure messages that reach stderr, last_tick.json and `status` + +`medium · confirmed · +0 lines · S · BC` +Where: `watchdog/http.lua:73`, `watchdog/jsonrpc.lua:36-37`, `watchdog/l1.lua:81-84`, `watchdog/main.lua:271-274`, `watchdog/main.lua:176-182` + +- Claim: On any transport failure (DNS, refused connection, timeout), http.lua:73 formats the full request URL into the error. For the L1 RPC that is the provider URL, and Infura or Alchemy style URLs carry the API key in the path. jsonrpc.lua:36-37 passes the error through unchanged, and l1.lua:83 and l1.lua:205 wrap it as a transient error. main.lua:272-274 then logs it to stderr, main.lua:176-182 persists it in last_tick.json, and main.lua:146 prints it from `status`. This contradicts docs/watchdog/README.md:143-144 ('The L1 RPC endpoint is never persisted'). status.prom is not affected, because metrics.render emits no message (metrics.lua:44-73). The libcurl error text itself names at most the host. HTTP-status failures (jsonrpc.lua:45-48) do not include the URL. +- Fix: Drop the URL from http.lua's message and keep only the method. Callers already add context ('L1 RPC:', 'sequencer :'). If a host is wanted, log scheme://host only. Fix it in the same patch as the Rust register entry. + +### M2. config.json persists rotatable endpoint/provider tuning (sequencer_url, long_block_range_error_codes) next to deployment identity + +`low · partially · -12 lines · S · BC · config` +Where: `watchdog/config.lua:4-7`, `watchdog/config.lua:99`, `watchdog/config.lua:107`, `watchdog/config.lua:123`, `watchdog/config.lua:129` + +- Claim: Two facts hold. init requires CARTESI_WATCHDOG_SEQUENCER_URL (config.lua:99) although bootstrap.run never uses a sequencer (bootstrap.lua:20-62, main.lua:125). The range-error codes are frozen at init (config.lua:107, :129, :148-161), so switching to a provider with a different code means wiping the state directory and re-initializing, per README.md:162-163. But 'two sources of truth' overstates it. The URL is a documented persisted default with a tick-time override (README.md:150, config.lua:154), and the module header says so (config.lua:4-7). The failure mode is also loud and bounded: a missing code only makes wide getLogs ranges fail as transient (l1.lua:196-205), while small ranges keep working, and the default list covers the common providers. Proposed delta is closer to -5 than -12, because the tick needs a required-URL check in its place. +- Fix: Read both from the env at run time (codes keep their default) and stop persisting them. config.load ignores the old fields, so no version bump is needed. config.json then holds pure identity: version, chain_id, app_address, input_box_address, state_source, bootstrap_block. The AI2 element validation goes away with the field. +- Decision: Should the sequencer URL become a required tick-time env var (like the RPC URL), and should provider error codes be tunable without re-init? + +### M3. Idempotent init checks only identity, not the schema of an existing config.json (AI2 residual) + +`nit · partially · +2 lines · S · BC` +Where: `watchdog/bootstrap.lua:23-35`, `watchdog/config.lua:133-163` + +- Claim: The idempotent init branch compares only chain_id, app_address and state_source (bootstrap.lua:23-35, config.lua:134-138) and never runs config.load. The gap is reachable only through a config.json that was hand-edited but is still valid JSON. A torn or corrupt file already fails at bootstrap.lua:23, because read_json raises an operator error (store.lua:119-121). Every value init writes is validated before it is written (config.lua:94-130). A v1 directory gets 'initialized for another deployment ... wipe it to re-initialize' (bootstrap.lua:28-29) rather than the version message (config.lua:145), but both prescribe the same remedy (wipe and re-init). Calling config.load in that branch is still a harmless one-line tightening. +- Fix: In the idempotent branch, call `config.load(store.dir, existing, {})` before comparing identity (one line). Init then fails with the same diagnosis tick would give. + +### M4. init silently deletes an agreed head when config.json alone is missing, which is an impossible state that should fail loud + +`low · confirmed · +6 lines · S · BC` +Where: `watchdog/bootstrap.lua:38-40`, `watchdog/bootstrap.lua:54-57`, `watchdog/main.lua:112-118`, `docs/recovery/cockroach.md:85-97` + +- Claim: With no config.json, bootstrap.lua:38-40 deletes checkpoints/ wholesale on the assumption that any checkpoint came from an interrupted init. It does so before validating anything: the removal at line 40 runs before the chain-id, InputBox, source and trust checks at lines 43-52. An interrupted init can leave only one checkpoint, at its bootstrap block (lines 54-57 publish before config.json is written), and an agreed head always has block > bootstrap_block (tick.lua:72, :106). So a missing config.json beside an agreed head is an impossible state that gets 'handled' by deletion. The code's own diagnostics lead into it. A missing config.json yields 'not initialized; run init' (main.lua:115; operator-deployment.md:395 attributes that message only to an empty directory). operator-deployment.md:250-253 endorses running init before every tick. operator-deployment.md:177 says config.json 'needs no backup', which contradicts cockroach.md:85. Init then exits 0 and silently forces a full replay from the bootstrap snapshot. A CM_SNAPSHOT_DIR pointing inside checkpoints/ is deleted before it is loaded. Safety holds (the re-bootstrap is from a trusted snapshot), and backups kept outside the state directory survive (operator-deployment.md:183-184), so the severity stays low. Better proposal: move the removal after validation; remove only a single checkpoint whose block equals cfg.bootstrap_block; raise an operator error ('checkpoints without config.json; restore config.json or wipe checkpoints/') otherwise, including when bootstrap_dir lies under checkpoints/; and fix the operator-deployment.md:177 wording. +- Fix: In the no-config branch, remove only a checkpoint whose block equals cfg.bootstrap_block. Raise an operator error if any other checkpoint exists. Invariant: an agreed head is deleted only by a newer agreement. + +### M5. Error messages missing context or unbounded: rmdir without a path, full HTTP error bodies + +`low · confirmed · +2 lines · S · BC` +Where: `watchdog/store.lua:66`, `watchdog/tick.lua:108-113`, `watchdog/sequencer.lua:24-26`, `watchdog/jsonrpc.lua:48` + +- Claim: remove_tree's `assert(lfs.rmdir(path))` (store.lua:66) raises only strerror. The vendored lfs remove_dir calls pushresult with a NULL info (luafilesystem/src/lfs.c:617-621, 221-229), so a failed prune (tick.lua:109-113) logs 'internal failure: Directory not empty' with no path. os.remove on line 68 does name the file. Separately, require_ok puts the whole non-2xx GET body into the error (sequencer.lua:25). jsonrpc truncates to 200 bytes (jsonrpc.lua:48), and http:download already caps its error body at 512 bytes (http.lua:113-116). So only the inclusion_block and digest GETs are unbounded. +- Fix: Raise `cannot remove : ` in remove_tree, and truncate the sequencer body to the same 200 bytes as jsonrpc. + +### M6. Module headers and the test-guest README restate docs/watchdog/README.md and the e2e oracle + +`low · partially · -80 lines · S · docs-only` +Where: `watchdog/store.lua:4-21`, `watchdog/tick.lua:4-21`, `watchdog/incident.lua:4-14`, `watchdog/incident.lua:104-116`, `watchdog/sequencer-watchdog:16-28` + +- Claim: The module headers in store.lua:4-21, tick.lua:4-21 and incident.lua:4-14 do restate README sections (README.md:24-48, 165-193, 195-224), and one copy has already drifted. store.lua:18-21 says config.json is 'written once by init, which is re-run after a crash', but a torn config.json makes init itself fail (bootstrap.lua:23, store.lua:119-121), and README.md:184-186 and operator-deployment.md:394 say to wipe first. The wrapper's `case`-vs-`[[` aside (sequencer-watchdog:27-28) is self-evident. Two parts of the proposal are wrong. (1) incident.lua:104-116 is collect()'s parameter contract (the shape of `evidence`), not a README copy, so keep it. (2) The test-guest recipe (test-guest/README.md:60-89) cannot be replaced by e2e.lua. e2e's oracle compares final root hashes only (e2e.lua:84-137), not CLI exit codes 1/7/101 or the inspect-after-exception behavior documented at lines 99-117. The outcome table (lines 93-98) is keyed by the recipe's run names. Also keep the wrapper sentence on why the paths are appended, since it guards against a real regression. A realistic delta is about -40 lines, not -80. +- Fix: Cut each module header to one line plus the one non-obvious invariant and a README anchor. Cut the wrapper comment to one line. In test-guest/README keep the outcome table and the emulator quirks (lines 99-117) and replace the bash recipe with a pointer to e2e.lua. + +### M7. `status` fails entirely on a torn last_tick.json or config.json, right after the host crash when operators most need it + +`nit · partially · +4 lines · S · BC` +Where: `watchdog/main.lua:130-148`, `watchdog/store.lua:113-124`, `watchdog/incident.lua:32-45` + +- Claim: `status` reads config.json and last_tick.json without pcall (main.lua:133, :146), so invalid JSON fails the whole command (store.lua:119-121), while the incident reads degrade gracefully (incident.lua:32-35, 41-44). The window is small. A torn last_tick.json lasts at most one tick period, because tick never reads that file and record() rewrites it on every run (main.lua:176-182). A torn config.json also fails tick and init, and the documented remedy is to wipe the state directory (README.md:184-186). During the window, the latch stays visible through the tick's exit 2 and cartesi_watchdog_divergence_info in status.prom (metrics.lua:60-64). Wrapping the two reads is fine, but this is robustness, not 'tests'. +- Fix: Wrap both reads in `status` the way incident.evidence does, reporting `{unreadable = true}` for that field so the rest of the summary still prints. + +### M8. Provenance gaps: json.lua has no pinned commit, and the guest-tools/libcmt pairing is held by prose in two files + +`low · partially · +8 lines · S` +Where: `watchdog/third_party/json.lua:2-4`, `watchdog/third_party/lua-curl/UPSTREAM`, `watchdog/third_party/luafilesystem/UPSTREAM`, `sdk/guest/README.md:19-28`, `examples/canonical-app/Dockerfile:5-8` + +- Claim: json.lua names only '@ 0.1.2 (commit on master)' plus a local change (third_party/json.lua:3-5), while lua-curl and lfs have UPSTREAM files with a commit or tag (and lfs also a sha256). The guest-tools pin is wider than 'prose in two files'. The version and its sha512 are duplicated in two Dockerfiles (examples/canonical-app/Dockerfile:7-8 and watchdog/test-guest/Dockerfile:8-9), and the libcmt version is repeated in sdk/guest/README.md:17-28, libcmt-sys/Cargo.toml:6 and libcmt-sys/build.rs:5. toolchain-pins.env has no guest-tools entry. scripts/verify-toolchain-pins.sh already checks drift for the other pins (CM version, linux image, kernel), so the proportionate fix reuses it. Add MACHINE_GUEST_TOOLS_VERSION and its sha512 to toolchain-pins.env, then add a few lines to the existing script that require both Dockerfile ARGs to match it (or pass them as --build-arg). This is not new machinery. +- Fix: Add watchdog/third_party/json.UPSTREAM with the upstream commit and a one-line description of the %d change. Move MACHINE_EMULATOR_TOOLS_VERSION (and its sha512) into toolchain-pins.env next to CARTESI_MACHINE_VERSION so one file states the pairing. No verification machinery: updates are rare. + +### M9. Dead and duplicated small helpers + +`nit · unverified-nit · -20 lines · S` +Where: `watchdog/abi.lua:92-97`, `watchdog/machine.lua:196-201`, `watchdog/abi.lua:64-68`, `watchdog/json.lua:6-12`, `watchdog/tick.lua:34-37` + +- Claim: abi.decode_bytes has no caller. machine.sha256 re-implements abi.hex_from_bytes. The json facade's new() returns a shared module. tick.run re-checks the latch that main already checked (main's check exists because config may be unreadable). The timestamp format is written twice in main. tick injects machine.publish into the evidence data table as a capability. +- Fix: Delete decode_bytes. Reuse a single hex helper. Make watchdog/json.lua `return require("watchdog.third_party.json")` (or require it directly). Keep exactly one latch check, the one in main.lua with tests going through main or asserting a precondition. Pass `machine` to incident.collect instead of storing publish in evidence. Reuse now(). + +### M10. Two small places that do not fail loud on malformed L1 data + +`nit · unverified-nit · +3 lines · S · BC` +Where: `watchdog/jsonrpc.lua:14-19`, `watchdog/jsonrpc.lua:87`, `watchdog/l1.lua:35-40`, `watchdog/abi.lua:106-111` + +- Claim: hex_to_integer uses tonumber(s, 16), which in Lua 5.4 silently wraps for more than 16 hex digits and goes negative at or above 2^63. A malformed safe-head or log position can therefore become a plausible number. decode_evm_advance guesses whether a selector is present from length parity instead of checking for EvmAdvance 0x415bf363. +- Fix: Reject more than 16 hex digits or a negative result in hex_to_integer. Require the EvmAdvance selector and raise otherwise. + +### M11. Image-side Lua search-path redundancy + +`nit · unverified-nit · -2 lines · S` +Where: `watchdog/Dockerfile:80-81`, `watchdog/Dockerfile:97-101`, `watchdog/main.lua:13-19`, `watchdog/sequencer-watchdog:22-52` + +- Claim: The image copies share/lua/5.4 into both /usr/local/share and /usr/share. It sets CARTESI_WATCHDOG_LUA_DEPS and LUA_CPATH to the same /opt/watchdog/lib, which main.lua then prepends again. The wrapper's path composition serves the documented local-build (host .deb) deployment and is load-bearing. The image duplicates are not. +- Fix: Drop the second COPY (the wrapper already appends /usr/share/lua/5.4) and drop CARTESI_WATCHDOG_LUA_DEPS from the image ENV, or drop /opt/watchdog/lib from LUA_CPATH. Keep the wrapper logic. docker-smoke guards the change. + +### MS1. `clear` creates a missing state directory, so a mistyped path during an incident reports 'no divergence is latched' + +`low · skeptic-added · +2 lines · S · BC` +Where: `watchdog/sequencer-watchdog:55-57`, `watchdog/main.lua:96-110`, `watchdog/main.lua:150-153`, `watchdog/incident.lua:187-191` + +- Claim: The wrapper runs `mkdir -p` on the state directory for `clear` as well as init and tick, and commands.clear calls open_store(state_dir, true). So a mistyped CARTESI_WATCHDOG_STATE_DIR on `clear` creates an empty directory plus run.lock, and the command fails with 'no divergence is latched'. During an incident this reads as 'already cleared' while the real watchdog keeps exiting 2. It also contradicts main.lua:96-97, which says only commands that write may create the directory so that a mistyped path fails. `clear` only ever acts on an existing incident/ and has nothing to create. +- Fix: Open the store without create in commands.clear, and limit the wrapper's mkdir to init and tick. For clear, fail when the directory is missing before taking the lock. Roughly a 3-line change. + +Keep (load-bearing): + +- Latch-before-evidence ordering (main.lua:203-286, incident.lua:92-102, tick.lua:45-55): exit 2, the watchdog_event line and status.prom must not wait on multi-GiB evidence collection or be lost to a full disk. +- The `incident/` directory as the latch, with an unreadable record still latching as unreadable_marker (incident.lua:28-37): JSON is not fsynced, so the directory and not the file carries the latch. +- L1 completeness proof (l1.lua:164-233): contiguous indices plus getNumberOfInputs pinned at B turn silent log truncation by a provider into a failed tick instead of a false comparison; the split rule is shared with Rust through the partition vector. +- Deriving the InputBox from Application.getInputBox() at init (l1.lua:92-123): a configured address could name another release's InputBox, whose empty history would look complete. +- Clone-per-input executor with the revert-root assertion (machine.lua:203-248): mirrors the reference CLI's --revert-mode=stored and is tested against the CLI oracle (e2e.lua:89-140). +- Target block fixed from /digest before replay (tick.lua:67-74): a long replay never chases a moving target. +- The bootstrap special case in behind() (tick.lua:57-65): tells a never-agreed head from a real regression. +- Head-by-checkpoint-name with sync_stored+rename publish and no pointer file (store.lua:149-176, machine.lua:110-115): a crash leaves either the old head or the new one. +- check_source plus the trusted_start template-hash check at init (bootstrap.lua:51-52, canonical.lua:18-26): this is what closed AI3. +- Three-class errors module (errors.lua): 44 lines, and it maps cleanly to exit 1 with the class named in logs and last_tick.json. +- `replay` (replay.lua): the canonical-side producer of a checkpoint at a chosen block that cockroach recovery uses today and the future shared recovery tool will consume. Keep it, but keep application-specific export out of it. +- The wrapper's flock and Lua path composition (sequencer-watchdog:29-65): the lock serializes writing commands, and the path logic serves the documented host-.deb local-build deployment. +- Vendored native modules compiled from in-tree sources with UPSTREAM files (third_party/lua-curl, luafilesystem): no build-time download. +- The test guest plus CLI-oracle e2e (watchdog/test-guest, tests/e2e.lua): the only proof that the in-process executor matches the reference host semantics. + +## N. Docs corpus and comment verbosity + +Verdict: The docs corpus is about 7,350 lines and 50.8k words (README 569, AGENTS.md 464, docs/** 6,314). Production Rust has 4,958 comment lines against 27,396 code lines. The essential content is real: the duality map, the invariant register, scheduler semantics, recovery and its TLA+ models, the watchdog runbook. But the corpus is about 1.5-2x the size that content needs. Three habits cause most of it. First, the same fact is restated in many places: the fee derivation appears in 7 places, the danger-arm order in 4, the count semantics in 5, and the scheduler-mirror co-location note in 3. Second, essays sit where a sentence should: a 60-line rustdoc on check_danger, threat-model table cells over 400 characters, and 37 of 80 module headers breaking the existing 1-4 line rule. Third, finished material stays in the tree: two dated records whose retirement condition is met, and a 546-line TLA+ model of a rejected design. About 700 lines document the egress sync, cross-generation checkpoints and projection replay, which the planned shared rebuild tool replaces; those should be frozen, not polished. The existing rules on review codenames and history narration mostly hold (no codenames found, about 6 narration phrases), so the drift is about repetition and length rather than tone. AGENTS.md costs about 4.9k words in every session and could be about half that size by keeping the duality map, the fail-loud and complexity principles, Always/Ask/Never, the commands and the Reading Routes, and linking out for everything else. On logs, the sampled error and warn calls carry structured context; the one known secret leak (the RPC URL in the startup log) is already an open register entry. + + +### N1. Freeze the egress-sync / cross-generation / projection-replay docs (~700 lines) + +`medium · partially · -650 lines · M · docs-only` +Where: `docs/protocol/projection-replay.md:1-125`, `docs/plans/2026-07-track3-feed-replay-design.md:1-120`, `docs/review/2026-09-16-track3-validation.md:1-150`, `docs/protocol/application-history.md:70-103`, `docs/protocol/application-history.md:120-178` + +- Claim: The cited docs (~700 lines) do describe the shared-state client sync that the direction replaces: projection-replay.md:1-9 (readers rebuild projections from historical L1 inputs), cockroach.md:140-161 (an indexer replays through the new era's C and rebinds its claim), README.md:271-440 (WS claim plus /history and /historical-l1-inputs), and the Track 3 plan. The freeze posture is right. Two parts of the deletion plan are wrong. (a) docs/review/2026-09-16-track3-validation.md also holds the POST /tx ACK latency regression evidence that the ADR's performance posture relies on (2026-08-authority-boundary-adr.md:186-193). The /tx path is not superseded, so move those ACK numbers to a benchmark record before deleting the file. (b) application-history.md:70-103 (checkpoint compatibility after standard recovery) is replaced by the future sync design, not by the cockroach rebuild tool, because automatic recovery still bumps the generation. The freeze note should also fix coordination-tracks.md campaign item 1 ('Merge the implemented egress API... Bart can then integrate'), which is stale now. +- Fix: Do not edit these docs for verbosity now. In one small docs patch, replace Track 3's table row and section in coordination-tracks.md with 3-4 lines recording the new direction and naming these docs as frozen. Leave the contract docs untouched, since AGENTS forbids amendment banners in current contracts. When the shared rebuild tool lands, delete projection-replay.md, the Track 3 plan and the 2026-09-16 record, cut application-history.md to its coordinates table (about 60 lines), and rewrite README.md:271-440 against the new sync. That sequencing saves about 650 lines without double work. +- Decision: Confirm that Track 3 is formally closed and that the 2026-09-16 latency numbers are no longer needed as the ADR's 'performance posture' regression evidence (docs/plans/2026-08-authority-boundary-adr.md:186-193). + +### N2. AGENTS.md baseline (464 lines, 4,874 words) restates its owners; a lean baseline is ~230 lines + +`medium · partially · -220 lines · M · docs-only` +Where: `AGENTS.md:145-180`, `AGENTS.md:182-209`, `AGENTS.md:211-233`, `AGENTS.md:235-255`, `AGENTS.md:286-320` + +- Claim: AGENTS.md (464 lines, 4,874 words) restates owners in several places. The Application Trait Contract says 'the essentials follow' after naming its owner (AGENTS.md:237), and the module map overlaps README Project Layout. A leaner baseline is reasonable. The proposal would lose facts and links, though. 'Use a dedicated submitter address' and the devnet account-9 fact (AGENTS.md:312-315) exist nowhere else, so they need a new home such as README Running or cockroach.md before Command Configuration is cut. docs/review/register.md:192 (the open OpenAPI-drift entry) links AGENTS.md#http-endpoints for the openapi change-together rule, so turning HTTP Endpoints into a Reading Routes row breaks that anchor and the rule. AGENTS.md:337-340 also endorses an 'intentional baseline' on purpose, so how much to keep is the maintainer's call, not a rule violation. +- Fix: Keep these: Mission plus Requirements (10 lines); Design Principles compressed (12); the duality map with its three implementations and two mechanical facts (25); staleness rule plus the two recovery paths in 8 lines with links; Threat Model brief (8); the fail-loud and comment rules (8); Documentation Practice (12); Shell and Commands (15); Always/Ask/Never (25); Definition of Done (6); Reading Routes (22); a 6-term glossary (chunk, frame, batch, Tip, safe_block, inclusion lane); Type Boundaries (6). Cut the Application Trait Contract to 3 lines plus a link, and Domain Truths to the deposit/rejection/nonce one-liners. Keep the module map here and delete README's copy (see N11). Command Configuration and HTTP Endpoints become a Reading Routes row each. This saves about 2.4k words per session. +- Decision: Is the module map baseline material (keep it in AGENTS) or reachable via README and module headers (cut it)? Recommend AGENTS, since agents need it first. + +### N3. Fee derivation / oracle policy is stated in 7 places; batch.rs carries a 33-line fee essay + +`low · partially · -45 lines · S · docs-only` +Where: `sequencer-core/src/batch.rs:7-39`, `README.md:501`, `AGENTS.md:190`, `AGENTS.md:218`, `docs/threat-model/README.md:24` + +- Claim: The real duplicate is the 33-line header at sequencer-core/src/batch.rs:7-39. It restates, nearly line for line, the schema comment block at 0001_schema.sql:540-562: the formula, log base, oracle conversion and tenfold slack. The schema also repeats the tenfold slack at :578. README.md:501 and AGENTS.md:218 add clause-level repeats. fee.rs:226 is not a restatement: it tells SUGGESTED_MAX_FEE_SLACK apart from log_slack, a non-obvious note worth keeping. AGENTS.md:190 is already a one-line glossary entry with an owner link. Realistic saving is about 30 lines, which makes this low, not medium. Track 5 (coordination-tracks.md:16) notes that log-space fees may become defunct, which argues for trimming to a pointer rather than polishing. +- Fix: Pick two owners: the schema comment beside batch_policy_derived owns the formula, and the threat-model oracle row owns source trust and outage policy. Example rewrite of the batch.rs header: `// Batch sizing and frame fees are log-space DA charges; the formula and\n// its knobs live with batch_policy_derived in 0001_schema.sql.` Cut README.md:501 to one line plus a link, and drop the log_slack clause from AGENTS.md:218. + +### N4. check_danger rustdoc is 60 lines, states the arm order twice, and duplicates the recovery README + +`medium · confirmed · -52 lines · S` +Where: `sequencer/src/storage/recovery.rs:173-232`, `docs/recovery/README.md:130-149`, `docs/invariants.md:152-158`, `docs/invariants.md:459-461` + +- Claim: The arm order is documented in the rustdoc as a numbered list (1-6), then as a 'Returns the first variant' chain, then as a five-bullet 'trust hierarchy'. Clock faults are explained twice inside the same comment. The recovery README already owns the order, names check_danger_in, and gives the rationale; I4 and Do-not-simplify repeat it again. +- Fix: Rewrite to: `/// Classify danger in one read transaction. Arms fire in order: divergence,\n/// stale view, closed batch, Tip, clock fault, estimate. The order is the\n/// dispatch policy (I4; docs/recovery/README.md "When safe-head progress stops").\n/// now_ms is injected so storage tests need no clock mocking.` + +### N5. Module-header rule (1-4 lines) is broken by 37 of 80 headers, 24 of them over 10 lines (606 lines total) + +`medium · confirmed · -250 lines · M` +Where: `AGENTS.md:328`, `sequencer/src/storage/mod.rs:4-29`, `sequencer/src/commands/run/workers.rs:4-26`, `sequencer/src/storage/lifecycle.rs:4-25`, `sequencer/src/l1/submitter/worker.rs:4-25` + +- Claim: Counts reproduced exactly: 80 files with //! headers, 37 over 4 lines, 24 over 10, 606 lines in total. One correction makes the case stronger. storage/mod.rs:6-25 does not list 'all 13 sibling modules': the crate declares 15 (storage/mod.rs:31-45), and the header leaves out `open` and `safe_accepted_batches`. The enumerated list has already drifted. Apart from workers.rs, the convert.rs header (24 lines) also states a real governing posture: contract-bound conversions fail loud, clock and query bounds saturate. Compress it, don't drop it. +- Fix: Run one pass over the 24 headers longer than 10 lines and cut each to its purpose and governing constraint, about 4 lines. Keep the non-obvious build-enforcement note in workers.rs:20-26, which justifies the explicit shape and is cut-resistant; compress it to 3 lines. For storage/mod.rs, keep 'one file per writer role; schema and valid_* views in migrations/0001_schema.sql' and delete the file list. Do this as its own docs-only patch. + +### N6. Scheduler-mirror co-location note appears 3 times; 'orthogonal parameter' trivia twice + +`low · confirmed · -32 lines · S` +Where: `sequencer/src/storage/safe_accepted_batches.rs:91-130`, `sequencer-core/src/protocol.rs:344-347`, `sequencer-core/src/protocol.rs:399-422`, `docs/invariants.md:110-114` + +- Claim: Three places explain why populate_safe_accepted_batches keeps an inline nonce advance instead of advance_expected_batch_nonce, and one of them narrates a past refactor. Two places explain why batch_submitter is a parameter rather than a field, which is not a contract. +- Fix: Cut the populate_safe_accepted_batches doc to about 8 lines: `/// Mirror scheduler acceptance over new submitter inputs into safe_accepted_batches;\n/// a foreign or byte-different acceptance records canonical_divergence instead (I9, I15).\n/// The only persistent cursor is the latest accepted row: recovery can reuse a nonce\n/// after rejected rows, so those are rescanned until a later acceptance. The expected\n/// advance stays inline because it interleaves with the divergence check (I1).` Delete protocol.rs:344-347 and :420-422; I1 owns the co-location. + +### N7. 21-line rustdoc on a #[cfg(test)] helper + +`low · partially · -19 lines · S` +Where: `sequencer/src/storage/ingress.rs:116-138` + +- Claim: ensure_open_tip (ingress.rs:116-136, #[cfg(test)] at :137) carries a 21-line rustdoc and should shrink. One sentence must move rather than be deleted. The genesis Tip's leading safe inputs are 'sequenced but not executed', and the lane's catch-up replay executes them, so there is no cold-start drain (:124-126). That fact exists only on this test helper. The production mechanism's doc, open_fresh_tip_in_tx at ingress.rs:468-479, says only 'draining all currently-undrained safe inputs into its first frame', which reads as if they were executed. Move the sentence there. Net saving is about 16 lines. +- Fix: Rewrite to: `/// Test-only: open the genesis Tip on a fresh DB (no-op if a Tip exists).\n/// Production uses ensure_open_tip_for_recovery, which rechecks startup facts.` + +### N8. Retire the 2026-09-18 and 2026-09-19 review records: their retirement condition is met + +`low · confirmed · -158 lines · S · docs-only` +Where: `docs/review/2026-09-18-stack-review-validation.md:8`, `docs/review/2026-09-19-review-followup-validation.md:9`, `docs/review/register.md:145-146` + +- Claim: Both records say 'Retire this record after (stack) landing when no ongoing review decision uses it'. The stack has landed on main, and the only incoming link is register.md:146, which merely points at flake reproduction context. +- Fix: Delete both files (158 lines). In register.md:145-146 replace the link with 'Reproduced 2026-09-18 under a parallel run; isolated and serial runs passed.' + +### N9. docs/recovery/history keeps a 460-line TLA+ model of a rejected design, run by check-all + +`low · partially · -520 lines · S · docs-only` +Where: `docs/recovery/history/README.md:1-77`, `docs/recovery/history/optimistic.tla`, `docs/recovery/history/optimistic.cfg`, `docs/recovery/justfile:12-16`, `docs/recovery/README.md:249-254` + +- Claim: The facts are right: optimistic.tla is 460 lines plus a 77-line README, the justfile's check-all includes the roughly 3-minute check-optimistic (docs/recovery/justfile:12-16), and only recovery/README.md:252 links the history. The justification is wrong in two ways. docs/review/README.md:33 governs review notes, not recovery design docs. And the model is not just a 'rejected design': history/README.md:17-19 records a clean 194M-state check of the optimistic alternative after the Silver-only fix, and :62-70 states the downtime tradeoff that would reopen it. That makes it a revisit-trigger asset in the sense of the 'preserve the reason for a deliberate tradeoff together with the assumptions that would change the decision' rule (docs/review/README.md:41-42). The cheaper option that needs no decision is to drop check-optimistic from check-all (one line) and keep the directory. Deleting it stays the maintainer's call. +- Fix: Inline the roughly 25-line counterexample under 'Why the closed-suffix policy is safe' in docs/recovery/README.md. Delete docs/recovery/history/ and the check-optimistic recipe. Record the removal commit in that paragraph so the model can be recovered with git show. +- Decision: Does the maintainer want to keep the bounded optimistic model as runnable evidence? The memory notes a recovery TLA+ design-doc location, so it may be valued. + +### N10. README environment-variable list is incomplete versus config.rs and contains history narration + +`low · confirmed · -4 lines · S · docs-only` +Where: `README.md:143-146`, `README.md:166`, `README.md:542`, `AGENTS.md:317-320`, `sequencer/src/commands/config.rs:89-101` + +- Claim: README's required/optional variable lists omit the fee-oracle source variables that setup pins, MAX_BATCH_OPEN_SECONDS, FEE_ORACLE_POLL_INTERVAL_MS, and the CHECKPOINT_*/RECOVERY rebuild variables. AGENTS.md:317-320 already says config.rs owns these names. README.md:166 is 'previously/now' narration, and README.md:542 ends in a dangling colon. +- Fix: Keep the two required-variable lines and the example. Replace the optional list with 'All options, defaults and validation: ` setup --help` / `run --help` (config.rs).' Delete line 166 and fix line 542. + +### N11. README duplicates AGENTS (module layout, reading list, test prerequisites); snapshots/README is an index-only page + +`low · partially · -55 lines · S · docs-only` +Where: `README.md:503-534`, `README.md:540-550`, `README.md:552-565`, `AGENTS.md:152-180`, `AGENTS.md:375-379` + +- Claim: README Project Layout (README.md:503-520) overlaps AGENTS Workspace and module layout (AGENTS.md:152-180), and the two have already drifted: README omits canonical-app, canonical-test and sdk. Further Reading (README.md:552-565) overlaps Reading Routes. Two corrections. docs/snapshots/README.md:8-11, the three-boundary paragraph ('Counts and clocks alone do not establish acceptance'), is not repeated in lifecycle.md; a grep finds it only in snapshots/README.md. Fold it, don't drop it. And there is one inbound link (README.md:560), not 3-4. +- Fix: README keeps a 5-line workspace overview plus 'see AGENTS.md#architecture-map'. Replace Further Reading with a link to AGENTS Reading Routes. Fold docs/snapshots/README.md's two useful sentences into lifecycle.md's intro and delete it, updating 3-4 links. + +### N12. Threat-model actor table hides essays in table cells (9 cells, 7.6k characters) + +`low · partially · +0 lines · S · docs-only` +Where: `docs/threat-model/README.md:18-30` + +- Claim: Only 4 of the 9 lines over 400 characters are actor-table cells: lines 22, 23, 24 and 28, about 4.2k characters, with the Uniswap row at 1,475. The other five (lines 43, 81, 85, 113, 115) are one-line prose paragraphs or bullets in Self-trust, In-scope, Out-of-scope and L1 block-time coupling. They are long but are not table essays. The proposal to split the oracle, RPC and InputBox policy out of the cells still stands. +- Fix: Limit each cell to one sentence (trust level plus capability). Move the oracle, RPC and InputBox policies into short '### Fee oracle' / '### RPC endpoint' subsections under 'Actors and trust', keeping the existing anchor target so links keep working. Line count is roughly neutral; words drop about 30%. + +### N13. Fail-loud policy and divergence-check bounds restated across AGENTS, invariants, README; register layout drift + +`low · confirmed · -45 lines · S · docs-only` +Where: `AGENTS.md:331`, `docs/invariants.md:14-68`, `docs/invariants.md:32-59`, `docs/invariants.md:309-367`, `docs/invariants.md:452-473` + +- Claim: AGENTS.md:331 restates the invariants.md check policy in a 10-line bullet. The policy section itself carries three admission/maintenance/selection bullets (28 lines) that restate the authority-boundary ADR, which they cite as owner. README.md:50-61 restates I9/I15's bounds. I15 (59 lines) has grown four sub-essays: runtime reaction, race bound, watchdog boundary, baseline-aware frontier. The Do-not-simplify list sits between I18 and I19 instead of after the register. +- Fix: Cut AGENTS.md:331 to 'Impossible states fail loud; never handle or absorb them. Decision test: docs/invariants.md.' Reduce invariants.md:32-59 to one line linking the ADR. Cut README.md:50-61 to 3 lines plus a link. Move Do-not-simplify below I20. Leave I15's substance alone (it is load-bearing) but turn its sub-bullets into terse 'holds/enforced/breaks' fragments. + +### N14. Style-rule tweak to stop the drift: 'one fact, one home' plus size caps + +`low · confirmed · +2 lines · S · docs-only` +Where: `AGENTS.md:328-331`, `AGENTS.md:333-360`, `docs/review/register.md:62-63`, `docs/plans/2026-07-track3-feed-replay-design.md:63-64` + +- Claim: The current rules forbid codenames and narration, and those hold. What drifts is repetition, length and defensive disclaimers. There are 30 'not a proof / does not establish / is not implied' phrases in living docs and the register, such as 'This guard cannot guarantee timely L1 inclusion.' and 'This is reference evidence, not private-engine conformance or a deployment benchmark.' Nothing in the rules caps rustdoc length or forbids restating an owner's lists. +- Fix: Replace AGENTS.md:328-329 with four rules. (1) One fact, one home: elsewhere, name it in a clause and link; never restate its conditions, orderings or lists. (2) Module header at most 4 lines and rustdoc contract at most about 6; longer reasoning goes to the owning doc or invariant, linked. (3) State a non-guarantee once, at its owner, only where a reader would otherwise assume the guarantee. (4) Table cells hold one sentence. Optionally add a warn-only CI awk check for //! headers over 6 lines and /// runs over 20 lines, about 10 lines of script. It protects only doc hygiene, so add it only if the drift recurs after the cleanup. +- Decision: Adopt the caps as review norms only, or also add the warn-only CI check? + +### NS1. flush_and_wait rustdoc (35 lines) duplicates the recovery README's flush section + +`low · skeptic-added · -28 lines · S` +Where: `sequencer/src/recovery/flusher.rs:157-191`, `docs/recovery/README.md:155-174` + +- Claim: The flush_and_wait rustdoc is the third-longest in production code. It restates the completion predicate, the no-op nonce range, the zombie-coverage rationale and the safe-block return contract. docs/recovery/README.md 'Flush the covered wallet slots' already owns all four, with the I14 link. +- Fix: Cut to about 6 lines: `/// Consume every covered wallet slot at safe depth with 0-ETH self-transfers;\n/// returns once Pending <= Safe && Safe >= watermark + 1 (the watermark conjunct\n/// covers zombies the local node forgot; I14). Returns the safe block of that\n/// observation: the caller must not cascade before its re-synced view reaches it.\n/// Procedure and rationale: docs/recovery/README.md "Flush the covered wallet slots".` Leave the loop and the watch-timeout outer retry unchanged; the Do-not-simplify rule on flusher retry loops concerns code, not this comment. + +Keep (load-bearing): + +- AGENTS.md:65-86 duality map (three implementations that must agree, drain attribution, empty-batch rule): the single most important baseline fact for avoiding scheduler/sequencer divergence. +- AGENTS.md:444-464 Reading Routes table: cheap routing that is what allows the rest of the baseline to shrink. +- AGENTS.md:402-427 Always/Ask First/Never: compact, and encodes the ask-first contract areas. +- docs/invariants.md register format (holds / enforced by / depended on by / breaks) and the Do-not-simplify list: the guard against locally sound refactors; move the list but keep its content. +- docs/invariants.md I15 substance (docs/invariants.md:309-367): race bound and watchdog boundary are load-bearing; compress the wording, not the facts. +- docs/protocol/scheduler-semantics.md algorithm steps and the count-transition table (:152-174): the canonical owner of acceptance semantics. +- docs/recovery/README.md 'Why the closed-suffix policy is safe' and the two current TLA+ models (preemptive.tla, admission.tla): the safety argument for flush-before-cascade. +- docs/recovery/cockroach.md checkpoint eligibility, flush and stopping-block sections: safety preconditions that survive the planned shared rebuild tool. +- docs/watchdog/incident-runbook.md and operator-deployment.md: operational procedures; the maintainer wants incident procedures in a runbook with tooling and drills. +- docs/review/README.md lifecycle plus register.md: the process works; it just needs applying (N8). +- docs/review/2026-09-25-sender-index-commit-cost.md: retained measurement with live consumers at register.md:207 and :236. +- sequencer/src/recovery/flusher.rs:157-191 flush_and_wait doc: explains the durable watermark anchor of the loop condition, which is non-obvious and safety-critical. +- sequencer/src/commands/run/workers.rs:20-26: explains that the explicit per-worker shape is compiler-enforced, the reason the components-vec redesign was refuted; trim it, keep the point. +- sequencer-core/src/protocol.rs:353-362 'Deliberately omitted structural rejects': deliberate absence recorded once at its seam, as the doc policy asks. diff --git a/docs/review/2026-10-13-whole-repo-review.md b/docs/review/2026-10-13-whole-repo-review.md new file mode 100644 index 00000000..31dd5795 --- /dev/null +++ b/docs/review/2026-10-13-whole-repo-review.md @@ -0,0 +1,468 @@ +# Whole-repo review, 2026-10-13 + +Read-only review of `1a9615d` (= `origin/main`) through five lenses: +over-engineering, abstractions, correctness, simplification, and taste. The +maintainer's backlog notes were triaged against the code, and the three consumer +surfaces were judged from the consumer's seat: the `Application` trait and C ABI, +the scheduler spec, and the egress API. + +Method: 14 area reviewers, one adversarial skeptic per area, independent traces +for medium- and high-severity correctness claims, a future-direction mapper, +and a synthesizer. No builds or tests were run. Of 182 findings, 69 were +confirmed, 68 partially confirmed, 5 refuted and 4 disputed. The orchestrator +re-checked K1, BS1, CS1, L2, LS1, N6, N9, N10, N13, N14 and FEE1 in the code. +Every finding id resolves in the [findings appendix](2026-10-13-whole-repo-findings.md). + +Retire both files when the waves below land or are re-planned. Open items move +to the [register](register.md). + +## Verdict + +- **Consensus core is sound.** No high-severity bug turned up in gate order, + nonce handling, the drain, the fold or the divergence freeze. +- **Risks sit at boundaries, not in algorithms.** + - The guest's compiled sequencer address is never checked against the pinned submitter (BS1). + - Signature acceptance is defined by alloy, not by the spec (B1). + - The fee floor overcharges at sub-gwei gas (CS1). + - 43 tests pass without running (K1). + - A release dispatch can mislabel artifacts (L2), and the guest kernel's hash is not pinned in-tree (LS1). +- **Over-engineering is local, not structural.** + - Durable leases implement a process-local refcount (I2). + - Fee inverses in the consensus crate have only test callers (C3). + - Benchmark verdict code can never evaluate (K6). + - The poster has a redundant shutdown select (F4). + - The hostile-L1, durability and concurrency machinery is proportionate. +- **Large files are mostly tests.** Copied fixtures (K2–K4, about 1k lines) and + duplicated benchmark tooling (K6/K7, about 0.8k) are the main harvest outside + superseded code. +- **Diagnostics are the weakest lens.** + - Root causes drop out of exit and abort lines (H1). + - The RPC URL leaks on both the Rust and Lua sides (register, M1, G10). + - Retryable conditions log at ERROR on every tick (H3, F2). +- **Docs run 1.5–2× their content.** Facts repeat in 3–7 places, 37 of 80 + module headers break the 1–4 line rule, and AGENTS.md costs about 4.9k words + per session. +- **The future direction supersedes about 7k lines:** projection replay, the + WS/replica bootstrap and `setup --recovery`. Freeze them; most polish findings + there are traps. + +## Plan of attack + +Progress: W1, W2, W3a, W3b and W4 have landed on `claude/repo-code-review-103cc5`. +Each was followed by a devnet and watchdog e2e run and an adversarial review +whose surviving findings were folded into their commits. W3a left the guest +binary byte-identical. W3b, W4 and the Cartesi Machine v0.22.0 bump moved the +template hash, with behavior pinned by consensus vectors; the hashes are in the +[guest image record](2026-10-14-guest-image-refresh.md). The input-reader review's commits (compared against the sling +reader, branch `claude/input-reader-review-fda7ad`) are folded in on top, adapted +to this branch: + +| Their commit | Here | +|---|---| +| `2bfac83` | `fix(l1): split a failed eth_getLogs range on any error` | +| `15f7d0a` | `fix(l1): resend rate limits briefly and never resend log caps` | +| (new) | `test(l1): pin log order and refusal on both sides` | +| `83b9166`, plus `4532b33`'s redirect refusal | `fix(l1): keep the RPC URL out of errors and logs, keep the cause` | +| `1d3bf98` | dropped: `http.origin` leaked text after an `@` in a path or query, and the watchdog already names no URL | +| `19b667c` | `fix(setup): pin the deployment block only once it is safe` | +| `ad80cab` | `fix(l1): cross-check the input payload's chain and app; sharper reader errors` | +| `e626e96` | `docs: record the reader's L1 assumptions and bounded-ingestion design` | +| `e959fc0` | `feat(storage): make safe_inputs append-only` | +| `4532b33` | `feat(l1): separate read and submit RPC providers; pin HTTP/1.1` | + +A log missing its block, transaction or log index is refused on both sides, and +the shared log-sort vector pins it for an absent or null field. A malformed +position is split like a transport failure and reported as `Provider` in Rust, +but refused as transient in Lua; both retry. + +Rules for every wave: behavior changes and refactors ship as separate patches, +frozen code (see [Freeze list](#freeze-list)) gets bug fixes only, and +wire/schema/ABI changes and guest behavior changes wait for the contract wave. A +guest-image move that preserves behavior may land earlier when the consensus +vectors pass unchanged (W3b). + +### Step 0: decisions (settled 2026-10-13) + +1. **Bart: done.** His client treats cockroach recoveries as absent, and it can + keep doing so once the tool exists. The [Track 3 plan](../plans/2026-07-track3-feed-replay-design.md#follow-up-sequence) + and the [tracks board](../plans/2026-07-coordination-tracks.md) still describe + him integrating scheduler replay; correct that text in W1. +2. **Fees stay in log space.** The batch-level fee is deferred together with + batch-scoped replay protection, which would enable it. + - The oracle-side numerics become sequencer-local work: C2, then CS1, in W2. + CS1 relaxes a schema CHECK; approved. + - Track 5 (the fee table) is unblocked but consensus-relevant, so it moves to + W5 with Bart, taking L12 (`build.rs` → checked-in table) with it. + - The NUM note's batch-fee half stays deferred, recorded in + [batch-scoped replay protection](../plans/2026-10-batch-scoped-replay-protection.md). +3. **N8 typed signing: deferred.** Out of the current scope. +4. **Recovery-tool shape: a later dedicated session.** The direction mapper's + sketch is under [Consumer surfaces](#consumer-surfaces). + - Bart's client ignores cockroach recoveries, so a v1 can be the extraction + of `setup --recovery` into a keyless tool that emits only the sequencer + bundle; the client output can wait. + - Whether checkpoints carry scheduler state stays coupled to the fridge bound. + +### W1: correctness and safety (about 18 small patches, no decisions) + +| Item | Change | +|---|---| +| K1 | Integration helpers panic on bind failure instead of silently passing 43 tests. | +| K9 | Make the 429 test deterministic; poll the txpool instead of sleeping 7 s. | +| L2 | Release checks out `inputs.tag` and fails if the tag is missing. | +| LS1 | Pin the guest kernel SHA in `toolchain-pins.env`. | +| LS2 | Mark `-` tags as prerelease. | +| DS1 | Treat `ChannelClosed` during requested shutdown as graceful (exit 0, not 1). | +| G6 | `TipAlreadyOpen` refuses instead of retrying an impossible state. | +| H11 | Idle poll interval rejects 0 (`range(1..)`). | +| H8 | `run`/`flush` refuse a missing data dir before creating it, naming the path. | +| B3 | `fold_replay` asserts every replay and seed input is at or below C. | +| A6 | C adapter maps lifecycle-only statuses from execute/validate/drain to `Internal`. | +| M4 | Watchdog `init` refuses, instead of deleting, an agreed head beside a missing config. | +| MS1 | Watchdog `clear` never creates the state dir. | +| M3 | Idempotent watchdog `init` validates the existing config schema. | +| E3(5) | SDK `subscribe()` gets the same timeout as its siblings. | +| Docs | AS1 (AGENTS.md:216 rejection list), BS1 (spec sentence and deployment checklist), GS1 wording, `lifecycle.rs:237` comment, Track 3 and tracks-board text updated to match step 0. | +| Register | Close N9, N10, N11, N12, N13, N20, FEE1, AI1, AI3 and the stale testsi entry; fold FEE2 into fee-observation visibility; widen the RPC-URL entry to the reader, poster, flusher, oracle and watchdog. | +| N9 | Drop the `docs/recovery/history` model from `check-all`; it stays as a dev analysis tool. | + +Exit: the suite fails on bind error, the release pins the kernel and checks out +the tag, and the register reflects the closures. + +### W2: operator surface (about 8 patches, behavior only) + +- **Logging, one PR.** Embed `{source}` in Display (H1, D1, E6); put paths in + I/O errors (H2); log retries at WARN and verdicts at ERROR (H3, F2); add timing + knobs to the startup event (H10); log post-200 stream failures (I6, M5, C8). +- **RPC URL redaction.** One choke point at the provider boundary (register, M1, + G10). +- **Listener split.** Separate public and internal listeners (N24, register), + with D8 folded in. +- **Fee oracle numerics.** One f64 log expression replaces the integer price + pipeline (C2). Then the signed exponent, so sub-unit gas prices stop flooring + at 1 unit/gas (CS1, schema CHECK change). Keep the cheap NUM comment. +- **Smaller items.** + - SDK HTTPS for ingress only, plus the reqwest/thiserror/toml version alignment (L4). + - Validate batch-open together with the danger timing (register). + - Fix setup's exit classification for plain-setup sites (register). + - Have `flush` wait on slot consumption (G11). + - Record per-batch snapshot timing (I5). +- Exit: a synthetic credential URL never reaches logs or status, every abort line + names its root cause, and the routes are split. + +### W3: harvest (pure refactors, roughly −2.5k lines) + +- **W3a tests and benchmarks, first.** + - Adopt one test-placement convention (N4) with one `test_support` module (K2), one stub app (K3), and recovery_tests fixtures plus the schema tests relocated (K4). + - Drop the benchmark verdict code (K6), then dedupe the tooling (K7), then add DB size and drain time (K8, N17). + - Smaller items: K10–K12, J6 with the wallet business-failure test from the register, J11, F3, F5. +- **W3b production.** + - sequencer-core: B6 (private ProtocolTiming fields), B7, B8, C3 (dead fee inverses), C5. + - sequencer: D2 (test fixtures out of `storage/ingress.rs`), D4, F4, F7, F10, H6, H7, H12, I7, E4 (pure move). + - Examples: J1, J5, J10. + - Build: L1, L3 step 1, L5, L6, L7, L10 (shellcheck, luacheck, `just ci` + removed), L11, L13, L14 (the readme path; the tar and libtest-mimic + hoists). L12 moved to Track 5 in W5. + - Scheduler-code refactors (B6–B8) compile into the guest. Run the canonical + guest tests and the duality test for each, and expect the template hash to + move. + - Renames: rename the submitter's wallet `nonce` to `wallet_nonce` (N14); test literals for MAX_WAIT_BLOCKS (N16). + - From the input-reader integration: `test-sequencer` runs at the default + thread count (N18); a 3xx at setup's chain-id check is a configuration + error (exit 30). + - Added during W3b: the F1 and J9 decisions, the process-lock test flake's + cause (a child spawned by a concurrent test briefly holds a copy of the + lock's descriptor), consensus vectors, the remaining test-only `pub` + storage items behind `cfg(test)`, a Rust 1.99, dependency and nixpkgs + refresh, and a reproducible guest build: cross added `rust-src` only on a + later run, so the template hash depended on build-cache history. + - Also added: action pins checked offline in CI and audited upstream before + each release, the release published with the `gh` CLI so no third-party + code holds `contents: write`, the Node 20 actions bumped, and the runner + image pinned to `ubuntu-24.04`. +- Exit: no `#[path]` test modules, one `test_support`, a benchmark crate of + about 3.3k lines, and no `pub` test-only items in production modules. Test + fixtures live in `test_support` or `storage::test_helpers`; `#[cfg(test)] + pub(crate)` seams on private state and test constructors or doubles beside + their type (the worker test spawners, the poster's test hooks and mock, + `ProcessLock::test` and `acquire_released`, `SafeInputRange::empty_at`, + `DumpInfo::at_baseline`) stay where they are. + +### W4: docs lean pass (docs only, about −1k doc and −350 comment lines) + +- **Order.** + 1. Adopt the style rule first (N14: one fact, one home, plus size caps). + 2. Shrink AGENTS.md to about 230 lines (N2), keeping the account-9 and #http-endpoints anchors. + 3. Trim the 24 longest module headers (N5). + 4. Retire the 09-18 and 09-19 records (N8). + 5. Move the threat-model essays out of table cells (N12). + 6. Then the rest of N3–N13, B5, B10, F9, G7, G8, H13, I8, M6. +- **Porter spec (B11).** About 130 lines: SSZ layout, domain, `UserOp` type + and constants. Describe the signature acceptance set (B1) as currently + implemented and mark it pending; making it normative binds Bart's port (5c). +- Independent docs patches can land beside any wave; the bulk waits for W3 so + the text describes final code. +- **Landed** (about −1.2k net lines: −450 docs, −725 comments, −40 code): + - The style rule heads AGENTS.md's Documentation Practice, as norms with no + CI check. AGENTS.md is 347 lines and 3.6k words (from 478 and 5.1k): the + 230-line target would have deleted rules homed only there. + - The porter spec replaces scheduler-semantics.md (309 → 216 lines), with + the pending rules listed in the register; I1, I3 and I15 tightened. + - Threat-model essays moved into subsections; the 09-18 and 09-19 records + retired; the egress freeze recorded in the coordination plan. + - Module headers within the 1–4-line cap across the sequencer, guest crates + and watchdog, except `fold.rs` (W6) and `sequencer-core/build.rs` (W5). + - `advance_expected_batch_nonce` moved from `sequencer-core` into the + submitter: a resume heuristic, not a scheduler mirror (B5). + - README's Running example uses account 9 and a fixed fee, and its variable + list matches `config.rs`. + - Skipped as already fixed: N7, N9 (W1 kept the model as a dev tool), most of + G7/G8/H13/I8. N1's deletions wait for the recovery tool. The ~22 remaining + non-guarantee phrases follow the new rule when their owner is next edited. + +### W5: contract wave (needs a sync with Bart before any change) + +The sub-batches double as the agenda for that sync; each ships separately. + +- **5a seam v1 (C ABI).** A1, A2, A7, A3 header trim, Track 6 (emit callback, + path suffix, ABI version getter, linker policy), and L10's + `undocumented_unsafe_blocks` lint with its SAFETY-comment decision for the + FFI crates. N8 is deferred. +- **5b batch-format bump (wire).** `[u8; 65]` signature (N7), the full SSZ + overhead fix (register), A8. Drain sealed batches first and land before the + first production deploy; an undecodable batch from the submitter key is + terminal since W3b (F1), so the drain is load-bearing. +- **5c guest-image batch (consensus, with the C++ port).** + - C1 fee rule; B1 acceptance set (document as-is, or narrow); B9 empty batches. + - Track 5 fee table, with L12, which Bart's port consumes byte-for-byte. + - The fridge bound, serializable `SchedulerState` with a "scheduler" inspect + query, and B13 guest reports. + - The BS1 setup cross-check and conformance vectors, extending + `tests/fixtures/consensus_vectors_v1.json` (W3b) as a new versioned file. + - J5's strict-order snapshot decode, which changes what the guest accepts. + +### W6: recovery tool lands (direction; roughly −5k net) + +- **Tool.** New `recovery` crate: move `fold.rs` as a pure move first, then add + an outputs observer and `resume_from`. +- **Import.** `setup --import` keeps the B≤C check, `DeferUntilAnchorSet` and + the rebuilt-anchor test. Delete `setup --recovery` (G1, H5). +- **Supporting moves.** Move `dump_info.rs` into the bundle crate and drop + on-disk `info.toml` (I1), since the tool owns the export format; GS1 + old-watermark input; genesis case (N1, G3). +- **Deletions.** Projection replay (E1) and its docs and tests. +- **One baseline-migration rewrite.** Tool-minted era, drop the cut ledger, + in-memory leases (I2), H14. CS1 lands earlier, in W2, by editing the + baseline migration in place, which the migration policy allows today. +- **After the new sync is designed.** Retire the WS wire format, + `/latest_snapshot` and history headers (I3), the `/history` reshape (E2), + identity types (A4) and `cold_replica`. Keep in-era generation signalling in + some form. + +## Decisions + +Step 0 settled the four that blocked planning. The rest, by the wave they gate: + +| Wave | Decision | Recommendation | +|---|---|---| +| W1 | Release posture (L1): SHA-pinned actions, job-level tokens, no caches in release | Adopt; dispatch rebuilds existing tags only | +| W2 | Retry log level (H3, F2) | Verdict-driven: retries WARN, terminal ERROR | +| W2 | Keep the `terminal_faults` black box (H9) | Keep; fix its forever-warn (HS1) | +| W3b | Own-key undecodable batch (F1, disputed) | Decided: terminal under self-trust (`UndecodableOwnBatch`; `run` aborts with SIGABRT 134, like any terminal runtime fault), after a refactor patch | +| W3b | C-engine PermissionDenied (J9) | Decided: operational (exit 1), so native and C engines agree | +| W3b | Guest-image move outside the contract wave (U0) | Decided: allowed for behavior-preserving commits behind the consensus vectors, as one contiguous batch | +| W3b | Operator-visible renames and exit codes (U3, U4) | Decided: `wallet_nonce` log keys in their own commit; a 3xx at setup's chain-id check exits 30 | +| W3b | CI supply chain (U5, U6) | Decided: every action pinned by SHA; cargo-deny blocks on sources, reports advisories without blocking, no schedule. Dependabot declined: it moves a pin without verifying the new code, so its PRs are either rubber-stamped, undoing the pin, or reviewed as carefully as a manual bump. Pins are bumped deliberately before each release instead; the procedure and this reason live in `scripts/verify-action-pins.sh`. Revisit if pins go stale unnoticed again, the set of actions grows, or someone can own a weekly review | +| W4 | Contract owner: README or `openapi.yaml` (E9) | Moot for the frozen egress half; decide with the new sync | +| W1 | `docs/recovery/history` TLA+ (N9) | Decided: drop it from `check-all`; it stays as a dev analysis tool | +| W3 | CI double run (L3) | Decided: step 1, CI on pull requests plus pushes to `main` and `v*` tags, superseded PR runs cancelled. Nightly runs declined; revisit when one-off PR failures stop being a signal or someone owns a red nightly | +| W4 | Chain-id failover (FS1) | Decided: out of scope per "one consistent node"; revisit when a fallback RPC tier lands. The poster's per-send recheck stays, since a swapped chain would raise the durable watermark | +| W5 | Signature acceptance set (B1) | Documented as pending in the porter spec (W4); narrow to v∈{27,28} and low-s in the port's image. The vector table is pinned (W3b): both paths accept the high-s twin and any v ≥ 35 | +| W5 | Out-of-range frame fee (C1) | Structural `FeeAboveMax` rejection | +| W5 | Zero-frame batches (B9) | Reject only if the port forces a new image anyway | +| W5 | `scheduler_accepts` mirrors gate d (B2, disputed) | Keep the omission; the trace found no reachable case | +| W5 | Batch "nonce" naming (N14) | Not "index": it collides with `batches.batch_index`. If renamed at all, "sequence number", inside 5b | +| — | MAX_WAIT_BLOCKS (N16, H4) | Status quo; a setup flag adds unverifiable drift. Revisit only if a deployment needs a different window | +| — | Flashbots or fanout (N22) | Decline Flashbots; defer fanout; record the reason and a revisit trigger | +| W6 | In-memory leases (I2) | Registry, with a "single serving process" assumption; drop `lease_count` in the rewrite | +| W6 | Fee-observation stamp (register) | Keep the per-poll write (it costs under 0.1%) unless the column goes | + +## Backlog triage + +| Note | Status | Evidence | Next | +|---|---|---|---| +| N1 genesis recovery ergonomics | open | `--recovery` needs a hand-made bundle; the loader already accepts genesis (setup/mod.rs:449-452) | Genesis case in the tool (W6) | +| N2 dump-dir metadata | partially | `next_batch_nonce` is stored 3× (SQLite, `info.toml`, `checkpoint.toml`) | Drop on-disk `info.toml` (I1, W2); one receipt in the tool (W6) | +| N3 snapshot API, mmap/CoW, machine-emulator#398 | partially | The watchdog already uses `rename_stored`/`remove_stored` (machine.lua:114, 233-241); the sequencer's publish point is the SQLite row | Adopt nothing; give Bart a reflink recipe (W4) | +| N4 test layout | done | Seven placement styles; one `#[path]` module (recovery.rs:642) | Rule in AGENTS.md, moves landed (W3a) | +| N5 clippy exclusions | stale premise | CI clippy runs `--workspace --all-targets --all-features` over all 20 members (ci.yml:66) | Close; lint the shell scripts instead (L10) | +| N6 exponentiation/log | partially | TickMath is already f64 (7cfb474); the scheduler table is Track 5 | Log space stays: oracle math in W2 (C2); table in W5 with Bart | +| N7 `[u8; 65]` signature | open | `Vec` (batch.rs:62-68); the DB already enforces 65 | 5b, with the SSZ overhead fix | +| N8 parametric `UserOp` | deferred | EIP-712 has no sum types; `data` signs as hex | Out of scope (step 0); if revived, typed signing via an app hook over opaque transport | +| N9 safe_block ≤ inclusion_block | fixed | Gate d precedes staleness (scheduler/mod.rs:295-317) | Close | +| N10 time-only frame rotation | fixed | Rotation is safe-block distance only (inclusion_lane/mod.rs:246-250) | Close | +| N11 generation on cascade | fixed | Generation advance plus 409 on reconnect (subscribe.rs:58-70) | Close; in-band notice belongs to the new sync | +| N12 paginated replay and offset C | fixed | Pager, per-page revalidation, base K (canonical.rs:47-71) | Close; the historical half is frozen (E1) | +| N13 nested `.DS_Store` | fixed | The unanchored `.gitignore:11` pattern matches nested paths | Close | +| N14 nonce → index | done | Collides with `batches.batch_index` | The submitter's account nonce is `wallet_nonce` (W3b); the batch nonce stays (see Decisions) | +| N16 MAX_WAIT_BLOCKS configurable | done | Single constant (lib.rs:21); never changed | Status quo; tests take it from `sequencer_core` (W3b) | +| N17 benchmark DB size and drain time | done | Only a memory report exists | Reported per sweep step (W3a) | +| N18 cross-compilation and compile times | done | Guest image built twice in CI; 4 duplicate dependency stacks; no profile tuning | One CI run per push (L3), the guest image built once (L7), line-tables-only debuginfo (L13, `target/debug` −22%), parallel `test-sequencer`, one tungstenite stack (W3b) | +| N19 logs under error | done | ANSI and key redaction done; causes now ride in Display, and `error_chain` renders dependency errors' causes; levels follow the verdict | W2 and the input-reader integration | +| N20 hot-path clones | stale premise | No per-op clone that matters; fsync and checkpoints dominate | Close | +| N21 supply chain | done | Deps already hoisted; one unused dep; duplicates; no advisory gate | L4 (W2); cargo-deny (L11), hoists (L14), and held majors with their revisit triggers at their manifest lines (W3b) | +| N22 Flashbots/fanout | decide | The poster reads from the same node; fanout only helps a >4 h propagation failure | Decline; record the reason (W4) | +| N23 conditional snapshots | decide | Exact per-batch snapshots are load-bearing (snapshot_dumps.rs:325-330) | Keep per-batch; add timing (I5) | +| N24 split servers | open | One merged router (http.rs:331-347) | W2 | +| N25 verbose docs | done | Rules exist; length and repetition drift | One fact, one home and size caps adopted, about −1.2k lines (W4) | +| AI1 BUSY/LOCKED classifier test | fixed | Classifier tests exist at every layer | Close | +| AI2 watchdog config validation | partially | Residual needs a hand-edited `config.json` | M3 (W1) | +| AI3 watchdog snapshot validity | fixed | `check_source` plus `trusted_start` (bootstrap.lua:51-52) | Close | +| FEE1 TokenHandle latch | fixed | TokenHandle removed (9a3046f) | Close | +| FEE2 oracle writes per poll | stale premise | Max-age no longer exists; cost under 0.1% | Fold into fee-observation visibility | +| NUM numeric stack | partially | Price math is f64; the scheduler consumes only the published fee | Comment and C2/CS1 in W2, C3 in W3; the batch-fee half is deferred with batch-scoped replay protection | + +Register entries the review re-checked: the SSZ overhead entry is still open (5b). +The testsi cycle-target entry is fixed (1283765; delete it). The RPC-URL entry is +wider than recorded (W2). The listener entry is N24 (W2). The HTTPS/WSS entry +splits: HTTPS for ingress now, WSS dropped. Admission overlap and the setup +misclassification belong in the setup reshape (W6), apart from the plain-setup +sites (W2). + +## Consumer surfaces + +**`Application` trait and C ABI.** The trait is sound for execution: progress is +engine-owned and checked at the shared boundary, and the integer enums refuse +unknown values. The friction is at the seam: + +- Host-only rejection codes appear in the app's vocabulary (A1). +- Engines must fsync a directory the host owns (A2). +- The stateful drain and path callback, and the missing ABI version, are open Track 6 items. +- The contract is written four times (A3). + +Ship one seam-v1 change agreed with Bart. Let the recovery tool, not each engine, +own baseline nonces and pending-direct eligibility; that drops the +nonce-enumeration ABI and the per-engine exporter burden. + +**Scheduler spec.** The algorithm prose is clear and the fold is pure. A C++ port +still lacks: + +- the SSZ layout, the domain and the `UserOp` type; +- the signature acceptance set (B1); +- the sequencer-address provenance rule (BS1); +- the out-of-range fee rule (C1); +- language-neutral conformance vectors. + +Scheduler state (app, queued directs, next nonce) is private, so recovery +re-seeds (A,B] instead of resuming. Direction: a porter spec plus trace vectors +now, and a serializable `SchedulerState` designed together with the fridge bound +in one guest-image change. + +**Egress API.** The premise (client = replica of sequencer state) is wrong for +full-database clients: + +- Snapshot-plus-subscribe loses the client's projection. +- Cross-era recovery makes every client embed the scheduler. +- Messages carry inputs without outputs or outcomes. + +What survives: the ingress routes, `/finalized_state`, `/digest` and +`/inclusion_block` (the watchdog's only dependency), `/finalized_snapshot`, leases, +the canonical page reader with `HistoryClaim` (native restart uses it), the +listener split, and in-era generation signalling. + +The recovery-tool sketch (from the direction mapper): + +- **Inputs:** deployment identity, a checkpoint (genesis or scheduler state at B), + the stop block C from `flush`, and an input source over (B,C]. +- **Outputs:** written file-first, with `receipt.toml` last as the commit point + (tool-minted era, B, C, N', K, digest). One output is a sequencer bundle for + `setup --import`. The other is a versioned client trace of (B,C] with outcomes + and app outputs. +- **Client rule:** a client imports at an era boundary and never subscribes + across one. + +## Correctness findings + +| Id | Sev / status | Where | Scenario | Fix | +|---|---|---|---|---| +| BS1 | medium, single-source, grep-checked | wallet.rs:73,85; setup/mod.rs:580,591 | The guest's compiled `sequencer_address` differs from the pinned submitter, so every batch becomes a direct input. The frontier and fold still report acceptance; only the watchdog notices | Spec sentence and checklist (W1); setup cross-check via inspect (5c) | +| B1 | medium, partially | scheduler/mod.rs:350-358; storage/ingress.rs:723 | A high-s signature is admitted, stored verbatim and accepted by the Rust guest. A spec-following low-s port skips it and diverges | Normative wire/signature section plus vectors (W4); decide narrowing (5c) | +| CS1 | medium, single-source, code-checked | math.rs:33-40; fee.rs:136-138; schema:573 | USDC fee token, ETH at $2,500, gas under 0.4 gwei: floored at 1 unit/gas, so 4× overcharge at 0.1 gwei | Signed exponent plus relaxed CHECK (W2) | +| K1 | medium, confirmed, traced | integration_tests/* (4 helpers) | A sandbox denies bind, 43 tests return early, and the suite shows green | Panic on bind error (W1) | +| L2 | medium, confirmed, traced | release.yml:42-272 | A dispatch from `main` naming a tag publishes main's build under that tag | Check out `inputs.tag` (W1) | +| LS1 | medium, single-source, code-checked | canonical-app/justfile:25-37 | The kernel is verified against a checksum from the same release, so a replaced asset changes the template hash unnoticed | Pin the SHA in-tree (W1) | +| M1 | medium, confirmed | watchdog/http.lua:73; main.lua:146,176-182 | A transport failure writes the RPC URL, with its API key, to stderr, `last_tick.json` and `status` | Redact (W2) | +| C1 | low, confirmed, traced | fee.rs:68-71; scheduler/mod.rs:229-262 | A frame fee above MAX_EXPONENT plus a max_fee=65535 op makes the guest panic deterministically; needs an encoder bug or a stolen key | Structural rejection (5c) | +| A6 | low, confirmed | c-app-engine/src/lib.rs:70-85 | An engine bug returns NOT_FOUND from execute, which becomes non-terminal I/O and a restart loop | W1 | +| G6 | low, confirmed | storage/recovery.rs:115-119 | An impossible lock-violation state is retried and hidden | W1 | +| DS1 | low, single-source | inclusion_lane/mod.rs:143-150,439-443 | SIGTERM races the frontier check, and a clean shutdown exits 1 | W1 | +| GS1 | low, single-source | setup/mod.rs:652; flush.rs:30 | The cockroach flush reads the watermark from the wiped DB, so zombie coverage falls back to the pool view | Docs now; the tool takes the old watermark (W6) | +| B3, H8, H11, M4, MS1, K9, AS1, J9, FS1, FS2, LS2 | low | see appendix | Fail-loud gaps, misconfiguration traps, flaky timing | W1–W2 | +| B2, F1, G4, H4 | disputed | see appendix | A skeptic confirmed each, but an independent trace refuted it | See Decisions | + +## Freeze list + +Bug fixes only until W6 or the new sync design: + +| Item | Size | Fate | +|---|---|---| +| Projection replay: `/historical-l1-inputs`, SDK history, `historical_bootstrap`, `projection-replay.md`, README/OpenAPI sections (E1) | ~1.6k core, ~3.5k with docs and tests | Delete in W6 | +| WS wire format and SDK snapshot→subscribe bootstrap (E3) | ~1.4k | New sync; the page reader and `HistoryClaim` survive | +| `/history` baseline and `accepted_checkpoint` (E2), `/latest_snapshot` and history headers (I3) | ~0.5k | Reshape or delete with the new sync | +| Era/generation identity types (A4), cut ledger and compatibility query | ~0.6k | New sync decides; in-era generation survives | +| `setup --recovery` orchestration, config and errors (G1, H5) | ~0.45k (~1.6k subsystem) | About 150 lines move to the tool; the rest is deleted | +| `fold.rs`; cockroach reader seams (F8); genesis plumbing (J2) | ~0.4k | Move or partially delete in W6 | +| `batch_policy` log-additive design (C6) | ~0.25k | Frozen pending the fee decision | + +Not superseded, despite first impressions: the snapshot lease supervisor (D7), +`/finalized_state` and `/digest` with sha2 (L17), `generation_tests.rs`, and the +engine path callback. + +## Not doing + +Each item was refuted or declined. Re-propose only with new evidence. + +- Prior juries: a JoinSet supervisor, collapsing PreparedRuntime, flattening + RecoveryError, merging BootstrapError variants, a TerminalityOf trait, moving + phase→progress into drive_recovery, a bool `/tx` gate, AppWithProgress, a + components-vec Workers, UniswapConfig::pinned, retain-over-expect, and internal + retry loops. +- This review: + - Dropping the lane's drain cursor (D3): it is an independent I6 witness. + - The flusher's two-call TOCTOU (G5): unreachable under the trust model. + - A reusable dump-contract kit (J7): Track 6 owns it. + - Renaming nonce to "index" (N14). + - A `--max-wait-blocks` flag (N16). + - Flashbots (N22) and sparse snapshots (N23). + - Hot-path clone work (N20). + - The max-age/2 write rule (FEE2). + - Nonce-enumeration ABI exports: the tool owns baseline nonces. + - A closure-based guarded helper (G12): prefer `begin_guarded`. + - A two-connection contention test (AI1): it would test SQLite. +- The input-reader review (sling comparison): contiguity-only chunk commits, + warn-then-error transport logging, a reader bound at `i64::MAX`, HTTP/2 or + keepalive, per-profile timeouts, 60 s or 20 s timeouts, a hash-pinned safe + head, a poster count witness, `getInputHash` as a content witness, and tests + pinning read/submit routing. A separate submit (private relay) URL is + deferred: `create_keyed_providers` is the seam. +- W3b: + - Migrating the vendored guest crates (libcmt-sys, trolley, testsi) to edition + 2024: they compile into the guest and are a documented vendored fork. + - Declaring every internal crate once in `[workspace.dependencies]` (L14b), + and `publish = false` in `[workspace.package]` (L14): each member would + opt in for nothing, since nothing here is published. + - Pointing rollups-types at the workspace alloy entries: a guest SDK crate + needs neither `serde` nor `k256`, and inheritance cannot remove features. + - Dependency majors: tower-http 0.7, bindgen 0.73 and tokio-tungstenite 0.30, + with reasons and revisit triggers at their manifest lines; k256 0.14 and + sha2 0.11 wait for alloy and RustCrypto. +- W4: a CI check for the documentation size caps (norms first; revisit if + drift returns), and bounding archive downloads per connection (the + endpoints are operator-only, and resource exhaustion is out of scope). +- Keep, despite looking heavy: + - Exit-verdict projection, SYNCHRONOUS FULL, and schema triggers and typeof guards. + - Per-batch exact snapshots, GC beyond the frontier, and the I1 duality test. + - The reader's double completeness witness, the watermark-before-broadcast rule, and the stateless submitter. + - The full keep lists are per area in the appendix. Splitting every failed + log range supersedes the reader keep item on configurable range-error + codes, and M2 narrows to the watchdog's persisted sequencer URL. diff --git a/docs/review/2026-10-14-guest-image-refresh.md b/docs/review/2026-10-14-guest-image-refresh.md new file mode 100644 index 00000000..bfae761f --- /dev/null +++ b/docs/review/2026-10-14-guest-image-refresh.md @@ -0,0 +1,87 @@ +# Guest image refresh, 2026-10-14 + +**Consumer:** the release notes of the next canonical guest image, and the next +change that moves the template hash, which compares against these values. +**Retire** this record once that change records its own hashes. + +W3b and W4 of the [whole-repo review](2026-10-13-whole-repo-review.md) moved +the template hash with no intended behavior change: a Rust toolchain and +dependency refresh, guest-crate refactors, and W4's comment edits. Two later +changes ship in the same image: the Sepolia build's dedicated sequencer +address, and Cartesi Machine v0.22.0 with its paired kernel. What moves the +hash, and how a behavior-preserving move is checked, is in +[`docs/cartesi-machine.md`](../cartesi-machine.md#versions-are-all-or-nothing). + +## Method + +- Hashes: `just canonical-print-build-hashes` (devnet) and + `just canonical-build-machine-image-sepolia` plus `cartesi-machine --load=… + --initial-hash` (Sepolia), cross 0.2.5, guest tools 0.18.0. Every row up to + the W4 one uses cartesi-machine 0.21.0 and kernel `6.5.13-ctsi-2`; the last + row uses 0.22.0 and `6.18.56-ctsi-1`, two cold builds agreeing. Every value + below is from a cold build (no `target/` for the guest or its host side), + with `rust-src` in the cross toolchain (see `docs/cartesi-machine.md`). Two + cold builds of the W3b final commit (`756a8ac`) agreed, and so did builds + from three checkout paths. +- Behavior: [`tests/fixtures/consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json), + pinned at `6000e3a` before anything moved and byte-identical at the final + commit. The workspace tests check it natively at every commit, and + `just canonical-test-guest` (15 cases, 5 golden) checks it in the built image + after each dependency step and after the refactors. A + stdlib-only Python reimplementation (Keccak, EIP-712, ABI encoding, fee table, + secp256k1 recovery) reproduced 47 of the pinned values when they were pinned. +- Dev shell: nixpkgs moved from `104240a` (2026-08-03) to `8edc0c7` + (2026-10-08) first; both images were bit-identical before and after, and the + devnet and watchdog e2e suites passed on the new shell. + +## Results + +Devnet initial hash and cycles to the first yield: + +| Step | Commit | Initial hash | Cycles | +|---|---|---|---| +| Baseline, Rust 1.95.0 | `6000e3a` | `0x489e84ce7aaf451f36ebea813b3c1d9b12137ee4b78c91d6f73a1fdac372a180` | 51284983 | +| Rust 1.99.0 | `6009104` | `0x6c323c329e712445d911be8972c92c78808dd4943cbbef9765e4c3370ef00ffb` | 51252817 | +| alloy-core 1.7.3 | `02d8fa9` | `0x94b6a4779094aaea1f5c7adce8b3b8a2fda94bead6ed2802c6312c069be7e9d6` | 51252675 | +| 11 guest-compiled crates | `fb37bc9` | `0xc1fc8f03cfdb69adb1b9c382fddc27ecad383fcf725e3b1a2d986ef1689f5b1f` | 51248352 | +| Guest refactors (B6–J5) | `8d03166` | `0xa006cd4d34685b2804737bcef48e3b398999c30081106a5745216b08fdad223b` | 51222421 | +| 6 updates in the guest's build graph | `711f312` | `0xa4c01357665937d80f006c9c3726c49d0a80fe7aea2d3b8bf2199ea7333d388d` | 51222416 | +| W4 guest comments and the nonce-scan move | `7dc3fb5` | `0xa0c7296d256b401f4103097a3b0b29105b19215f10dde048f400bb3a820277c9` | 51222416 | +| Cartesi Machine 0.22.0 and kernel 6.18 | `11607ff` | `0xcd299178e6e860db692e20267edecf220313393ce23c0103a34b8456f5fb0ae8` | 53967473 | + +The commits before `6009104` (libtest-mimic, tokio-tungstenite, async-trait) +and the remaining compatible updates after `711f312` left every artifact +byte-identical. Among the guest refactors, every commit that touched guest +source moved the dapp binary, including those whose changed code the guest never +calls (B6, C3b, J1); the test-only J5 commit and the manifest commit did not. +`711f312` updates only crates that run on the host inside proc macros and +bindgen, yet moves the binary. W4's guest commits change only comments, apart +from moving `advance_expected_batch_nonce` out of `sequencer-core` into the +submitter; the cycle count is unchanged, but comment edits shift the line +numbers rustc embeds in panic locations, so the image moves. cpufeatures, which +only keccak uses and only on aarch64 hosts, went with the host-only updates: the +x86_64 cross build never compiles it. + +`aa0fc0f` changes only the Sepolia build's compiled sequencer address, so the +devnet dapp and rootfs stay as W4 left them; it was not measured on its own. +`11607ff` changes no guest byte: the devnet dapp and rootfs equal W4's, and the +template moves through the emulator (new machine identifiers and uarch) and +the kernel alone. The v0.22 commits after it change the watchdog, testsi and +docs, none of which reaches the image. + +| Image | Baseline (`6000e3a`) | Final (`11607ff`) | +|---|---|---| +| devnet dapp sha256 | `9415789ca67687000081ef4038cd6ac7b1508c0f9c2d42db81ee9281c492bee8` | `5b896252a4f327d195acd333c4b75ca61485402823e9e6213c10f3f23a11d201` | +| Sepolia dapp sha256 | `ca9fd50eda3773c628ef7d5d47eeba532d1b97c22f7e4f7b0a4f01d986482bd7` | `63d15d391ae42dd63d8e6c3e37c78f814fbae74c3459d6371a8a855d4d2edfd1` | +| Sepolia initial hash | `0xc6fd1515ba68c7256b16e17ced3bd48f9d5ede5c8079c14729f2ef5275f11457` | `0xeaf87115bd3bb9d2a902f07ea63a198626d3c16d943cd9df3dfaffbcac320c7f` | + +`f0d1ccf` is the last commit that reproduces the baseline image. + +## Limits + +The vectors are samples: equivalence beyond them rests on `canonical-test` and +the devnet e2e suite, which runs the Rust and C hosts and the duality test. +Guest report text is not pinned: it is not consensus and may change with the +toolchain. Cycle counts are not consensus either. A warm `target/` built partly +before `rust-src` arrived mixes both kinds of embedded paths and gives a hash +no cold build reproduces; measure hashes cold. diff --git a/docs/review/register.md b/docs/review/register.md index 9d83bc5f..a806c3c6 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -52,16 +52,6 @@ an earlier checkpoint or genesis. The [scheduler contract](../protocol/scheduler and [application contract](../protocol/application-contract.md) own the boundaries the implementation must update together. -### Validate batch-open and danger timing together - -[`RunConfig`](../../sequencer/src/commands/config.rs) accepts any positive -`max_batch_open_seconds`, while `protocol_timing()` validates only the separate -timing fields. Incompatible settings can repeatedly recover a low-volume open -batch before its seal deadline. Next: validate their relationship at the runtime -configuration boundary, with checked units and an explicit submission margin; -test refusal and a valid low-volume closure. This guard cannot guarantee timely -L1 inclusion. - ### Batch-size accounting omits SSZ overhead The lane estimates each operation as `71 + max_method_payload_bytes()`. @@ -78,72 +68,17 @@ Next: compare the intended bound with serialized batches across payload/frame counts, then correct the estimate and check the separately configured `batch_policy.log_user_op_bytes` used for fee accounting. -### Support HTTPS and WSS in the Rust SDK - -The [client constructor](../../sdk/rust-client/src/lib.rs) rejects every HTTPS -endpoint, leaving its HTTPS-to-WSS URL branch unreachable. Next: support both -HTTP/HTTPS and their WS/WSS counterparts, verify the dependency TLS features, -and exercise certificate-verified requests and subscriptions. Preserve local -HTTP use; URL acceptance alone is not evidence that TLS works. - -### Separate public ingress and internal egress listeners - -[`http.rs`](../../sequencer/src/http.rs) serves both routers on one listener. -Publishing that listener wholesale also exposes unauthenticated internal state -and snapshot routes. Next: finish the planned per-side bind configuration and -verify that every public route in the [API contract](../../README.md#api) (today -`POST /tx`, `GET /fee`, `GET /nonce`, and `GET /domain`) is on the public -listener and no internal route is. Keep network access controls as the -deployment boundary; no new authentication subsystem is implied. - -### Fix the pinned test framework's cycle target - -The pinned `testsi::run_machine_increment` passes constant `1 << 28` to an -absolute-cycle-target API. Long runs can stop advancing and report a timeout -caused by the driver. Evidence: [the pinned source](https://github.com/GCdePaula/cartesi-tools-rs/blob/ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22/host/testsi/src/machine.rs#L159) -and the [workspace dependency](../../Cargo.toml). Next: fix the target progression -in that tooling, update the pin, and verify guest progress past the old target -before relying on long-run capacity measurements. This is an upstream tooling -task needed by sequencer validation. - -## Confirmed discrepancies - -### Setup misclassifies some deterministic L1 configuration failures - -Setup wraps reader bootstrap failures as live-worker failures, yielding exit 1; -normal-run startup classifies deterministic reader bootstrap failures as -terminal, exit 30. Malformed RPC URLs exercise this distinction. Discovery -failures such as a wrong InputBox also occur in setup, but run does not repeat -that discovery. The impact is a misleading operator hint in a one-shot command. - -Evidence: [`setup`](../../sequencer/src/commands/setup/mod.rs), -[`InputReaderError::is_terminal_invariant`](../../sequencer/src/l1/reader.rs), -and `classify_input_reader` in [startup recovery](../../sequencer/src/recovery/mod.rs). -Next: classify failures by the setup phase and pin the external exit code; -preserve genuinely transient provider failures. - -### Startup logs the full RPC URL - -The `sequencer startup` event includes `eth_rpc_url` verbatim. Operator URLs can -carry credentials in userinfo, paths, or query parameters; private-key -redaction does not cover this field. - -Evidence: [`commands/run/mod.rs`](../../sequencer/src/commands/run/mod.rs). -Next: omit the field or define a safe endpoint representation, and check -diagnostic/help paths with a synthetic credential-bearing URL. No credential -exposure in an actual deployment was established by this review. - ## Bounded investigations and cleanup -- **Intermittent process-lock test failure.** The macOS workspace suite can - report `Locked` at the final reacquisition in - `dropped_runtime_scope_keeps_lock_until_detached_worker_stops` in - [`workers.rs`](../../sequencer/src/commands/run/workers.rs); an isolated rerun - passes. The worker drops its scope before signalling completion, so a simple - worker-completion race does not explain the failure. Identify any remaining - descriptor/process ownership before changing the assertion or lock behavior. - Reproduced during the 2026-09-18 stack closeout; isolated and full serial - runs passed. See the [current validation record](2026-09-18-stack-review-validation.md). +- **Confirmation watches outlive an RPC outage** (checked 2026-10-13). alloy's + watch timeout reaps only transactions it has not yet seen + (`alloy-provider` `heart.rs`), so if the endpoint dies after a batch + transaction was seen, the poster's watch waits out the outage instead of + failing. The heartbeat logs each failed poll at ERROR, and shutdown still + cancels the wait. Under the fail-stop model this costs nothing, since nothing + useful can happen meanwhile. Revisit on a partial failure where block polling + stalls while the endpoint otherwise serves; the fix is an outer timeout around + the watch. Evidence: [`submitter/poster.rs`](../../sequencer/src/l1/submitter/poster.rs). - **Intermittent C-host recovery WebSocket reset.** At `dc4dd78` on 2026-09-19, `c_host_recovery_after_stale_batches_test` failed an expected message receive with `Connection reset without closing handshake` in the @@ -156,29 +91,37 @@ exposure in an actual deployment was established by this review. timeouts, teardown, or retry behavior. Evidence: the [recovery scenario](../../tests/e2e/src/test_cases.rs) and [WS receive helper](../../tests/harness/src/ws.rs). -- **Transient SQLite contention stops the submitter.** Read handles use a - 50 ms busy timeout; a storage/open failure escapes the submitter loop. - BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing - respawn/recovery under a restarting supervisor. Frequency and benefit of - local retry are unmeasured. Check contention before choosing a bounded +- **Transient SQLite contention stops the submitter and the input reader** + (input-reader half checked 2026-10-13 at `1a9615d`). + Submitter read handles use a 50 ms busy timeout; the input reader opens + writer handles (5 s busy timeout) for its cursor read and its append. In + both, a storage/open failure escapes the worker loop. BUSY/LOCKED are + nonterminal but project to unclassified exit 1, causing respawn/recovery + under a restarting supervisor. Frequency and benefit of local retry are + unmeasured. Check contention for both workers before choosing a bounded retry or timeout change; preserve other errors. Evidence: [`storage/open.rs`](../../sequencer/src/storage/open.rs), [`submitter/worker.rs`](../../sequencer/src/l1/submitter/worker.rs), + [`l1/reader.rs`](../../sequencer/src/l1/reader.rs), [`commands/error.rs`](../../sequencer/src/commands/error.rs). - **Overlapping admission policies.** The generic [`lifecycle` preflight](../../sequencer/src/storage/lifecycle.rs) contains - setup/rebuild branches, but production calls it only for run/flush. - [`setup`](../../sequencer/src/commands/setup/mod.rs) has its own admission - path with an intentionally tested already-complete no-op. Consolidate or - narrow the unused branches when next changing admission; there is no + setup/rebuild branches, but production calls it only for run/flush; setup + and rebuild reach the shared completion rule only inside + `complete_baseline_setup`. [`setup`](../../sequencer/src/commands/setup/mod.rs) + has its own admission path with an intentionally tested already-complete + no-op. Consolidate when the shared recovery tool reshapes setup; there is no demonstrated conflicting live route. - **Fee-observation visibility.** [`log_gas_price_updated_at_ms`](../../sequencer/src/storage/fee_oracle.rs) is persisted, with no production reader. Decide whether operator SQL inspection suffices or a real consumer needs exposure before adding an endpoint or removing the stamp. The accepted - [oracle outage policy](../threat-model/README.md#actors-and-trust) is an - economic tradeoff, independent of whether a health field exposes age. + [oracle outage policy](../threat-model/README.md#fee-oracle) is an + economic tradeoff, independent of whether a health field exposes age. The + oracle rewrites the stamp on every successful poll, unchanged price included, + so a stale stamp means failing polls. That is one FULL-sync commit per + successful poll, an estimated (unmeasured) cost under 0.1% of writer time. - **Recovery across external effects.** Model/storage tests do not replace process-level zombie re-injection or a restart between flush completion and cascade commit. Before adding harness machinery, identify the missing @@ -240,6 +183,48 @@ at `6b2a233`. size and unmeasured. Evidence: [`ingress/api.rs`](../../sequencer/src/ingress/api.rs), [`storage/queries.rs`](../../sequencer/src/storage/queries.rs). +- **Per-batch snapshots run on the lane.** Every batch close writes its + snapshot before the sealing commit, so admission waits for + `Application::create_dump` and the sequencer's fsyncs. Closes fire on the fee + policy's byte target or the batch-open deadline, not on L1 posting, so under + load several land per L1 block and the stall grows with throughput over target + size. Exact per-batch snapshots stay: restart, acceptance comparison, and + recovery select the exact batch's snapshot + ([lifecycle](../snapshots/lifecycle.md#creation-and-restart)). For a + large-state native engine, the mitigation is a `create_dump` that clones files + on a copy-on-write filesystem (XFS with `reflink=1`, btrfs, APFS); see the + [clone costs](../cartesi-machine.md). Unmeasured beyond the batch-close + event's `snapshot_ms`. Revisit, per the + [capacity assumption](../protocol/application-contract.md#5-operational-capacity-for-l1-reconciliation), + when its p99 on deployment storage exceeds about 250 ms or retained snapshots + exceed the deployment's disk budget. Evidence: + [`inclusion_lane/snapshot.rs`](../../sequencer/src/ingress/inclusion_lane/snapshot.rs) + (added 2026-10-13 at `462e4d1`). +- **Bounded ingestion for syncs without a lane** (checked 2026-10-13 at + `1a9615d`). A reader tick fetches `[cursor, safe]` whole, holds every event in + memory (decoded event plus raw log, roughly twice the input bytes; + unmeasured) and commits once; one failed single-block `get_logs` restarts the + span. In `run` the span stays under `l1_read_stale_after_blocks`. Setup, + startup syncs after downtime and `setup --recovery` have no bound: a fresh + database scans every input since the app's deployment, our batches included + (the InputBox caps an encoded input at 64 KiB). Current histories are + megabytes, and rerunning on a larger host is the fallback. The known + mechanism commits per block window without archive state. When the received + indices are contiguous from the stored count and the last one sits at block + `b`, the stored and received inputs in blocks before `b` are complete, + because InputBox indices follow L1 order. Commit those at head `b - 1`, with + that block's header timestamp and without refreshing `synced_at_ms` (I12), + and hold the inputs at `b` for the next window; an empty window commits + nothing; the window that reaches the sampled safe head keeps today's count + check; reject logs outside the requested window; a later failure still + returns an error. A window end is never committed on contiguity alone: the + lane would stamp frames past a dropped tail input. The recovery fold still + loads every input after its checkpoint. Act when `safe_inputs` payload bytes + approach a quarter of the recovery host's memory, when a recovery sync fails + on a transient mid-crawl, or when the reader becomes a crate shared with + another node. Evidence: [`l1/reader.rs`](../../sequencer/src/l1/reader.rs), + [`l1/partition.rs`](../../sequencer/src/l1/partition.rs), + [`storage/l1_inputs.rs`](../../sequencer/src/storage/l1_inputs.rs). ## Verification gaps @@ -253,8 +238,8 @@ at the smallest useful boundary when working on that behavior. | Canonical divergence | Storage freeze and startup refusal are covered separately. Compose accepted divergent input, runtime stop, and refusal after respawn in a process test; frontier must remain frozen. See [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). | | Rebuilt anchor | Anchor unit mechanics and rebuild round-trip are covered. Exercise a later full-tear cascade after a nonzero-anchor rebuild and verify submission resumes at that anchor. See [I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce). | | Same-block directs | `multi_deposit_reconciliation_test` accumulates directs in separate blocks. Queue portal sends, mine once, assert equal receipt blocks, and verify canonical/WS order and attribution in the [E2E scenarios](../../tests/e2e/src/test_cases.rs). | -| Wallet business failure | Mixed replay is covered. Pin insufficient transfer/withdrawal amounts after successful fee validation: fee/nonce/progress advance, the transfer/withdrawal has no further effect, and replay agrees. See the [wallet implementation](../../examples/app-core/src/application/wallet.rs) and [Application contract](../protocol/application-contract.md#2-replay-safety--rejection-inclusion-and-failure). | | Boot failure exits | Exit 20 now has a process assertion; exits 40/1 still need exact assertions in applicable setup/failure scenarios, so the supervisor receives the intended recovery/retry hint. Start in the [E2E scenarios](../../tests/e2e/src/test_cases.rs) and [exit contract](../../sequencer/src/commands/error.rs). | +| SDK HTTPS | A test pins that `https://` endpoints open with a TLS ClientHello. Certificate rejection is untested; it relies on reqwest's rustls platform verifier, with no override in the SDK. Add a rejection test if the SDK gains custom roots. See the [client](../../sdk/rust-client/src/lib.rs). | | Uniswap-mode boot | Source-boundary tests pin setup validation, lazy runtime refresh, and transient quote retention. Existing sequencer E2Es use fixed mode. Decide whether a mock-pool boot scenario warrants its harness cost when changing oracle integration. | Warm restore already has state/cursor agreement tests and snapshot downloads @@ -262,6 +247,34 @@ have lease/GC coverage. If restart cost becomes a requirement, add an assertion that distinguishes restoring a dump from a correct but expensive genesis replay; no new snapshot lifecycle is implied. +## Canonical rules open for the C++ port + +The [scheduler spec](../protocol/scheduler-semantics.md) marks these rules +**pending**. Each changes the canonical machine image or its wire format, so +they are agreed with the port's author. The wire-format bump ships first, on +its own, after draining sealed batches and before the first production deploy; +the rest ship in the port's image. Checked 2026-10-14 at `32bf8de`; +evidence: the spec's links and +[`consensus_vectors_v1.json`](../../tests/fixtures/consensus_vectors_v1.json). + +- **Signature acceptance set.** The guest and `POST /tx` accept every `v ≥ 35` + and the high-`s` twin, as alloy does. Proposed: `v ∈ {27, 28}` and low `s` + only. +- **Out-of-range frame fee.** A `fee_price` above `MAX_EXPONENT` halts the + machine once an op reaches the wallet's fee conversion. Proposed: reject such + a batch structurally. +- **Empty batches** are valid at any age and consume a nonce, though the + sequencer never sends one. Reject them only if the port forces a new image + anyway. +- **Wire format.** A fixed 65-byte signature field, with the + [SSZ overhead](#batch-size-accounting-omits-ssz-overhead) fix; drain sealed + batches before the switch. +- **Riding the port's image:** the + [fee table](../plans/2026-07-coordination-tracks.md#track-5--fee-exponentiation-lut), + the [fridge bound](#bound-the-canonical-direct-input-fridge), and a setup check + of the guest's compiled `sequencer_address`. With them come versioned gate + and signature vectors, adding the unpinned `s ≥ n` and `v` from 29 to 34. + ## Integration work owned elsewhere - [Track 3 follow-ups](../plans/2026-07-track3-feed-replay-design.md): diff --git a/docs/snapshots/README.md b/docs/snapshots/README.md deleted file mode 100644 index 63eb0de5..00000000 --- a/docs/snapshots/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Snapshots - -Application snapshots are immutable, durable copies of application state at a -known execution boundary. They let the inclusion lane resume with load and -replay. A snapshot may contain optimistic state; L1 acceptance determines which -artifact can back a canonical comparison or recovery export. - -Keep three boundaries separate: the engine embeds its executed-input count and -safe-block clock; SQLite associates an artifact with a local batch or era -baseline; safe L1 acceptance selects a comparison checkpoint. Counts and clocks -alone do not establish acceptance. The -[history guide](../protocol/application-history.md) explains these coordinates -and snapshot-plus-replay bootstrap. - -- [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle) - — engine dump methods, durability, immutable artifacts, independent restore, - and the canonical comparison file. -- [Wallet format](format.md) — the wallet's layout, deterministic SSZ encoding, - and decode rules. -- [Lifecycle](lifecycle.md) — creation at batch close, restart selection, - acceptance-derived comparison checkpoints, recovery exports, retention, - download leases, and crash safety. Acceptance is a separate durable fact; - artifacts are never promoted or rewritten. - -For automatic startup repair, see [automatic recovery](../recovery/README.md). -For rebuilding after database loss or a sequencer bug, see -[cockroach recovery](../recovery/cockroach.md). The root -[README](../../README.md) owns endpoint shapes. diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index 68f409f4..ddc4779b 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -53,16 +53,15 @@ metadata come from the sequencer; see [application history](../protocol/applicat ### Determinism -`WalletApp` stores balances and nonces in `HashMap`s, so iteration order -is nondeterministic. Before encoding: +`WalletApp` keeps balances and nonces in `BTreeMap`s keyed by `Address`, +ordered by the 20 address bytes compared lexicographically, so the encoder +emits: -- `balances` entries are sorted ascending by `address` -- `nonces` entries are sorted ascending by `address` +- `balances` entries in ascending `address` byte order +- `nonces` entries in ascending `address` byte order This guarantees byte-identical snapshot files for byte-identical logical -state, regardless of insertion order. Tests assert this property by -calling `create_dump` twice on the same `WalletApp` and comparing the -resulting state files for byte equality. +state, regardless of insertion order. ### Decode Rules @@ -74,8 +73,8 @@ The decoder rejects: - Zero `executed_input_count` with a nonzero `last_executed_safe_block`. Duplicate checks prevent an entry from silently overwriting another during -restore. The decoder accepts unique entries in any order; encoding the restored -state sorts them. Deterministic emitted bytes do not imply that the decoder +restore. The decoder accepts unique entries in any order; re-encoding the +restored state emits them in address order. Deterministic emitted bytes do not imply that the decoder accepts only that ordering. ## Versioning diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 903c3cec..37119dab 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -4,6 +4,12 @@ Snapshots preserve application state at a declared execution boundary. Their contents and boundary are immutable. L1 acceptance is a separate durable fact: there is no promotion operation and no mutable finalized pointer. +Keep three boundaries separate: the engine embeds its executed-input count and +safe-block clock; SQLite associates an artifact with a local batch or era +baseline; safe L1 acceptance selects a comparison checkpoint. Counts and clocks +alone do not establish acceptance. The [history guide](../protocol/application-history.md) +explains these coordinates. + ## Artifact and boundary Every artifact is a sequencer-owned directory: @@ -35,10 +41,13 @@ an application count, and replacement branches, which can reuse a nonce. ## Creation and restart -Initially every batch close creates a snapshot. The lane makes the artifact +Every batch close creates a snapshot. The lane makes the artifact and metadata durable **before** one transaction seals the batch and registers its snapshot. A failed database commit leaves an orphan directory; startup sweeps it. A committed close therefore has its required snapshot. +Creation runs on the lane, so user ops queued behind a close wait for it. Each +close logs the creation time as `snapshot_ms`, at DEBUG, or at WARN once it +reaches half the `POST /tx` acknowledgement budget. Setup similarly makes the baseline artifact durable before publishing the complete era baseline, recovery root when applicable, and setup-completion @@ -108,8 +117,7 @@ archive. Supply that extracted directory to `setup --recovery`, with the checkpoint block recorded in its receipt. The loader checks the receipt against the immutable dump metadata and configured block. Copying a bare local `dumps//` directory is insufficient: it is a restore artifact and has no -acceptance receipt. Export creates the receipt without modifying local files; -startup no longer stamps or repairs acceptance metadata in `info.toml`. +acceptance receipt. Export creates the receipt without modifying local files. ## Retention, leases, and crash safety @@ -136,13 +144,12 @@ of all checkpoint resources. Filesystem deletion failure leaves a harmless orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing state and is forbidden. Startup clears leases left by the dead process, checks the rollback checkpoint's `info.toml` and format version, collects obsolete -rows, and sweeps orphan directories before workers start. On supported Linux and -macOS hosts, the sweep reads every retained artifact's filesystem object identity -(device and inode) before deleting any orphans. Comparing those identities -preserves alternate spellings, symlinks, and mount aliases of the same artifact. -Retained identity lookup failures stop the sweep before deletion: missing -references or structural I/O errors are terminal; operational I/O errors retain -their normal classification. Application restoration runs afterward in the launched inclusion -lane, before processing new user ops; the metadata check does not validate the -application bytes. Missing or corrupt referenced artifacts also fail loud when -read or restored. +rows, and sweeps orphan directories before workers start. The sweep matches +artifacts by filesystem identity (device and inode), not by path, because setup +and run can spell the same directory differently; it resolves every retained +artifact before deleting anything and stops if one cannot be resolved +([`startup_hygiene.rs`](../../sequencer/src/commands/run/startup_hygiene.rs)). +Application restoration runs afterward in the launched inclusion lane, before +processing new user ops; the metadata check does not validate the application +bytes. Missing or corrupt referenced artifacts also fail loud when read or +restored. diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 1641ee5f..2cd9aa53 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -17,17 +17,92 @@ What we are protecting: | Actor | Trust | Capabilities | |-------|-------|--------------| -| InputBox contract | Trusted | Authenticates `msg_sender` on `addInput`. Must be rollups-contracts **v3+**: the reader anchors its scan at the application's deployment block, sound only because the v3 InputBox reverts `addInput` for not-yet-deployed apps. Bootstrap witnesses this via `version()` and refuses pre-v3 boxes. Use correctly; do not model forgery. | -| Our Ethereum node | Trusted, fail-stop | Inside our infra. May become unreachable; will never lie. | -| RPC endpoint (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`) | Trusted, fail-stop — **must be one consistent node** | The code supports exactly **one** endpoint, shared by reader, submitter, poster, flusher, and fee oracle; no fallback tier exists yet. Behind a load-balanced fleet, lagging replicas can silently truncate `get_logs` ranges and desynchronize the flush/re-sync views. The reader fails loud on an incomplete InputAdded set: a per-app index contiguity check (right prefix) plus a `getNumberOfInputs` count witness pinned at the scanned safe block (complete prefix), so a dropped/clamped/truncated-tail input is detected before the safe head advances rather than silently skipped, and recovery refuses if the re-sync lags the flush view. (The count witness is fetched before the scan; when it matches the stored input count the reader advances the safe head without a `get_logs` crawl — same trust base, since a fail-stop node's pinned-block count cannot understate without lying.) The residual fleet exposure is a node that lies *consistently* about both its logs and its input count — outside the fail-stop model. A semi-trusted fallback tier (Infura/Alchemy) is future work. | -| Operator env / CLI flags | Trusted, **mistakes foot-gun-guarded** | Setup configuration is authoritative — including the reviewed Uniswap V3 WETH/fee-token pool the fee oracle quotes. The complete source is pinned in deployment identity; run cannot replace it. The operator is trusted, not infallible: the supported operator-mistake class is *accidental concurrent or stale use of one data directory* — two processes on one dir (kernel process lock), a mistyped `--data-dir` (open refuses paths with no database). Deliberate operator subversion, copied-directory coordination, and distributed fencing remain out of scope (the lock is a cheap local foot-gun guard, not distributed fencing — [ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership)); mechanisms defending this class are judged against this boundary rather than re-litigated per review. | -| Uniswap V3 pool (fee oracle) | Semi-trusted L1 state | Spot manipulation of a deep pool is mitigated by a 30-minute TWAP plus 10× slack in `batch_policy.log_slack`. Residual risks: TWAP lag during real moves and thin/wrong pool misconfiguration. Multi-hop pricing is out of scope. Setup writes the first Uniswap quote under the same hard L1 requirement as the rest of setup; a failed quote leaves setup incomplete. `run` does not gate recovery or admission on another quote: it starts from the persisted `batch_policy.log_gas_price`, constructs the source from setup-validated identity without RPC, and launches a refresher that logs/retries transient quote failures indefinitely while retaining that price. `log_gas_price_updated_at_ms` records successful observation for telemetry; it is not an expiry gate. A shared-endpoint outage or stale view is already caught by safe-head progress, while a pool/`observe`-specific failure with a healthy input reader is accepted as an unbounded economic residual: stale-low pricing can subsidize DA/weaken the fee spam barrier and stale-high pricing can reject users, but neither changes canonical execution because the frame's persisted fee is immutable and enforced by both sides. The 10× slack is a margin, not a proof against arbitrary market movement. Deterministic setup-time source misconfiguration (`WrongTokenPair`, `MissingPoolCode`, chain-id mismatch), fatal arithmetic, and persistent storage faults remain terminal. | -| Batch-submitter private key | Private | Held in operator infra. Not reachable by the network. | -| Sequencer's own code | Trusted during normal operation | Tests/review prevent bugs; runtime invariant checks fail loud. Bugs require diagnosis and correction, followed by an operator rebuild if local state cannot be trusted. See "self-trust" below. | -| **L1 mempool and block builders** | **Fully adversarial** | May reorder, delay, drop, or selectively include submitted transactions. Private mempools mean "dropped" is indistinguishable from "delayed indefinitely." | -| HTTP clients at `POST /tx`, `GET /fee`, `GET /nonce`, and `GET /domain` | Untrusted | Arbitrary public callers. May submit malformed, malicious, or replay payloads. The `GET` routes are intentional public reads: the open-frame fee, the signing domain, and any sender's soft-confirmed nonce. The nonce is a live per-address count of included ops, published before their batches reach L1 (up to `max_batch_open`, 2 h by default); only the egress feed carries it otherwise. It reveals activity timing, not op contents, and moves only after the op is sequenced, so a watcher cannot front-run the op it counts. | -| WebSocket subscribers at `/ws/subscribe` | Internal, but untrusted for data-exposure | Intended for internal indexers. Treat as public for what is exposed. | -| Direct-input senders on L1 | Untrusted | Arbitrary L1 accounts calling InputBox. May submit any calldata. | +| InputBox contract | Trusted | Authenticates `msg_sender` on `addInput`; must be rollups-contracts v3+ ([InputBox version](#inputbox-version)). | +| Our Ethereum node | Trusted, fail-stop | Inside our infra; may become unreachable but never lies. | +| RPC endpoint (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`) | Trusted, fail-stop — **must be one consistent node** | One endpoint serves every L1 client, with no fallback tier ([RPC endpoint](#rpc-endpoint)). | +| Operator env / CLI flags | Trusted, **mistakes foot-gun-guarded** | Setup configuration, including the reviewed fee-oracle pool, is authoritative ([Operator mistakes](#operator-mistakes)). | +| Uniswap V3 pool (fee oracle) | Semi-trusted L1 state | Prices L1 gas in the fee token; a stale or skewed price is an accepted economic risk ([Fee oracle](#fee-oracle)). | +| Batch-submitter private key | Private | Held in operator infra, out of the network's reach. | +| Sequencer's own code | Trusted during normal operation | Tests and review prevent bugs; runtime invariant checks fail loud ([Self-trust](#self-trust)). | +| **L1 mempool and block builders** | **Fully adversarial** | May reorder, delay, drop, or selectively include submitted transactions; with private mempools, "dropped" is indistinguishable from "delayed indefinitely". | +| HTTP clients at `POST /tx`, `GET /fee`, `GET /nonce`, and `GET /domain` | Untrusted | Arbitrary public callers that may submit malformed, malicious, or replay payloads; the `GET` routes are [public reads](#public-reads). | +| WebSocket subscribers at `/ws/subscribe` | Internal, but untrusted for data-exposure | Intended for internal indexers; treat as public for what is exposed. | +| Direct-input senders on L1 | Untrusted | Arbitrary L1 accounts calling InputBox with any calldata. | + +### InputBox version + +The reader's scan starts at the application's deployment block. That is sound +only because a v3 InputBox reverts `addInput` for an application that does not +exist yet; bootstrap checks `version()` and refuses an older box. Setup pins +the block only once it is at or below a safe head sampled before the read, so +no reorg can move the floor. Do not model InputBox forgery. + +### RPC endpoint + +The code supports exactly one endpoint, shared by the reader, submitter, +poster, flusher, and fee oracle. Behind a load-balanced fleet, lagging replicas +can silently truncate `get_logs` ranges and desynchronize the flush and re-sync +views; recovery refuses if the re-sync lags the flush view. + +The reader fails loud on an incomplete `InputAdded` set +([I6](../invariants.md#i6-the-frame-clock-accounts-for-a-complete-l1-interval)): +index contiguity proves the right prefix, and the InputBox count pinned at the +scanned safe block proves it complete. When that count equals the stored count, +the reader advances without a `get_logs` crawl; a fail-stop node's pinned-block +count cannot understate without lying. + +Each ingested input's `EvmAdvance` chain id and application must match the +pinned identity; the endpoint's chain id is checked once per reader process. +Two residuals are accepted: a mid-life chain swap behind the same URL that +advances the head only through the count-equality skip, and a node that lies +consistently about both its logs and its input count, outside the fail-stop +model. A semi-trusted fallback tier (Infura/Alchemy) is future work; +[L1 chain semantics](#l1-chain-semantics) says what to pin before adopting one. + +### Operator mistakes + +The operator is trusted, not infallible. The complete fee-oracle source is +pinned in deployment identity, and `run` cannot replace it. The supported +mistake class is accidental concurrent or stale use of one data directory: two +processes on one directory (a kernel process lock) or a mistyped `--data-dir` +(open refuses a path with no database). Deliberate subversion, +copied-directory coordination, and distributed fencing are out of scope; the +lock is a local foot-gun guard +([ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership)). +Judge mechanisms for this class against this boundary. + +### Fee oracle + +A TWAP (30 minutes by default, pinned at setup) and the tenfold margin in +`batch_policy.log_slack` ([formula](../../sequencer/src/storage/migrations/0001_schema.sql)) +mitigate spot manipulation of a deep pool. The margin does not bound arbitrary +market movement. Residual risks are TWAP lag during real moves and a thin or +wrong pool; multi-hop pricing is out of scope. + +Setup writes the first quote under the same hard L1 requirement as the rest of +setup. `run` gates neither recovery nor admission on another quote: it starts +from the persisted `log_gas_price`, builds the source from the pinned identity +without RPC, and refreshes in the background, retrying transient failures +indefinitely and keeping the last price. `log_gas_price_updated_at_ms` is +telemetry, not an expiry gate. + +A shared-endpoint outage or stale view is already caught by safe-head progress. +A pool-specific failure with a healthy reader is an accepted, unbounded economic +risk: a stale-low price subsidizes DA and weakens the spam barrier, and a +stale-high price rejects users. Neither changes canonical execution, because +the frame's persisted fee is immutable and enforced by both sides. Setup-time +source misconfiguration (`WrongTokenPair`, `MissingPoolCode`, chain-id +mismatch), a TWAP tick outside the Uniswap range, and persistent storage faults +are terminal; a price above the representable range saturates at `MAX_EXPONENT`. + +### Public reads + +The `GET` routes are intentional public reads: the open-frame fee, the signing +domain, and any sender's soft-confirmed nonce. The nonce counts a sender's +included ops before their batches reach L1 (up to `max_batch_open`, 2 h by +default); otherwise only the egress feed carries it. It reveals activity +timing, not op contents, and moves only after the op is sequenced, so a watcher +cannot front-run the op it counts. ### Self-trust @@ -40,7 +115,7 @@ application checkpoint and L1. The checkpoint must be trusted independently of the faulty local state. Historical inputs, including malformed batches, receive the canonical scheduler's normal treatment during replay. -This is **not** a prohibition on checking. Internal invariants are enforced loudly wherever a check is near-free — the type system, SQL constraints and triggers, boundary assertions — because failing loud preserves safety, while a silently-tolerated bug that externalizes (a signed batch, an ack, a feed event) is state divergence: as severe as theft and undefendable at runtime. Loud failure is not automatically self-healing: transient faults may clear on restart, but persistent invalid state is terminal and may require inspection or cockroach recovery. The rule, in short: **assert real invariants, fail loud, never absorb silently, never handle gracefully.** The decision test and the register of cross-module invariants live in [`docs/invariants.md`](../invariants.md). +This is **not** a prohibition on checking: internal invariants are asserted loudly wherever a check is near-free (types, SQL constraints and triggers, boundary assertions). The [check policy](../invariants.md#the-check-policy-fail-loud) owns the rule, its reasons, and the decision test. Inputs from untrusted actors are validated rigorously, as ever. @@ -78,7 +153,7 @@ blocking production diagnostics would require revisiting that assumption rollbackable soft confirmations, the watchdog's digest comparison, and the I15 divergence freeze. - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders -- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after baseline block `C` against the valid closed batch we sealed at that nonce. The complete prefix through `C` is opaque and is never reinterpreted using the recovered nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). +- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after baseline block `C` against the valid closed batch we sealed at that nonce. The complete prefix through `C` is opaque and is never reinterpreted using the recovered nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. Cockroach recovery does not read the previous database's watermark, so that flush relies on the provider's pool view; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). - L1 reorgs up to safe depth - Malicious `POST /tx` callers: malformed signatures, spoofed sender, replay across chains or apps, nonce manipulation - Malicious direct-input senders: arbitrary payload, any intent; sender authenticity is guaranteed by InputBox @@ -100,27 +175,36 @@ These are preconditions the sequencer takes as given. They are neither "trust" n ### L1 block-time coupling -The wall-clock fallback in [`sequencer/src/storage/recovery.rs`](../../sequencer/src/storage/recovery.rs) estimates missed blocks as: - -``` -estimated_missed_blocks = (now - last_safe_progress_ms) / CARTESI_SEQUENCER_SECONDS_PER_BLOCK -``` +The wall-clock fallback estimates missed blocks as the time since the last safe-head advance divided by `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` ([automatic recovery](../recovery/README.md#when-safe-head-progress-stops)). This assumes a **known, bounded-variance relationship** between elapsed wall-clock time and mined L1 block count. The assumption has three parts: 1. **Known average block time** — `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default 12s, Ethereum mainnet) accurately reflects the target chain's block cadence. 2. **Bounded variance** — over the danger-threshold window (~4h on mainnet), the delta between `elapsed_seconds / avg_block_time` and actual mined blocks is small. On Ethereum mainnet this holds: slot proposers occasionally skip, but >99% of slots produce a block. -3. **Wall clock is accurate enough for elapsed-time estimation.** A discrete jump of a full block-time or more against either persisted safety baseline is detected and makes the L1 view unusable until the clock or a new safe-head observation catches up; sub-block skew is tolerated as quantization noise, and the fault is evaluated only after the observed-safe danger checks. Gradual or systematic drift remains an external assumption. +3. **Wall clock is accurate enough for elapsed-time estimation.** A clock that falls a full block-time or more behind a persisted safety baseline is detected and refuses operation; sub-block skew is tolerated. Gradual or systematic drift remains an external assumption. -**Where it matters.** The missed-block estimate is a fallback for a safe head that stops advancing, whether the RPC is unreachable or still answers with a stalled view. Clock usability is also checked whenever persisted safety baselines are aged: observed block numbers remain authoritative and their danger verdicts run first, but a clock a full block-time or more out of step with either baseline still makes that view unusable for estimation and for worker admission. +**Where it matters.** The missed-block estimate is a fallback for a safe head that stops advancing, whether the RPC is unreachable or still answers with a stalled view. **Violation modes.** - **Chain with unstable block time.** A chain where average block time drifts substantially (e.g., PoW networks under major hashrate swings) would make the estimate less reliable. Mitigation: `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` should be tuned conservatively (overestimate block time → underestimate missed blocks → more cautious recovery triggers). - **Operator misconfigures `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`.** Typo or copy-paste error pointing at the wrong chain's cadence. Operator-trust scope. -- **Significant host clock drift.** A sequencer host whose clock lags or leads the real-world by minutes per day could slowly desynchronize its danger estimates from reality. A detectable backward crossing of a persisted baseline refuses operation; gradual drift may not. +- **Significant host clock drift.** A sequencer host whose clock lags or leads the real-world by minutes per day could slowly desynchronize its danger estimates from reality. **Corollary for test design.** To deterministically exercise the wall-clock fallback, tests must maintain this coupling: when advancing the L1 block count, they should also advance (or simulate) the corresponding wall-clock interval. Our e2e harness does the reverse — it rewinds `l1_safe_head.synced_at_ms` to an older timestamp, which is semantically equivalent to advancing the wall clock. +### L1 chain semantics + +The input reader, the frame clock, and the canonical scheduler use L1 block numbers and the `safe` tag as Ethereum defines them: + +1. **Safe blocks are permanent.** The reader ingests through the sampled safe head and never rereads below it, and frames commit to that head. The sequencer reads `safe`, not `finalized`, for deposit latency: on Ethereum, finalized trails safe by about one more epoch (6.4 minutes), a large share of the [deposit-availability target](../../AGENTS.md#invariants). +2. **EVM `block.number` is the RPC block number.** The application's deployment block, which floors the scan, and each input's `EvmAdvance.blockNumber`, by which the scheduler drains and ages it, are EVM values. The reader uses them as log and safe-head coordinates, and frames carry RPC block numbers. + +**Violation modes.** +- **A safe block is replaced**: an Ethereum justified checkpoint that never finalizes, or an L2 whose safe head rests on unfinalized parent-chain data (the OP Stack). Frames and soft confirmations already account for inputs L1 no longer has. The reader then stalls on an index or count mismatch, unless the replacement keeps the input count, in which case nothing detects it. The reader pins no block hash: it logs a warning only when a provider's safe head is numerically behind the persisted one. +- **Arbitrum and Orbit chains** return the parent chain's number from `block.number`. The scan floor lands far below the deployment, and the first ingested input fails the reader's block cross-check on every retry, after setup completes and, at the latest, once our first batch lands, with soft confirmations already issued. + +**Revisit** before targeting any chain other than Ethereum L1 or its testnets. For an Arbitrum-family chain, first add a setup check that `block.number` at the sampled safe block equals that block's number. Before adopting a load-balanced or fallback endpoint tier, also pin the reader's count read to the sampled block's hash, so a backend that clamps a future block number errors instead of answering for another block. + ## How to apply this doc in code review For each code path under review: @@ -137,4 +221,4 @@ For each code path under review: ## Related documents - [`../recovery/README.md`](../recovery/README.md) — recovery design, TLA+ formal verification -- [`../../AGENTS.md`](../../AGENTS.md) — architecture, coding conventions, hot-path invariants +- [`../../AGENTS.md`](../../AGENTS.md) — architecture map, conventions, and working rules diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index aa92aea9..800b47e4 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -50,8 +50,8 @@ retry: a failed tick's retry is the next scheduled one. ### Canonical execution The watchdog drives the machine in-process through the `cartesi` Lua module, -with the reference rollup host semantics the `cartesi-machine` CLI and Dave -implement ([details](../cartesi-machine.md#rollup-host-semantics)). It clones +with the rollup host semantics of Dave's sling node and the CLI's epoch driver +([grounding and pins](../cartesi-machine.md#rollup-host-semantics)). It clones the head into a working directory and runs the machine in place on it; before each input it clones the working directory as a snapshot: @@ -71,16 +71,21 @@ watchdog and the reference CLI and require equal root hashes. A provider that silently drops logs must fail the tick, not produce a false comparison. The reader requires InputBox indices to run contiguously from the head's input count, and checks the count it reached against -`InputBox.getNumberOfInputs(app)` pinned at block B. Every input's payload must -name the configured application and chain. Long log ranges are split on the -provider error codes that mean "range too large", exactly like the Rust reader -(shared vector: `tests/fixtures/l1_partition_vector.json`). +`InputBox.getNumberOfInputs(app)` pinned at block B (a scan that stops at a +canonical fixed point latches instead, without this check). Every input's +payload must name the configured application and chain. A log query that fails +over more than one block is split in half and retried, and one that fails on a +single block fails the tick, exactly like the Rust reader. Logs sort by (block, +tx index, log index), and a log missing any of them fails the tick, as in the +Rust reader. The shared vector `tests/fixtures/l1_partition_vector.json` pins +both rules on both sides. **Scan floor.** The Rust reader starts at the application's deployment block, -which is sound only because it also witnesses that the InputBox rejects inputs -for undeployed applications. The watchdog starts at its bootstrap block and -proves completeness from the InputBox count instead; do not copy the -deployment-block floor without the witness. +which is sound because setup pins that block only once it is safe and the +reader's first scan proves, from index 0 and the InputBox count, that no input +precedes it. The watchdog starts at its bootstrap block and proves completeness +from the InputBox count instead; do not copy the deployment-block floor without +that witness. The reader holds one provider response at a time: each successful log range is decoded and fed to the machine before the next is fetched. @@ -116,17 +121,25 @@ schedulers must also prevent overlapping ticks (systemd, or Kubernetes `concurrencyPolicy: Forbid`). `init` checks the bootstrap machine: it must wait for an input and yield the -configured state source, and it must be a trusted start, which `replay` -requires of its `--from` machine too. Its block must be safe on the RPC; its -input count comes from `getNumberOfInputs` at that block, which needs an RPC -with state there (an archive node for old blocks); and when no input precedes -the block, the machine must be the application's template (its root hash -equals the on-chain `getTemplateHash()`). +configured state source, the tick must be able to run it in place (its files +writable, see [filesystem](operator-deployment.md#filesystem)), and it must be +a trusted start, which `replay` requires of its `--from` machine too. Its +block must be safe on the RPC; its input count comes from `getNumberOfInputs` +at that block, which needs an RPC with state there (an archive node for old +blocks); and when no input precedes the block, the machine must be the +application's template (its root hash equals the on-chain +`getTemplateHash()`). Exit 1 covers three failure classes, which the log line and `last_tick.json` name: `transient` (the L1 RPC or the sequencer's HTTP API; the next tick may succeed), `operator` (configuration, the state directory, an image), and -`internal` (anything else, including filesystem failures such as a full disk). +`internal` (anything else, including a filesystem write that reports a full +disk). A full disk while the working machine runs in place is different: the +emulator's write through a shared mapping faults, and the kernel kills the +process by SIGBUS (on Linux signal 7: exit 135 in a shell, `status=7/BUS` under +systemd) before it writes `last_tick.json` or `status.prom`, so only the +stale-heartbeat alert sees it; `replay` can die the same way, and the watchdog +keeps no free-space floor ([why](#why-it-is-built-this-way)). Exit 2 means a divergence: found by this tick, or latched earlier. It stays latched until `clear`. A tick that finds a divergence exits 2 even when it cannot create `incident/` (a full state directory); it logs the divergence as @@ -138,22 +151,21 @@ again if it persists. `init` reads the environment and writes `config.json`, which later commands read: the chain id, the application, the InputBox it derives from the -application, the state source, the bootstrap block, the sequencer URL, and the -range-error codes. The bootstrap block matters after `init` too: it tells a -never-agreed head from an agreed one (see [the tick](#the-tick)). The L1 RPC -endpoint is never persisted. +application, the state source, the bootstrap block, and the sequencer URL. The +bootstrap block matters after `init` too: it tells a never-agreed head from an +agreed one (see [the tick](#the-tick)). The L1 RPC endpoint is never persisted, +and failure messages name no request URL. | Variable | Read by | Meaning | |---|---|---| | `CARTESI_WATCHDOG_STATE_DIR` | all | Absolute path of the state directory | | `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` | init, tick, replay | L1 JSON-RPC endpoint; never persisted, so it can rotate | -| `CARTESI_WATCHDOG_SEQUENCER_URL` | init (persisted); tick (optional override) | Sequencer operator API base URL | +| `CARTESI_WATCHDOG_SEQUENCER_URL` | init (persisted); tick (optional override) | Base URL of the sequencer's internal listener | | `CARTESI_WATCHDOG_BLOCKCHAIN_ID` | init (optional) | Expected chain id; defaults to the RPC's `eth_chainId` | | `CARTESI_WATCHDOG_APP_ADDRESS` | init | Application address; init derives the InputBox from its `getInputBox()`, as the sequencer does | | `CARTESI_WATCHDOG_STATE_SOURCE` | init | `inspect`, or `range:/run.lock` | Another `init`, `tick`, or `clear` is running | | `flock is required (util-linux)` | Use the devshell, or install util-linux | -| `sequencer /finalized_state/inclusion_block: GET failed: …` | The devnet stack stopped, or the exports come from an earlier run | +| `sequencer /finalized_state/inclusion_block: GET failed: …` | The devnet stack stopped, or the exports come from an earlier run | | `divergence inclusion_block_regressed latched at block 0` | The state directory comes from an earlier devnet run; remove it and run `init` again | diff --git a/docs/watchdog/incident-runbook.md b/docs/watchdog/incident-runbook.md index 599ee151..3d814abd 100644 --- a/docs/watchdog/incident-runbook.md +++ b/docs/watchdog/incident-runbook.md @@ -100,7 +100,10 @@ Use the application's template machine with `--from-block 0` (`replay` checks it against the on-chain template hash), or an archived checkpoint whose block you trust. `replay` needs an RPC with state at `A` (an archive node for old blocks), absolute paths, and an `--out` that does not exist; it never writes -the state directory. +the state directory. Run it with a watchdog release on the deployment's +Cartesi Machine version: templates, checkpoints and `incident/canonical` load +only in the version that stored them +([upgrades](operator-deployment.md#upgrades)). | `replay`'s `sha256` equals | Conclusion | |---|---| diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index 53b1c116..156b6728 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -22,12 +22,11 @@ the sequencer's operator routes. | `GET /finalized_state/digest` | When the sequencer's accepted block has advanced | | `GET /finalized_state` | After a mismatch, to keep the sequencer's bytes as evidence | -The sequencer serves these routes on the same listener as its public API, -without authentication. The +The sequencer serves these routes on its internal listener +(`CARTESI_SEQUENCER_INTERNAL_HTTP_ADDR`), without authentication. The [API reference](../../README.md#operator-snapshot-endpoints-internal-only) -owns them and why they must stay private: expose only the public routes at the -public ingress, and give the watchdog host an internal path to the operator -routes. The sequencer hashes the comparison file on each digest request and +owns them and why they must stay private: publish only the public listener, +and give the watchdog host a path to the internal one. The sequencer hashes the comparison file on each digest request and the watchdog waits up to 300 seconds for the answer, so a proxy in between needs a read timeout at least that long for multi-GiB states. @@ -49,9 +48,8 @@ InputBox, `eth_getCode`, and `eth_call` at past blocks: `init` reads the InputBox count at the bootstrap block, and every comparison reads it at the comparison block ([L1 completeness](README.md#l1-completeness)). Use an archive node when the bootstrap block, or a block you `replay` from, is older than the -state your provider keeps. Long log ranges are split when the provider answers -with one of the codes `-32005,-32012,-32600,-32602,-32616`; if yours uses -another, set `CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES` at `init`. +state your provider keeps. A log query that fails over more than one block is +split in half and retried; one that fails on a single block fails the tick. ## Runtime @@ -122,10 +120,9 @@ as `canonical-machine-image--vX.tar.gz` (checked against the release's The build prints the image's root hash. It needs the kernel (`just canonical download-deps`), `cross`, Docker buildx able to build -`linux/riscv64`, `xgenext2fs`, and the pinned `cartesi-machine`. A stored -machine loads only in the Cartesi Machine version that stored it -([versions are all-or-nothing](../cartesi-machine.md#versions-are-all-or-nothing)), -so the image and the watchdog must use the same version. +`linux/riscv64`, `xgenext2fs`, and the pinned `cartesi-machine`. The image +and the watchdog must use the same Cartesi Machine release +([why](../cartesi-machine.md#versions-are-all-or-nothing)). ## State source @@ -155,12 +152,24 @@ a copy-on-write filesystem a clone costs metadata; elsewhere it is a full sparse copy of the machine, which makes multi-GiB states slow ([Cartesi Machine storage](../cartesi-machine.md#storage-sharing-and-snapshots)). Keep the bootstrap machine on the same filesystem, so the clone `init` makes -of it is cheap as well. +of it is cheap as well. Leave its file permissions as `cartesi-machine +--store` and the release tarball set them: every file writable by its owner +except those of host read-only ranges, such as the PMAs +(`0000000000010000-1000.bin`), which the emulator maps read-only. The working +machine runs in place, and the emulator refuses to map a file without write +permission for writing +([storage](../cartesi-machine.md#storage-sharing-and-snapshots)), so `init` +opens a clone of the bootstrap machine as the tick will and refuses an image +whose files lack it. ### Checkpoint disk usage and backups Disk usage is roughly one machine for the head, plus a working clone and a -per-input snapshot while a tick runs (nearly free on copy-on-write), plus +per-input snapshot while a tick runs (on copy-on-write a clone costs metadata, +but the pages a catch-up writes, and the hash-tree pages they update, take +real blocks, up to a full machine for the working clone: about 40 MB per 100 +inputs in the +[measured run](../cartesi-machine.md#storage-sharing-and-snapshots)), plus incident evidence: the canonical machine at the divergence and two copies of the comparison bytes. Cleared incidents stay under `incidents/` until you remove them. With a `range` source, the watchdog also reads the whole range @@ -172,12 +181,13 @@ Back up, from the state directory: - the newest `checkpoints/-/`, the head: the stored machine at L1 block ``, after `` InputBox inputs, which the sequencer agreed with (or the trusted bootstrap, before the first agreement); -- `incident/` while a divergence is latched, and `incidents/`. +- `incident/` while a divergence is latched, and `incidents/`; +- `config.json`: `init` refuses a directory whose agreed head has lost it. -`init` writes `config.json`, so it needs no backup. Checkpoints are published -atomically, so the newest one is complete once a tick has finished. Run the -backup after the tick in the same scheduled job: the next tick removes the -previous head as soon as it publishes a new one. Names are zero-padded, so the +Checkpoints are published atomically, so the newest one is complete once a +tick has finished. Run the backup after the tick in the same scheduled job: the +next tick removes the previous head as soon as it publishes a new one. Names +are zero-padded, so the last one in sorted order is the newest. Stored machines are sparse files; copy them with a sparse-aware tool (`cp --sparse=always`, `rsync --sparse`, `tar --sparse`). Keep copies outside `checkpoints/`, which must hold nothing @@ -203,7 +213,7 @@ table says where its value comes from. |---|---|---| | Chain id | The L1 network: `11155111` for Sepolia, `1` for mainnet. Pin it: `init` refuses an RPC that serves another chain | `CARTESI_WATCHDOG_BLOCKCHAIN_ID` | | Application | The deployment; the sequencer's `CARTESI_SEQUENCER_APP_ADDRESS` | `CARTESI_WATCHDOG_APP_ADDRESS` | -| Sequencer URL | The internal address of the sequencer's listener ([network access](#network-access)) | `CARTESI_WATCHDOG_SEQUENCER_URL` | +| Sequencer URL | The sequencer's internal listener ([network access](#network-access)) | `CARTESI_WATCHDOG_SEQUENCER_URL` | | L1 RPC | Your provider ([network access](#network-access)); read at run time, never persisted | `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` | | State source | The application ([state source](#state-source)) | `CARTESI_WATCHDOG_STATE_SOURCE` | | Bootstrap machine and block | [Initialize](#initialize) | `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR`, `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` | @@ -226,7 +236,7 @@ Choose the bootstrap: ```bash cast call "$CARTESI_WATCHDOG_APP_ADDRESS" 'getDeploymentBlockNumber()(uint256)' \ - --rpc-url "$CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT" + --block safe --rpc-url "$CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT" ``` No input precedes that block, so `init` checks the image against the @@ -239,7 +249,11 @@ Choose the bootstrap: `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK=X`. `init` checks that the machine waits for an input and yields the state source. It cannot check that its state is right: a wrong bootstrap surfaces as a `state_mismatch` at the next - comparison, the runbook's "watchdog wrong" case. The + comparison, the runbook's "watchdog wrong" case. A Dave node seeds + mid-history only from the consensus's last settled machine root + (`getLastFinalizedMachineMerkleRoot`); the watchdog reads no consensus + contract, so nothing anchors such a checkpoint, and a wrong one costs a false + `state_mismatch` that `replay` from the template resolves. The [detection boundary](README.md#detection-boundary) states what `init` and idle ticks do not prove. @@ -283,7 +297,9 @@ console output. - **Divergence (`failed`).** Follow the [incident runbook](incident-runbook.md). - **Stale heartbeat.** No tick has finished recently: the scheduler stopped, a - tick hangs or is still replaying a long range, or every run exits at the lock + tick hangs or is still replaying a long range, a tick died by a signal (such + as SIGBUS on a full state-directory filesystem; see + [failure classes](README.md#commands)), or every run exits at the lock because another process holds it, such as a latching tick still collecting evidence (its divergence alert fires first). - **Sustained warning.** `sequencer-watchdog status` shows the last tick's @@ -303,15 +319,23 @@ A deployed application is pinned to the Cartesi Machine version its template was built with ([why](../cartesi-machine.md#versions-are-all-or-nothing)), and so is its watchdog: run a watchdog release whose `cartesi_machine_version` (printed with `CARTESI_WATCHDOG_PRINT_RELEASE_INFO=1`) matches the version the -application's template was built with. A release on another version cannot -load the state directory's machines and cannot reproduce the template. +application's template was built with. A release on another minor version +cannot load the state directory's machines; within a minor, only the release +you run holds the watchdog to the template's version (ours pin it exactly). +Keep a release on each Cartesi Machine version that a deployment still +runs, or whose checkpoints and incident evidence you may need to `replay`. + +A new Cartesi Machine version means a new application deployment, and its +watchdog starts from a new, empty state directory that `init` fills from that +release's canonical image. Keep the old deployment's state directory intact +beside the old release; do not wipe it while its evidence may be needed. ## Sepolia | Item | Value | |---|---| | Chain id | `11155111` | -| InputBox | `0xEbE9f4Dfc04ae10bBeE663859c3dc5A23f94eA3C`: rollups-contracts v3.0.0-alpha.10, the version the root `justfile` pins, deploys it at the same address on every chain, the devnet included | +| InputBox | `0xEbE9f4Dfc04ae10bBeE663859c3dc5A23f94eA3C`: rollups-contracts v3.0.0-alpha.10, the version `toolchain-pins.env` pins, deploys it at the same address on every chain, the devnet included | | Canonical image | `canonical-machine-image-sepolia-vX.tar.gz`, or `just canonical-build-machine-image-sepolia`. Its guest runs `WalletConfig::sepolia()`, the configuration of the release `wallet-sequencer` binary | | State source | `inspect` | | Application | Per deployment | @@ -355,6 +379,10 @@ re-detects fail-loud when the faulty state is next read, so the supervisor's handling of 30 and SIGABRT (stop and page, do not auto-restart) is what bounds a crash loop. +The final `sequencer exiting` line logs at ERROR for exits 30 and 1, and at +WARN for 10, 20 and 40, with `verdict` and `supervisor_action` fields; the +SIGABRT line says so. Alert on the exit status, not the log level. + - **Supervisor recipes.** systemd can act on the code directly: `RestartPreventExitStatus=30 ABRT` (SIGABRT/134 is terminal-class too). Kubernetes Deployments restart regardless of exit code, and there is no @@ -362,13 +390,18 @@ a crash loop. re-detection windows, serving traffic in between. There, the crash-loop bound is your alerting, not the restart policy: page immediately on `lastState.terminated.exitCode == 30` (and on signal exits / 134). +- **A fresh deployment needs a setup retry budget.** `setup` exits 20 until + the L1 safe head reaches the application's deployment block, about one to + two epochs on Ethereum (6.4–12.8 min); give a setup Job or unit a longer + retry budget (a Kubernetes Job's default backoffLimit 6 allows about + 10.5 min). - **After an unclean death (OOM, node reboot, SIGKILL) no action is needed**: the next start re-derives everything from facts. For postmortems, the `terminal_faults` table records the cause of every command that returned through its bracket with a terminal verdict (best-effort, append-only, traveling with the data directory — - `SELECT * FROM terminal_faults ORDER BY fault_id DESC`; the next `run` - also logs the latest row once at startup). Any death that did not return + `SELECT * FROM terminal_faults ORDER BY fault_id DESC`; each `run` boot + logs the latest row at WARN unless a later completed setup resolved it). Any death that did not return through the bracket — SIGKILL, OOM, a node reboot, a terminal runtime abort, a controller panic — leaves only the process logs. - **Untrustworthy local state requires manual recovery.** The reader's @@ -382,19 +415,24 @@ a crash loop. ## Troubleshooting (live deployments) -The wrapper prints the lock message itself; the watchdog prints the others -after the command and failure class, as in +The wrapper prints the lock and `flock` messages, and the missing-directory +message for `clear`, itself; the watchdog prints the others after the command +and failure class, as in `watchdog: tick: transient failure: `. | Message | Cause | |---|---| | `watchdog state is already locked: /run.lock` | Another `init`, `tick`, or `clear` holds the state directory, so the scheduler overlaps runs | | `flock is required (util-linux)` | Install util-linux; the wrapper needs `flock` for its lock | -| `state directory does not exist` | `status` or `replay` with a wrong `CARTESI_WATCHDOG_STATE_DIR` | +| `state directory does not exist` | `status`, `replay`, or `clear` with a wrong `CARTESI_WATCHDOG_STATE_DIR` | +| ` has checkpoint but no config.json; restore config.json or wipe the directory` | `config.json` was lost beside an agreed head; restore it from backup, or wipe and re-run `init` | | `/config.json is not valid JSON` | A crash tore `config.json` right after `init`; wipe the state directory and re-run `init` | | ` is not initialized; run init` | `tick` or `replay` on an empty state directory: a missing volume or a changed `CARTESI_WATCHDOG_STATE_DIR` | | `/config.json is not a version 2 watchdog config; wipe the state directory and re-run init` | See [upgrades](#upgrades) | -| `cannot load stored machine : …` | A machine stored by another Cartesi Machine version; see [upgrades](#upgrades) | +| `cannot load stored machine : expected "archive_version" 8 (got 7)`, or the `expected …` text alone with class `internal` (from a clone in `tick` or `replay`) | A machine stored by a Cartesi Machine release of another minor version; see [upgrades](#upgrades) | +| `the tick cannot run in place: cannot load stored machine …/work/probe: unable to open backing file '': Permission denied …` from `init` (as root: `backing file '' is read-only and cannot be shared for writing`) | The bootstrap machine's files lack write permission; run `chmod -R u+w `, or on a read-only filesystem copy it, make the copy writable, and point `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR` at it; then re-run `init` (see [filesystem](#filesystem)) | +| `cannot load stored machine …/work/working: …` with the same causes, from `tick` | The head checkpoint's files lack write permission (lost after `init`, or kept from a bootstrap machine an older release initialized); run `chmod -R u+w` on the head checkpoint, or wipe the state directory and re-run `init` | +| `cannot load stored machine : …` with another message | The emulator's message names the file or field it could not load | | ` was initialized for another deployment or state source; wipe it to re-initialize` | The `init` environment names another chain, application, or state source than the directory holds | | `no input precedes block , so the machine at must be the application's template, but its root hash differs from the on-chain template hash` | The image is not this application's template: another chain's image, another build, or another application | | `L1 RPC: eth_call: …` during `init` | The RPC has no state at the bootstrap block; use an archive node | @@ -402,6 +440,6 @@ after the command and failure class, as in | `sequencer /finalized_state/digest: HTTP 404` while `inclusion_block` answers | A proxy does not forward the digest route, or the sequencer comes from another release | | `sequencer /finalized_state/inclusion_block: HTTP 503`, with `canonical divergence prevents accepted checkpoint selection` | The sequencer's own divergence marker is set: a sequencer incident for [cockroach recovery](../recovery/cockroach.md#incident-playbook) | | `L1 RPC safe head is behind target block ` | The RPC lags the sequencer's L1 view; the next tick retries | -| `InputBox counts inputs at block but the scan reached index `, or `InputBox index where was expected: the L1 view is incomplete` | The provider returned incomplete logs; switch providers if it persists | +| `InputBox counts inputs at block but the scan reached index `, or `InputBox index where was expected: the L1 view is incomplete` | The provider returned incomplete logs, so switch providers; if it persists across them, a safe block below the head may have been replaced ([L1 chain semantics](../threat-model/README.md#l1-chain-semantics)), so confirm with `replay` from a trusted machine at a finalized block, then wipe the state directory and `init` from one | | `the L1 RPC does not serve chain ` | The RPC endpoint belongs to another chain | | `no flash drive or NVRAM carries the label "