diff --git a/.github/workflows/w012-validation.yml b/.github/workflows/w012-validation.yml new file mode 100644 index 00000000..fc565edf --- /dev/null +++ b/.github/workflows/w012-validation.yml @@ -0,0 +1,183 @@ +# W012's embedding boundary is outside the generic compiler CI suite. +name: w012-validation + +on: + pull_request: + paths: + - '.github/workflows/w012-validation.yml' + - 'compiler/**' + - 'spikes/w012-analyzer/**' + - 'tests/checker-corpus/**' + - 'test_wasm_parity.js' + - 'tools/env.sh' + push: + branches: [main] + paths: + - '.github/workflows/w012-validation.yml' + - 'compiler/**' + - 'spikes/w012-analyzer/**' + - 'tests/checker-corpus/**' + - 'test_wasm_parity.js' + - 'tools/env.sh' + +permissions: + contents: read + +jobs: + analyzer-parity: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + CARGO_BUILD_JOBS: '2' + CARGO_TARGET_DIR: ${{ github.workspace }}/compiler/target + GOFLAGS: '-p=2' + HYBRID_WASM_PATH: ${{ github.workspace }}/compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm + HYBRID_NATIVE_PATH: ${{ github.workspace }}/compiler/target/release/hyb + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Pinned Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: '1.26.9' + cache: false + - name: Node for full-envelope parity + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24.19.0' + - name: Pinned Rust and WASM target + run: | + set -euo pipefail + rustup toolchain install 1.99.0 --profile minimal --target wasm32-unknown-unknown + rustup default 1.99.0 + rustc --version + go version + node --version + - name: Build both checker artifacts from this checkout + working-directory: compiler + run: | + set -euo pipefail + cargo build --locked --release -p hyb + cargo build --locked --release -p hy-wasm --target wasm32-unknown-unknown + - name: Upload prebuilt WASM for independent Go-only consumer + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: w012-prebuilt-wasm + path: compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm + if-no-files-found: error + retention-days: 7 + - name: Complete native and WASM envelope parity + run: node test_wasm_parity.js + - name: Go wrapper parity, warning preservation, and failure boundaries + working-directory: spikes/w012-analyzer + run: | + set -euo pipefail + go test -race -count=1 ./... + go vet ./... + - name: Enforce gofmt for W-012 experiment and cache tests + working-directory: spikes/w012-analyzer + run: | + set -euo pipefail + unformatted="$(gofmt -l cmd/editbench/main.go cmd/editbench/main_test.go cmd/editbench/reused.go compilation_cache_test.go)" + if [ -n "$unformatted" ]; then + printf 'Unformatted Go files:\n%s\n' "$unformatted" + exit 1 + fi + - name: Reproduce 25-file 100-edit request-path comparison + working-directory: spikes/w012-analyzer + run: | + set -euo pipefail + go run ./cmd/editbench -root ../.. -files 25 -edits 100 -warmup 10 \ + -output "$RUNNER_TEMP/w012-edit-benchmark.json" + - name: Measure opt-in WASM cache in two separate Go processes + working-directory: spikes/w012-analyzer + env: + HYBRID_W012_CACHE_DIR: ${{ runner.temp }}/w012-wazero-compilation-cache + run: | + set -euo pipefail + mkdir -p "$HYBRID_W012_CACHE_DIR" + go build -o "$RUNNER_TEMP/w012-editbench" ./cmd/editbench + "$RUNNER_TEMP/w012-editbench" -root ../.. -files 2 -edits 2 -warmup 1 \ + -output "$RUNNER_TEMP/w012-cold-cache-first.json" + "$RUNNER_TEMP/w012-editbench" -root ../.. -files 2 -edits 2 -warmup 1 \ + -output "$RUNNER_TEMP/w012-cold-cache-second.json" + python3 - <<'PY' + import json, os + root = os.environ["RUNNER_TEMP"] + with open(os.path.join(root, "w012-cold-cache-first.json")) as f: first = json.load(f) + with open(os.path.join(root, "w012-cold-cache-second.json")) as f: second = json.load(f) + assert first["WASMSHA256"] == second["WASMSHA256"] + assert first["NativeSHA256"] == second["NativeSHA256"] + print("CROSS-PROCESS WASM FILESYSTEM CACHE, experimental only") + print("first cold Engine.GetEngine:", first["ColdWASMInitMS"], "ms") + print("second cold Engine.GetEngine:", second["ColdWASMInitMS"], "ms") + print("No speed threshold asserted; compare host-specific raw samples.") + PY + - name: Publish cross-process cache timings + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: w012-cross-process-cache-timings + path: ${{ runner.temp }}/w012-cold-cache-*.json + if-no-files-found: ignore + retention-days: 30 + - name: Publish raw request latency samples + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: w012-request-latency-raw + path: ${{ runner.temp }}/w012-edit-benchmark.json + if-no-files-found: ignore + retention-days: 30 + + go-only-consumer: + needs: analyzer-parity + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout source without build artifacts + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Pinned Go toolchain (no Rust installation step) + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version: '1.26.9' + cache: false + - name: Retrieve WASM built by separate Rust-capable job + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: w012-prebuilt-wasm + path: compiler/target/wasm32-unknown-unknown/release + - name: Execute Hybrid analyzer using Go and precompiled WASM + working-directory: spikes/w012-analyzer + env: + HYBRID_WASM_PATH: ${{ github.workspace }}/compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm + GOFLAGS: '-p=2' + run: | + set -euo pipefail + # Poison Rust commands: any attempt to invoke cargo/rustc by name + # breaks this test. This does not mean the hosted VM lacks Rust. + mkdir -p "$RUNNER_TEMP/forbid-rust" + printf '#!/bin/sh\necho RUST_TOOLCHAIN_INVOKED >&2\nexit 87\n' > "$RUNNER_TEMP/forbid-rust/cargo" + cp "$RUNNER_TEMP/forbid-rust/cargo" "$RUNNER_TEMP/forbid-rust/rustc" + chmod +x "$RUNNER_TEMP/forbid-rust/cargo" "$RUNNER_TEMP/forbid-rust/rustc" + export PATH="$RUNNER_TEMP/forbid-rust:$PATH" + test -s "$HYBRID_WASM_PATH" + go build -o "$RUNNER_TEMP/hybvet" ./cmd/hybvet + set +e + "$RUNNER_TEMP/hybvet" vet ./testpkg > "$RUNNER_TEMP/w012-negative.log" 2>&1 + result=$? + set -e + if [ "$result" -eq 0 ] || ! grep -q 'HYB-E3003' "$RUNNER_TEMP/w012-negative.log"; then + echo "Expected invalid Hybrid source to produce HYB-E3003 without Rust" >&2 + cat "$RUNNER_TEMP/w012-negative.log" >&2 + exit 1 + fi + # Editing only .hy must be visible on the next vet pass. + cp ../../tests/checker-corpus/09-a3-map-slot-valid.hy ./testpkg/sibling.hy + "$RUNNER_TEMP/hybvet" vet ./testpkg + echo "Go+prebuilt WASM check PASS; cargo/rustc not invoked." + diff --git a/PLAN.md b/PLAN.md index b1bd9332..dd872e97 100644 --- a/PLAN.md +++ b/PLAN.md @@ -50,6 +50,15 @@ All SPEC §3 rows verified — see research/claims.md. Headlines: rustygo 127/14 | 5 errors | per pre-registered table | **PASS all**: E1 2.58 ns/0 allocs (two-word HyResult), E2 0 allocs, E3 1 alloc, E4 identity + documented chain loss, E5 abort contract verified | adopt HyResult + sentinel table + wrap-on-demand | | 6 ownership usability | 10 programs, annotation count | **PASS**: 0 annotations / 0 managed / 0 outside-model (strictest reading; hand-trace); A1–A8 ambiguities + corpus gap recorded | Tier-0 real for MVP corpus; Polonius-style engine + rustc CI oracle | +## Contributor validation — W-012 EOF diagnostics (2026-10-19) + +- An unterminated block ending in LF/CRLF is a real HYB-E0002 at the logical EOF line, which Go's unpadded token table omitted. Preserve EOF with token-table-only padding; source bytes and coordinate bounds remain unchanged. +- Exact branch source 6dafd497 plus the fix, pinned Go 1.26.9 / Rust 1.99.0 / Node 24.19.0: old analyzer fails four EOF cases and passes two controls; fixed cases, full Go race suite and vet pass; freshly built same-checkout native/WASM complete envelope and verdict parity are 60/60. Hosted synthetic merges have a separate corpus and are not interchangeable artifact sources. +- Preserve the original run 39195170609 raw measurements with checked archive/file digests under spikes/w012-inprocess/validation-2026-10-19; extend future timing artifacts to 30 days. See W012_M1_REVIEW.md for evidence boundaries and the still-pending November 1 M1 review. +- Historical integration 61d5bd9 joined fd4f084d with f6e4d929, the PR-reported base SHA. Fork W012 validation 39210700789 and compiler CI 39210708370 passed with rebuilt native/WASM 61/61 verdict and full-envelope parity, Go race/vet, and Go-only consumer. The later documentation head 0d1dbf7 also passed those gates in fork runs 39211484374 and 39211484338. These results validate their recorded trees, not integration with the live main branch. +- Base correction (2026-10-19): Git transport and the upstream ref/branch APIs identified actual main as aa4a9d91; the earlier 0d1dbf7 lacked 251 main commits, with f6e4d929 as merge base. Integration 4149882 now joins 0d1dbf7 with aa4a9d91, retaining the EOF fix, raw evidence, WASM project, and all upstream changes; only STATUS.md conflicted. +- Completed validation of 4149882: upstream W012 39213961382 and compiler CI 39213961322, plus fork W012 39213960125 and compiler CI 39213960076, all PASS. Independently rebuilt native/WASM artifacts pass 113/113 verdict and complete-envelope parity in each repository; Go race/vet, Go-only consumer, and compiler differential agreement pass. The upstream and fork executed merges share published tree 9a27169f. At 07:13 UTC, upstream PR #556 reports clean/mergeable with base aa4a9d91; M1 remains pending for November 1. Historical 60/60, 61/61, earlier 113/113 and raw timings retain their source labels; see W012_M1_REVIEW.md. + ## Task List - [ ] Contributor tooling patch: shape.py parses supported options before file operations; invalid flags/positionals and help preserve generated files. Maintainer validated PR #351; main status conflict resolved. Native-Linux focused 9/9, Python tooling 107/107 pass. Compiler/runtime suites not run; merge and any award pending (17 October 2026). - [x] Installer checksum download must fail closed | Missing checksum fetch previously installed and executed an unverified archive; now aborts before extraction or replacement. Six offline installer tests added to quick CI; two regression tests fail on base `34525ee`, all six pass locally on Windows/Git Bash. Upstream CI/reward review pending (2026-10-15). diff --git a/STATUS.md b/STATUS.md index 70e4606d..0753dc80 100644 --- a/STATUS.md +++ b/STATUS.md @@ -8,3 +8,4 @@ - Wildcard holds (do NOT merge before window): #635 #637 #704 #709 #616 #446 #577 #572 #569 #566 #556 #545 #516 #505 #488 #480 #474 #464 #452 #405 #384 #381 #370 #379 #507 #412 #336; drafts #531/#465 stay with authors - CI quota healthy (4980/5000 after ~190 approvals this session); race tempo ~15 min/landing at peak — worktree-merge-push protocol held (two ort auto-merge traps caught: silent hunk deletion ×2, unpaired conflict marker ×2) - Next queue: wave tail + #711–#724 recon, then M1 posting (11-01), then $500 Wildcard Track ops +- W-012 actual-main integration 4149882: upstream and fork freshly rebuilt native/WASM verdict/full-envelope parity 113/113, Go race/vet/consumer and compiler CI PASS. PR #556 is clean/mergeable at 2026-10-19 07:13 UTC; M1 remains pending for November 1. Historical counts and raw timings retain their source labels; see W012_M1_REVIEW.md. diff --git a/W012_M1_REVIEW.md b/W012_M1_REVIEW.md new file mode 100644 index 00000000..24fdcc82 --- /dev/null +++ b/W012_M1_REVIEW.md @@ -0,0 +1,430 @@ +# W012 M1 evidence: current validation and historical snapshots + +## Reviewer decision brief (2026-10-19 JST) + +**Status: W-012 is scheduled for the 2026-11-01 M1 review. It is not yet M1-awarded or merged.** This is an executable feasibility and falsification record for embedding the existing Rust checker, not a production toolchain replacement or completed IDE integration. + +### Completed integration with actual main + +Integration commit `4149882c3f8cc5c636d559ec93cbb2f416098c65` merges branch head `0d1dbf711f7fb7c3147ca320c9435e86a8a6b0b1` with actual upstream `main`, `aa4a9d91b5edecd2bff37eba23fdd7befe60bd56`. Only `STATUS.md` conflicted; its upstream status was retained with a scoped W-012 note. The EOF fix, raw measurement archive, WASM project, and all upstream changes are preserved. + +Both repositories completed fresh validation of this integration: + +| CI repository | W012 validation | Compiler CI | Expected verdicts / complete native-WASM envelopes | +| --- | --- | --- | --- | +| Upstream | [39213961382 — PASS](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39213961382) | [39213961322 — PASS](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39213961322) | **113/113 / 113/113** | +| Contributor fork | [39213960125 — PASS](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39213960125) | [39213960076 — PASS](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39213960076) | **113/113 / 113/113** | + +The actual upstream checkout was merge `1e8596ab3d3b5e561176f1d2e5b10242dbfeb512`; the fork checkout was merge `824347060429dc587758ec1540a1a32c1438e05f`. Both complete trees are **`9a27169f0bd8de82069b2a948730369b7f3182fd`**, identical to published head `4149882`. Each W012 workflow independently rebuilt native `hyb` and WASM from its own checkout before comparing the entire normalized diagnostic envelopes. Each also passed the complete `go test -race -count=1 ./...` suite, `go vet ./...`, gofmt enforcement, and its separate Go-only consumer, whose log confirms `cargo` and `rustc` were not invoked. Compiler tests, corpus verdicts, differential agreement, runtime hygiene, and examples passed in both compiler workflows. The EOF regressions are included in the full analyzer suite; its nonverbose logs report package results rather than individual test names. + +**Current upstream status (2026-10-19 07:13 UTC):** PR #556 reports `mergeable_state=clean`, `mergeable=true`, head `4149882`, and base `aa4a9d91`; the actual main ref remains `aa4a9d91`. The integration conflict and missing new-head upstream validation are resolved. The PR remains open and M1 remains pending for November 1. The earlier HTTP 403 limits editing the upstream PR description; it does not describe the result of these successful code-validation workflows. This completion update changes only `PLAN.md`, `STATUS.md`, and this report; the tested code, tests, workflows, and archived benchmark bytes remain unchanged. + +**Correction to the previous base claim (2026-10-19 06:54 UTC):** `f6e4d929` was the SHA reported by PR #556's `base` metadata, not the live `main` branch. Git transport and both upstream ref and branch APIs independently identify `main` as `aa4a9d91`. Comparing `0d1dbf7` with that actual ref reports 35 commits ahead and **251 behind**, with `f6e4d929` as their merge base. The earlier `behind_by=0` comparison was valid only against `f6e4d929`; it did not establish integration with live main. The PR still reported `dirty`, and its merge ref still pointed to historical `b49e0606`. The upstream repository can read the current head object, and the PR's 46 changed-file records match the comparison against its merge base, so neither an inaccessible head nor a different patch explains the discrepancy. + +### Completed validation of the earlier recorded trees + +Intermediate commit `61d5bd9b0655a7f6e638f010cab6093358aa6f51` joined the EOF fix `fd4f084` with `f6e4d929`. Its [W012 validation](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39210700789) and [compiler CI](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39210708370) passed. Both checker artifacts were rebuilt from that checkout: **61/61** expected corpus verdicts and complete normalized native/WASM envelopes, `go test -race -count=1 ./...`, `go vet ./...`, compiler differential agreement, and the separate Go-only consumer all passed. Executed merge `75b89aa59aed286c141dd8c87e2238c31f5d24e6` and `61d5bd9` share tree `880e3aee3dfc92dd10893e0d92f09439224707a4`. + +The later documentation head `0d1dbf7` also passed [W012 validation](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39211484374) and [compiler CI](https://github.com/1932819331hy-debug/Go-Hybrid/actions/runs/39211484338), with the same **61/61** correctness gates and a Go-only consumer that did not invoke `cargo` or `rustc`. Executed merge `0e39fd07cf1047f3dce5110f6c974dda9ab3808e` and published `0d1dbf7` share tree `7e44239def4ef5fe829c83359638da226a961cd9`. These results and the local **60/60** and earlier hosted **113/113** results below each belong to their stated checkout. The new integration's 113/113 result comes from its own fresh builds and runs above; it does not reuse the older merge's result or artifacts. + +### Latest correctness fix: preserve real EOF diagnostics + +The Hybrid lexer reports EOF on the empty final line after a newline. Go's ordinary token line table omits that line, so an unterminated block ending in LF or CRLF previously returned an analyzer infrastructure error instead of its real `HYB-E0002` diagnostic. Empty source likewise had no usable EOF position. + +The analyzer now reserves one byte only in its token position table. The checker still receives the original source, and coordinate validation still uses the original content length. The new regressions verify LF, CRLF, repeated trailing newlines, the no-final-newline control, Unicode scalar-to-byte conversion, empty source, and rejection of positions beyond EOF. + +**Executed local validation:** Go 1.26.9, Rust 1.99.0 and Node 24.19.0, using source `6dafd4973114fe35c77133db89c19828a0ad07b6` plus the EOF fix. Both native and WASM checker artifacts were built from that same checkout. + +- The unmodified analyzer fails four new EOF cases; the no-final-newline and Unicode controls pass. +- The fixed analyzer passes all six cases. +- `go test -race -count=1 ./...` and `go vet ./...` pass. +- `node test_wasm_parity.js` reports **60/60** corpus verdict agreement and **60/60** complete normalized native/WASM envelopes for this branch checkout. +- `gofmt` and whitespace checks are clean. + +The branch's 60-fixture checkout is different from the 113-fixture synthetic merge used in earlier hosted CI. The earlier [validation run 39195170609](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609) tested merge `b49e0606be4746d90cb5399ebdb04166024676b5`, not the bare branch head. Mixing a WASM artifact from that merge with a native binary from the branch produces invalid parity evidence because their compiler trees differ. Only the same-checkout results above establish this fix's local correctness; new hosted checks remain separate evidence. + +### Performance evidence preserved for M1 review + +The [committed raw measurement archive](spikes/w012-inprocess/validation-2026-10-19/README.md) preserves byte-identical JSONs from run 39195170609, with checked ZIP/file SHA-256 values and separate source, merge and base identities. The timing values displayed here remain that historical snapshot; later CI measurements have not replaced or relabeled the archived data. Original attachments expire October 26, before the November 1 review; the repository archive preserves them, and subsequent request/cache attachments now retain data for 30 days. + +| Question | Observed evidence | Boundary | +| --- | --- | --- | +| Can the Rust checker run in-process? | Wazero executes the existing WASM checker; same-checkout native/WASM parity passes | A ready-to-install WASM distribution is not shipped | +| Is the public request path faster? | Preserved 25-file/100-edit snapshot: native **1.150 ms** p50; public `Engine.Check` **2.096 ms**; experimental sequential reuse **1.451 ms** | Public WASM was slower on that host; this is an after-write changed-file request proxy, not editor or whole-package latency | +| Does a compilation cache help cold startup? | Two distinct Go processes recorded **395.981 ms → 12.217 ms** with matching artifacts | This does not improve hot checks or cache analysis verdicts | +| Can Go run a precompiled checker without invoking Rust? | The separate Go-consumer job in run 39195170609 passed with cargo/rustc command names deliberately blocked | The hosted VM may still contain Rust; no installer or CI-image-footprint result is claimed | +| Are source diagnostics preserved? | Structured Engine results preserve warnings; the vet adapter reports errors only, including actual EOF positions | Stock gopls and unsaved `.hy` buffer integration remain unimplemented | + +**M1 design decision:** keep one Rust semantic implementation with a WASM adapter and retain native CLI for measured batch performance. The remaining product questions are editor integration, distribution, broad workloads, memory/RSS and CI footprint. Historical 18.7× and sub-millisecond prototype figures below must not be cited as performance of the current public path. + + +--- + +## Earlier maintainer decision memo (archival snapshot, 2026-10-19 JST) + +**Scope:** An executable M1 feasibility spike for [W-012 / #268](https://github.com/limitlessprojects/Go-Hybrid/issues/268), not a request to replace `hyb`, claim a stock-`gopls` plugin, or merge a production IDE feature. This answers solicitation #5 by investigating the *delivery/execution boundary*; W-011 independently attacks the `.hy`/Go syntax surface. The two approaches are adjacent and can compose. + +**Latest source/evidence:** [PR #556](https://github.com/limitlessprojects/Go-Hybrid/pull/556), commit `7b4d8c35e7349dba532084ae1555412ce6d314a7`; [W-012 validation run 39171041567](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39171041567) **PASS**; [compiler CI 39171041397](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39171041397) **PASS**. The validation log reports **113/113 corpus expected verdict agreements**, **113/113 full native/WASM normalized JSON-envelope agreements**, `go test -race -count=1 ./...`, and `go vet ./...` successful. Each fixture count belongs to that tested checkout, not an unbounded future corpus. + +**Execution/correctness hardening:** Rust's authoritative `hy-check` is compiled as a WASM guest and invoked from a Go `analysis.Analyzer` via wazero. Caller cancellation and an internal 30-second timeout fail closed; trap, malformed envelope and guest-memory errors cannot silently produce an accepted verdict. Anonymous module instances make concurrent checks on the shared runtime viable (the named-instance failure was reproduced on the rescue branch). Managed warning diagnostics remain nonfatal to `go vet`. Unicode-scalar positions convert to Go byte offsets and invalid source coordinates fail closed. Input discovery is deterministic; unreadable declared `.hy` inputs cannot be skipped silently. + +**Cache/invalidation boundary:** Stock `go vet -vettool` does not track sibling `.hy` bytes as action-cache dependencies. The supported `hybvet vet` wrapper therefore uses `go vet -a` plus a pre-run SHA-256 workspace snapshot, verifies the actual bytes at each declared/scanned `.hy` read, then verifies the snapshot again after vet. A dedicated regression tests valid→invalid→valid edits, a second `.hy` file, mutated/deleted/added files and stale snapshots. The conservative `-a` choice favors correctness over production-speed cache performance. + +**Falsification and cost — three maintainer questions:** + +1. **WASM/Wazero versus a native-Go checker rewrite:** The embedded WASM reuses one existing semantic engine instead of maintaining a second borrow-checker implementation. Portability and single-source semantics are plausible gains. This does not by itself prove a no-Rust *build* or no-binary runtime distribution; a compiled WASM artifact still has to be supplied. +2. **Latency:** The historical `0.269 ms / 18.7×` benchmark measured a pre-instantiated experimental guest path, not today's public `Engine.Check`. The separate 100-sample Linux snapshot for the corrected public wrapper measured p50 **2.143 ms** versus native subprocess+JSON p50 **1.204 ms**, i.e. the implemented wrapper was **slower** on that host. Latest head and real IDE keypress-to-diagnostic latency have not been rebenchmarked. Do not infer sub-millisecond user-visible checks or an end-to-end speedup. +3. **Fail-closed input/diagnostic contract:** Proven at the WASM ABI and on-disk Go analyzer boundary for the tested corpus and injected failure classes. A stock `gopls` instance does **not** dynamically load third-party analyzers or open `.hy` unsaved buffers. `pass.TypesInfo` supplies Go's *structural types*, not Hybrid's `Produce/Borrow/Mutate/Consume` ownership effects; typed descriptors and an editor-specific LSP/host are still needed. + +**M2 acceptance/failure experiment to run next (not yet claimed):** benchmark 100 sequential *real source edits* in a 25-file workspace against a native CLI and a deliberately specified in-process request path on the **same host**, reporting cold startup separately from warm p50/p95/p99, cancellation, memory/RSS and pre/post artifact sizes. Explicitly include CI onboarding/image-footprint cost with and without the Rust toolchain. If the only correct public path remains slower, publish that as a negative result and recommend retaining the native CLI for batch checks while limiting WASM to embedding use cases where deployment characteristics, rather than latency, justify it. + +**Review disposition requested:** independent **M1 feasibility verdict** on the measured tradeoff, not a claim of achieved `gopls`, browser playground or a $250 award. The report below retains historical observations unchanged for provenance. + +--- + +## Earlier hosted validation (2026-10-18 UTC / 2026-10-19 JST) + +The [latest hosted M1 parity workflow](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39171466042), against PR head `585dac8922a481124fcacd3cb2424f05c8b8e7e2` merged with its then-current base, passed **113/113 checker-corpus fixture agreements** and **113/113 complete normalized native/WASM JSON-envelope comparisons**. Its `go test -race -count=1 ./...` and `go vet ./...` steps succeeded. The [independent compiler CI](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39171466089) passed as well. + +This latest head fails closed on unreadable declared `.hy` files rather than silently skipping them; it also canonicalizes file names to prevent duplicate diagnostic reports and emits diagnostics in deterministic path order. Regression tests for the read-failure, path-alias, and ordering cases ran in hosted CI. The 113-fixture count describes this specific tested merge, not future upstream corpus revisions. + +**Prior hosted snapshot:** [run 39143438366](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39143438366) passed 79/79 at `fef92eecf31ee1c3fe383a2953e17ea1840b9ff0`. + +That head also adds a **30-second fail-closed limit to the public `Engine.Check`** and a cancellable `CheckContext` for embedding hosts. Context-log snapshots copy all slice backing arrays, and cancellation, subsequent runtime reuse, and snapshot isolation have regression tests. Hosted green CI establishes the tested behavior, not unsaved-buffer, LSP, or interactive editor integration. + +## Earlier 60-fixture correctness and benchmark snapshot (retained for provenance) + +The tested correction at `34c759f91e4ff76c8b6d587208cbf4bea6a51ca0` fixes a real Go-wrapper loss of accepted managed-safety warnings. On that older **60-fixture** checkout, raw native/WASM envelope parity was **60/60**; Go wrapper modeled-field parity improved from **58/60** to **60/60**, with the race suite passing. + +The original 100-sample cloud measurements put the **actual public `Engine.Check` at p50 2.143 ms / p99 7.176 ms**, versus native subprocess + JSON validation at **1.204 / 1.796 ms**. The reused-instance experimental boundary was **1.097 / 1.521 ms** and is not the analyzer's actual per-call instance strategy. These results establish neither an 18.7× speedup nor sub-millisecond IDE latency. The more recent timeout change was **not rebenchmarked** here; different hosts and harnesses prevent treating these figures as a controlled performance regression comparison. + +See the [measurement record, commands, limitations, and raw samples](spikes/w012-inprocess/validation-2026-10-18/README.md). Its 60-fixture and timing numbers are a historical snapshot, not the latest hosted acceptance run. + +## Historical review snapshot follows + +The original narrative below is retained for provenance. Its 55-fixture, 903.4 KB, 0.269 ms, 18.7×, and amortization figures describe the earlier recorded baseline only. They must not be quoted as measurements of the current corrected wrapper. Its architectural recommendations are hypotheses/tradeoffs rather than proof of completed editor integration. + +# W-012 M1 Technical Due Diligence & Confirmatory Architectural Audit +## In-Process Analysis Driver for Go-Hybrid: WASM/Wazero vs. Native Go vs. Subprocess Architecture + +- **Subject**: Issue [#268](https://github.com/limitlessprojects/go-hybrid/issues/268) — W-012 Wildcard Proposal +- **Evaluation Target**: Milestone 1 (M1) Feasibility & Proof-of-Concept Confirmatory Due Diligence +- **Base Upstream Commit**: `e2a5f7d5eb90becb210df274765adfa50c99b21b` (`upstream/main`, Session-15) +- **Local Branch**: `spike/w012-inprocess-analyzer` +- **Preregistration Commit**: `58ad41ae78f240680525a634aad8309fc9ad7281` +- **Protocol Class**: Preregistered Confirmatory Replication (N=100 iterations) +- **Status**: **STATUS: PUBLIC EVIDENCE — AWAITING M1 REVIEW** +- **Review Date**: 2026-10-18 + +--- + +## 1. Executive Verdict: CONDITIONAL GO (GO FOR REVIEW) + +- **Verdict**: **CONDITIONAL GO (GO FOR REVIEW)** (Pivot Architecture A to Persistent Embedded WASM / Standalone LSP Server; Terminate Architecture B Native Go Rewrite; Reject stock `gopls` plugin claims; Submit to 2026-11-01 M1 Review Window). +- **Core Summary**: + 1. **Architecture A (Rust -> wasm32-unknown-unknown -> Wazero)** is **TECHNICALLY PROVEN**: + - Compiles with 0 code modifications into a standalone **903.4 KB** (`925,098` bytes) WASM binary. + - Achieves **100.0% verdict parity across all 55 corpus fixtures** (39 REJECT, 2 ACCEPT-MANAGED, 14 ACCEPT). + - Achieves steady-state in-process check latency of **0.269 ms (p50)** and **0.278 ms (mean)**, delivering a representative **18.7x speedup** (≈ 17–19x) over native CLI process invocation (5.038 ms p50, 5.122 ms mean across 100 runs). + - No state drift was observed during the tested horizon across 2,000 sequential and 3,200 concurrent runs. Long-duration memory behavior remains unproven. + 2. **Architecture B (Self-hosted Native Go Rewrite)** is a **HARD NO-GO**: + - Rewriting `hy-check` (a large multi-thousand-line semantic rewrite; `hy-check` core alone is 16,663 LOC across 16 modules of Polonius-style alias tracking, non-lexical loan windows, and shadow scoping) creates permanent semantic divergence, duplicate maintenance overhead, and violates the project's zero-duplication invariants. + 3. **Proposal Flaws Corrected & Hardened**: + - **Stock `gopls` Plugin Impossibility**: `analysis.Analyzer` can read non-Go files as auxiliary inputs, but stock `gopls` does not parse `.hy` into Go AST/types nor dynamically load arbitrary third-party analyzers. + - **Baseline Subprocess Latency Corrected**: Issue #268's original estimate of 15–40 ms baseline subprocess latency was **falsified/overestimated** on this host (measured p50: 5.038 ms). However, in-process WASM (0.269 ms) retains a decisive 18.7x steady-state speedup. + - **Cold-Start Amortization & Break-Even**: Wazero cold JIT compilation takes **346.119 ms (p50)** without cache. However, Wazero's persistent filesystem cache (`NewCompilationCacheWithDir`) slashes compilation to **12.919 ms (p50)**, achieving break-even against the native CLI in **under 3 checks**. + - **Go `types.Info` Boundary**: `types.Info` can eliminate a meaningful subset of structural foreign-type stubbing (package whitelists, named type existence, method signatures, scalar atomic basic-type discrimination), but **cannot** infer Hybrid linear ownership, loan windows, or mutation effects (`Produce`, `Borrow`, `Mutate`, `Consume`). An ABI descriptor table remains mandatory. + - **Safety Invariant Enforced**: Diagnostic rejection (`Verdict::Reject`) and infrastructural execution failure (`Verdict::AnalysisFailure`) are strictly decoupled. Infrastructure failures fail closed and never masquerade as user code rejections. + +--- + +## 2. Scope of What Was Actually Proven + +To maintain absolute maintainer credibility, this audit explicitly segregates verified empirical proof from partial findings and disproven assumptions. + +### PROVEN +1. **Rust Checker WASM Target**: `hy-syntax`, `hy-check`, and `hy-diag` compile directly to `wasm32-unknown-unknown` without OS dependencies. +2. **Wazero In-Process Embedding**: Pure Go host embedding via `github.com/tetratelabs/wazero v1.9.0` executes without CGO or external toolchains. +3. **Corpus Verdict Parity**: 55/55 corpus fixtures match as normalized parsed JSON envelopes (input filenames normalized only) against the native CLI across all diagnostic codes and verdicts. +4. **Warm Latency Advantage**: Pure in-process WASM checking executes in **0.269 ms (p50)** vs **5.038 ms (p50)** for native CLI subprocesses (18.7x speedup). +5. **Thread and State Isolation**: No state drift or obvious memory-growth signal was observed during the 5,200-call test horizon in Rust and Wazero. +6. **Failure-Domain / Fail-Closed Separation**: Fault injection tests (WASM trap, malformed JSON, invalid memory bounds, alloc failure, context timeout) deterministically trigger `AnalysisFailure` and fail closed, never returning `Accept`. +7. **Filesystem Compilation Caching**: Wazero filesystem cache drops compilation time from 346.119 ms to 12.919 ms (p50), breaking even in 3 checks. +8. **On-Disk Singlechecker / `go vet` Integration**: Sibling `.hy` files are discovered via disk scan, and exact source spans map to `token.FileSet`, outputting exact `line:column` positions in `go vet`. + +### PARTIALLY PROVEN +1. **Go `analysis.Analyzer` Integration for Sibling `.hy`**: + - Proven: A real `analysis.Analyzer` can discover sibling `.hy` files on disk, register them in `pass.Fset` via `AddFile` + `SetLinesForContent`, and report exact `line:col` diagnostics. + - Limitation: The documented contract of `pass.ReadFile` only permits files listed in `OtherFiles`, `IgnoredFiles`, or `Files`. Stock Go toolchains omit `.hy` from these sets. The spike demonstrates on-disk `singlechecker`/`go vet` integration, not virtualized/unsaved editor-buffer integration. +2. **Go Semantic Context Replacing Structural Stubs**: + - Proven: `types.Info` can eliminate structural stubbing for imported packages, type names, and basic-type atomics. + - Limitation: Cannot infer Hybrid linear ownership, loan windows, or consuming calls. Hybrid-specific semantic descriptors remain necessary. + +### NOT PROVEN / DISPROVEN +1. **Stock `gopls` Dynamic Integration**: Disproven. Stock `gopls` has no mechanism to load external `analysis.Analyzer` binaries dynamically at runtime. +2. **Arbitrary Unsaved `.hy` Editor-Buffer Support**: Disproven under stock `go/analysis`. Non-Go files are not recognized as language documents in `gopls`. +3. **Pure Go Compiler Rewrite (Option B)**: Disproven as economically and architecturally unviable. + +--- + +## 3. Real `analysis.Analyzer` Spike & Contract Audit + +A minimal, real, runnable prototype was constructed in [`spikes/w012-analyzer/`](spikes/w012-analyzer/) adhering to `golang.org/x/tools/go/analysis`. + +### The `pass.ReadFile` Contract vs. On-Disk Execution +The official Go documentation for `Pass.ReadFile` specifies: +> *"The only valid file names are the elements of OtherFiles and IgnoredFiles, and names returned by Fset.File(f.FileStart).Name() for each f in Files. Analyzers must use this function (if provided) instead of accessing the file system directly. This allows a driver to provide a virtualized file tree (including, for example, unsaved editor buffers) and to track dependencies precisely to avoid unnecessary recomputation."* + +Empirical findings from [`analyzer_test.go`](spikes/w012-analyzer/analyzer_test.go): +1. **`pass.Files`**: Length strictly 1 (`sample.go`). Go parser ignores non-Go files. +2. **`pass.OtherFiles` & `pass.IgnoredFiles`**: Both are empty (`[]`). Non-Go/non-C files are not tracked by the Go toolchain. +3. **Execution Domain**: + > *The current spike demonstrates on-disk singlechecker/go-vet integration, not virtualized/unsaved editor-buffer integration.* + Sibling `.hy` discovery currently operates via directory scanning (`os.ReadDir`) using the directory of the Go package anchor. + +### Source-Position Mapping +Initial spikes emitted diagnostics at `token.NoPos` (`-:`). By registering discovered `.hy` files into `pass.Fset` via: +```go +fileToken := pass.Fset.AddFile(filename, -1, len(content)) +fileToken.SetLinesForContent([]byte(content)) +``` +Exact line and column positions from Hybrid diagnostics are translated to `token.Pos`: +```bash +$ go vet -vettool=./cmd/hybvet/hybvet ./testpkg +testpkg/sibling.hy:6:5: [HYB-E1002] `y` is declared and never used — Go refuses to build the emitted `y := ...` (declared and not used) (HYB-E1002) +testpkg/sibling.hy:7:5: [HYB-E1002] `z` is declared and never used — Go refuses to build the emitted `z := ...` (declared and not used) (HYB-E1002) +testpkg/sibling.hy:7:13: [HYB-E3003] `x` is read after ownership was moved away +exit status 1 +``` + +- **Verdict**: **`Exact on-disk .hy positions: CONFIRMED`**. +- **Overall Contract Verdict**: **`analysis.Analyzer contract: PARTIAL FEASIBILITY`** (On-disk `go vet` passes fully verified; virtualized editor buffer streaming not supported by stock driver). + +--- + +## 4. Corrected `gopls` Integration Analysis + +Issue #268 asserted: *"Zero-Install IDE Diagnostics (<1ms vs 35ms) inside gopls by implementing an `analysis.Analyzer`"*. This conflated batch `go vet` passes with editor language server protocols. + +### Definitive Technical Reality +> **Canonical Statement**: +> *analysis.Analyzer can read non-Go files as auxiliary inputs, but stock gopls does not parse `.hy` into Go AST/types nor dynamically load arbitrary third-party analyzers.* + +### Detailed Technical Breakdown +1. **Dynamic Plugin Architecture**: Unlike tools supporting shared object plugins (`.so`), stock `gopls` compiles all enabled analyzers directly into its binary (`golang.org/x/tools/gopls/internal/analysis`). There is no runtime configuration or flag to dynamically load an out-of-tree `analysis.Analyzer`. +2. **Document & Syntax Boundary**: `gopls` is built around the Go Abstract Syntax Tree. It tracks buffers corresponding to `.go` file URIs. A `.hy` file opened in an editor is not recognized by `gopls` as a managed language document. +3. **Realistic Integration Pathways**: + - **Path A: Upstream Analyzer into `gopls`**: Infeasible. The Go tools team does not upstream external DSL checkers into stock `gopls`. + - **Path B: Custom `gopls` Fork**: Feasible for enterprise forks, but completely forfeits zero-install. + - **Path C: Standalone LSP Server (`hyb lsp`)**: **The Proven Industry Standard**. Implementing an LSP server in Rust using `tower-lsp` or in Go wrapping Wazero. Editors route `.hy` files to `hyb lsp` and `.go` files to `gopls`. + - **Path D: W-011 Directive-on-Go Migration**: If W-011 succeeds in expressing Hybrid rules as `//hy:` directives inside valid `.go` files, `analysis.Analyzer` becomes natively compatible with Go tooling. + - **Path E: Editor Extension with In-Process Engine**: A VS Code extension embedding the 903 KB `hy_wasm.wasm` directly into Node.js/V8, providing <0.5 ms diagnostics with zero local daemon installation. + +--- + +## 5. Corrected `types.Info` / `foreign.rs` Matrix + +Go's `types.Info` provides rich lexical and semantic context (`Types`, `Defs`, `Uses`, `Selections`, `Scopes`, `Instances`, `Implicits`). However, Go's type system has **zero linear ownership or borrow concepts**. + +### Structural vs. Hybrid-Specific Semantics +- **Category A (Structural Context - Solvable via Go Types)**: + `TypesInfo` can eliminate a meaningful subset of structural foreign-type stubbing: + - Hardcoded package whitelists (`BUILTIN_PACKAGES` in `foreign.rs`). + - Named struct and interface type existence. + - Method signatures and return types. + - Scalar atomic discrimination: inspecting whether a `sync/atomic` type wraps a basic primitive vs an arbitrary pointer (`atomic.Pointer[T]`). +- **Category B (Hybrid Semantic Effects - Irreplaceable by Go Types)**: + Go types cannot infer or derive Hybrid-specific ownership semantics: + - `CallMode::Produce` (creates fresh owned handle). + - `CallMode::Borrow` (initiates an A2 borrow loan window). + - `CallMode::Mutate` (conflicts with live borrow loans). + - `CallMode::Consume` (Rule A7 ownership transfer moving receiver away, e.g. `finish()`). + - Handle roles (`OwnHandle` vs `BorrowHandle` element view). + - Fail-closed security boundaries for unproven types (Rule FC, GUARANTEES row 11). + +- **Conclusion**: `TypesInfo can eliminate a meaningful subset of structural foreign-type stubbing, but Hybrid-specific ownership/effect descriptors remain necessary.` These descriptors should be formalized as an external schema (`interfaces/descriptors.json` per SPEC OQ8). + +--- + +## 6. Confirmatory 100-Iteration Benchmark Ledger + +Measurements conducted on Apple Silicon under Darwin x86_64 emulation / arm64 runtime, running **100 iterations per case** against fixture `08-a3-map-slot-invalidated.hy`: + +### Confirmatory Benchmark Table (N=100 Iterations) + +| Execution Case | Phase | Min | p50 (Primary) | p90 (Primary) | p99 | Mean (Secondary) | +|---|---|---|---|---|---|---| +| **Baseline: Native CLI Subproc** | Fork + Exec + Check | 4.721 ms | **5.038 ms** | **5.476 ms** | 5.935 ms | 5.122 ms | +| **Case A: Fresh Runtime (No Cache)** | Module Compilation | 339.287 ms | **346.119 ms** | **371.130 ms** | 417.249 ms | 353.878 ms | +| | Module Instantiation | 0.170 ms | 0.210 ms | 0.270 ms | 0.340 ms | 0.216 ms | +| | First Check Execution | 0.700 ms | 0.760 ms | 0.880 ms | 1.120 ms | 0.772 ms | +| | **Case A Total Latency** | 340.200 ms | **347.100 ms** | **372.300 ms** | 418.700 ms | 354.967 ms | +| **Case B: Reused CompiledModule** | Module Instantiation | 0.060 ms | 0.075 ms | 0.110 ms | 0.140 ms | 0.078 ms | +| | Check Execution | 0.430 ms | 0.610 ms | 0.830 ms | 1.250 ms | 0.640 ms | +| | **Case B Total Latency** | 0.513 ms | **0.694 ms** | **0.947 ms** | 1.392 ms | 0.723 ms | +| **Case C: In-Memory Cache** | Cached Compilation | 9.583 ms | **9.887 ms** | **10.356 ms** | 21.417 ms | 13.384 ms | +| | Module Instantiation | 0.180 ms | 0.230 ms | 0.280 ms | 0.320 ms | 0.237 ms | +| | Check Execution | 0.710 ms | 0.780 ms | 0.890 ms | 1.020 ms | 0.803 ms | +| | **Case C Total Latency** | 10.500 ms | **10.900 ms** | **11.500 ms** | 22.800 ms | 14.437 ms | +| **Case D: Filesystem Cache** | Persistent FS Compilation | 12.337 ms | **12.919 ms** | **13.400 ms** | 14.090 ms | 12.974 ms | +| | Module Instantiation | 0.120 ms | 0.160 ms | 0.200 ms | 0.250 ms | 0.172 ms | +| | Check Execution | 0.710 ms | 0.890 ms | 0.990 ms | 1.130 ms | 0.923 ms | +| | **Case D Total Latency** | 13.200 ms | **13.980 ms** | **14.600 ms** | 15.500 ms | 14.356 ms | +| **Case E: Pure Warm In-Process** | Pre-instantiated Check | 0.260 ms | **0.269 ms** | **0.282 ms** | 0.740 ms | 0.278 ms | + +--- + +## 7. Break-Even Quantitative Analysis + +> [!NOTE] +> **Latency Baseline Finding**: Original 15–40ms normal CLI latency estimate was overstated on this host; real native CLI subprocess latency executes at **5.038 ms p50** (mean: **5.122 ms**). +> Representative steady-state speedup is **≈ 17–19x** (**18.7x** observed at p50). + +### Metric Comparison +- **Native CLI Subprocess Latency**: + - $T_{\text{subprocess\_p50}}$: **5.038 ms** + - $T_{\text{subprocess\_mean}}$: **5.122 ms** +- **Pure Warm In-Process WASM Latency**: + - $T_{\text{warm\_wasm\_p50}}$: **0.269 ms** + - $T_{\text{warm\_wasm\_mean}}$: **0.278 ms** +- **Representative Speedup (p50-based)**: + $$\text{Speedup}_{\text{p50}} = \frac{5.038\text{ ms}}{0.269\text{ ms}} \approx \mathbf{18.7\times}$$ + *(Mean-derived ratio: $5.122 / 0.278 \approx \mathbf{18.4\times}$)*. +- **Latency Advantage per Check ($\Delta T$ p50)**: + $$\Delta T = 5.038 - 0.269 = \mathbf{4.769\text{ ms saved per check}}$$ + +### Calculated Break-Even Thresholds (p50-Derived) +1. **Cold Process without Cache (Case A)**: + $$N_{\text{break-even\_cold\_p50}} = \frac{346.119\text{ ms}}{4.769\text{ ms}} \approx 72.6 \longrightarrow \mathbf{73\text{ checks}}$$ + *(Outlier-sensitive mean estimate: $353.878 / 4.844 \approx \mathbf{74\text{ checks}}$)*. + - For single-file one-shot vet passes without cache, native CLI remains faster due to JIT compilation overhead. +2. **Process with Filesystem Cache (Case D, `NewCompilationCacheWithDir`)**: + $$N_{\text{break-even\_fs\_p50}} = \frac{12.919\text{ ms}}{4.769\text{ ms}} \approx 2.71 \longrightarrow \mathbf{3\text{ checks}}$$ + *(Mean estimate: $12.974 / 4.844 \approx \mathbf{3\text{ checks}}$)*. + - With persistent filesystem compilation caching, in-process checking breaks even on the **3rd file check**. +3. **Long-Lived Service / LSP Server (Case B / E)**: + - Compilation cost is paid once at startup. The recorded pre-hardening warm microbenchmark measured **0.269 ms** and **18.7x**. This is not an end-to-end keystroke/save measurement. + +--- + +## 8. Failure-Domain / Fail-Closed Model + +```go +type AnalysisVerdict string + +const ( + VerdictAccept AnalysisVerdict = "Accept" + VerdictReject AnalysisVerdict = "Reject" + VerdictAnalysisFailure AnalysisVerdict = "AnalysisFailure" +) + +type AnalysisResult struct { + Verdict AnalysisVerdict + Diagnostics []Diagnostic // Populated ONLY on VerdictReject + Failure error // Populated ONLY on VerdictAnalysisFailure + Summary *Summary +} +``` + +### Safety Invariant +$$\text{VerdictAnalysisFailure} \cap (\text{VerdictAccept} \cup \text{VerdictReject}) = \emptyset$$ + +### Confirmatory Fault Injection Suite Results + +| Injection Scenario | Injected Condition | Expected Verdict | Observed Verdict | Status | +|---|---|---|---|---| +| **Positive Control** | Valid program (`09-a3-map-slot-valid.hy`) | `VerdictAccept` | `VerdictAccept` | **PASSED** | +| **Negative Control** | A3 violation (`08-a3-map-slot-invalidated.hy`)| `VerdictReject` | `VerdictReject` | **PASSED** | +| **Fault 1: WASM Trap** | Real minimal guest `unreachable` trap (plus original mock) | `VerdictAnalysisFailure` | `VerdictAnalysisFailure` | **PASSED** | +| **Fault 2: Malformed JSON**| Truncated / corrupt diagnostic payload | `VerdictAnalysisFailure` | `VerdictAnalysisFailure` | **PASSED** | +| **Fault 3: Invalid Pointer**| Out-of-bounds guest memory pointer (50000) | `VerdictAnalysisFailure` | `VerdictAnalysisFailure` | **PASSED** | +| **Fault 4: Alloc Failure** | Mock allocator error returned at ABI boundary | `VerdictAnalysisFailure` | `VerdictAnalysisFailure` | **PASSED** | +| **Fault 5: Timeout** | Pre-expired context plus real infinite-loop guest interrupted during execution | `VerdictAnalysisFailure` | `VerdictAnalysisFailure` | **PASSED** | + +- **Status**: **Safety fail-closed CONFIRMED; diagnostic/error-domain separation implemented.** + +--- + +## 9. W-011 Dependency Matrix + +| Capability | Without W-011 (Standalone `.hy` Surface) | With W-011 (Go Syntax Surface) | +|---|---|---| +| **Browser WASM Playground** | **Fully Functional**. Zero dependency on Go toolchain; compiles and runs `.hy` directly in V8/browser Monaco editor. | Fully Functional. | +| **Standalone LSP (`hyb lsp`)** | **Proposed architecture**. The embedded engine is demonstrated; no standalone LSP implementation or end-to-end editor latency is established here. | Redundant if gopls handles directives. | +| **`go vet` Integration** | **Partial Feasibility**. Discovers sibling `.hy` via directory scan anchor; emits exact `line:col` diagnostics. | **Native**. Runs directly on `pass.Files` with exact Go `token.Pos` source spans. | +| **Stock `gopls` Integration** | **Infeasible**. Stock `gopls` cannot parse `.hy` or load out-of-tree analyzer plugins. | **Infeasible dynamically**. Requires either upstreaming or custom gopls build. | +| **Custom `gopls` Build** | Requires hacking `.hy` grammar into gopls parser. | **Fully Functional**. Direct registration of analyzer on Go files. | +| **Live `TypesInfo` Integration** | Indirect. Requires companion Go package declaration for foreign stubs. | **Direct & Native**. Hybrid checker directly queries `pass.TypesInfo` of checked code. | +| **`.hy` Source Diagnostics** | Native byte-exact diagnostics via SPEC §9.6 envelope. | Obsoleted (diagnostics map to Go AST spans). | +| **Eliminate Foreign Stubs** | Partial. Eliminates package and scalar atomic lists. | **Complete**. All structural types inferred from active Go package. | + +--- + +## 10. Updated Assumption Matrix + +| # | Assumption in Issue #268 | Original Claim | Hardened Empirical Verdict | Final Evidence & Architecture Basis | +|---|---|---|---|---| +| 1 | Subprocess startup is a bottleneck | 15–40 ms latency | **PARTIALLY CONFIRMED** | Original absolute latency assumption (15–40 ms) was overstated (measured p50: 5.038 ms). However, relative in-process advantage is confirmed (18.7x speedup at 0.269 ms). | +| 2 | Rust checker compiles to WASM | Feasible | **CONFIRMED** | `wasm32-unknown-unknown` produces clean 903.4 KB binary with 0 OS dependencies. | +| 3 | Wazero simplifies deployment | High benefit | **PARTIALLY CONFIRMED** | Eliminates CGO and Rust toolchain requirements; requires compilation cache (12.919 ms) to avoid cold 350 ms penalty. | +| 4 | `TypesInfo` replaces `foreign.rs` | Fully replaces | **PARTIALLY CONFIRMED** | Eliminates structural stubs (packages, names, atomic types); cannot replace linear ownership or `CallMode::Consume/Borrow`. | +| 5 | `gopls` integration achievable | Easy plugin | **REJECTED** | Stock `gopls` has no dynamic plugin loader and ignores `.hy`. Must pivot to standalone LSP server. | +| 6 | `go vet` and `gopls` conflation | Distinct tools | **REJECTED (Conflated)** | Issue #268 incorrectly treated `analysis.Analyzer` as an in-process `gopls` plugin. | +| 7 | Fail-closed survives boundary | Safe | **CONFIRMED** | Hardened with explicit separation of `VerdictReject` vs `VerdictAnalysisFailure`. | +| 8 | Safe state persistence | No drift | **CONFIRMED** | 5,200 repeated passes in Rust and Wazero confirm zero state drift or obvious memory-growth signal. | +| 9 | Type transport simplicity | Simple | **REJECTED** | Serializing Go `types.Info` across WASM linear memory is complex; lightweight ABI descriptor table is far cleaner. | +| 10 | Self-hosted Go rewrite viable | Alternative | **REJECTED** | Multi-thousand-line rewrite (core `hy-check` is 16,663 LOC) creates permanent twin-compiler divergence. | + +--- + +## 11. Final Architecture Recommendation + +> **Architectural Conclusion**: +> *Architecture A remains viable for an embedded checker, go-vet/singlechecker tooling, standalone LSP/daemon, and browser WASM. The current Analyzer spike does not establish stock-gopls support for independent .hy buffers.* + +1. **Ship Native CLI (`hyb check`)**: Remains canonical for one-shot batch builds and CI. +2. **Ship Standalone LSP Server (`hyb lsp`)**: Delivers sub-millisecond IDE diagnostics without hacking `gopls`. +3. **Ship Browser WASM Engine**: Leverage `hy_wasm.wasm` immediately for the online interactive playground. +4. **Shelve Option B (Native Go Rewrite)**: Permanently reject twin-compiler maintenance. +5. **Retain Spike as Reference**: Keep `spikes/w012-analyzer` as an unmerged reference harness pending future W-011 developments. + +--- + +## 12. Submission Package Metrics & Sign-Off + +- **Technical Review Readiness**: Evidence available for formal review; no numerical readiness or payout probability is inferred. +- **Actual M1 Payout Probability**: **UNKNOWN / Competition-Dependent** (17 competing entries for the $500 Hybrid Bounty pool; dependent on maintainer review priority in Discussion slot 12 on 2026-11-01). +- **Reviewed Upstream Commit**: `e2a5f7d5eb90becb210df274765adfa50c99b21b` (`upstream/main`) +- **Preregistration Commit**: `58ad41ae78f240680525a634aad8309fc9ad7281` +- **Global Status**: **STATUS: PUBLIC EVIDENCE — AWAITING M1 REVIEW** + + +## Closeout validation — 2026-10-18 + +The earlier corpus script checked expected fixture verdicts and the first error rule; it did **not** establish complete native/WASM diagnostic equality. The updated script now invokes the native CLI for every fixture and compares complete parsed JSON envelopes, normalizing only the input filename (including its occurrences in diagnostic notes). This is semantic JSON equality, not raw byte equality. + +Executed against the existing native/WASM artifacts from the reviewed baseline: + +```text +Corpus fixture agreement: 55/55 +Native/WASM normalized envelope parity: 55/55 +``` + +`go test ./...` passes for the analyzer module, including ten malformed-envelope cases, acceptance of additive fields, the original five boundary fault injections, a real minimal guest trap, and a real infinite-loop guest interrupted by a 50 ms context deadline. The allocator-failure test uses a mock error; it does not exhaust a real guest heap. The malformed-output tests now include valid JSON with missing/unknown verdicts and contradictory counts. The Analyzer invokes the same validated boundary as the failure harness and propagates infrastructure errors rather than reporting a code diagnostic. Wazero context cancellation is enabled explicitly. + +A rebuilt singlechecker driver reports the expected `sibling.hy:6:5`, `:7:5`, and `:7:13` diagnostics under `go vet -vettool`; exit 1 is the expected rejection of the invalid fixture. + +Reproduction (from repository root; build both artifacts from the reviewed baseline first): + +```sh +export HYBRID_WASM_PATH="$PWD/compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm" +export HYBRID_NATIVE_PATH="$PWD/compiler/target/release/hyb" +node test_wasm_parity.js +cd spikes/w012-analyzer +go test ./... +go build -o /tmp/hybvet ./cmd/hybvet +go vet -vettool=/tmp/hybvet ./testpkg +``` + +Benchmark provenance: all canonical 100-run timing figures above are the recorded snapshot at `b3d73c2c3f211f69e856abd089adc4cad9a21f50`, before this closeout added envelope validation. They were not rerun or relabeled as measurements of the changed wrapper. The added validation cost and end-to-end editor latency remain unmeasured. The parity script's per-file timings are separate and are not the canonical benchmark. Long-duration memory behavior remains unproven. Stock-gopls plugins, unsaved `.hy` buffers, and a completed standalone LSP are not demonstrated. + diff --git a/compiler/Cargo.lock b/compiler/Cargo.lock index c6ecd677..08cd5390 100644 --- a/compiler/Cargo.lock +++ b/compiler/Cargo.lock @@ -32,6 +32,15 @@ dependencies = [ "hy-diag", ] +[[package]] +name = "hy-wasm" +version = "0.0.1" +dependencies = [ + "hy-check", + "hy-diag", + "hy-syntax", +] + [[package]] name = "hyb" version = "0.0.1" diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 8aad5f5c..73ffbfe2 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/hy-check", "crates/hy-emit", "crates/hyb", + "crates/hy-wasm", ] # Go-Hybrid compiler workspace — PLAN.md 2026-10-11 decision: diff --git a/compiler/crates/hy-check/tests/inprocess_isolation.rs b/compiler/crates/hy-check/tests/inprocess_isolation.rs new file mode 100644 index 00000000..669d6fd0 --- /dev/null +++ b/compiler/crates/hy-check/tests/inprocess_isolation.rs @@ -0,0 +1,81 @@ +//! inprocess_isolation.rs — verifies repeated and concurrent in-process checker invocations +//! for zero state leakage, zero drift, and deterministic verdicts (W-012 due diligence). + +use hy_check::{check_file, Verdict}; +use hy_diag::Severity; +use hy_syntax::parse_file; + +const FILE_ACCEPT: &str = include_str!("../../../../tests/checker-corpus/16-move-send-clean.hy"); +const FILE_REJECT: &str = include_str!("../../../../tests/checker-corpus/01-shared-map-across-tasks.hy"); + +#[test] +fn repeated_inprocess_checker_has_zero_state_drift() { + let ast_accept = parse_file("accept.hy", FILE_ACCEPT).expect("parse accept"); + let ast_reject = parse_file("reject.hy", FILE_REJECT).expect("parse reject"); + + // Run 1,000 alternating passes in process + for i in 0..1000 { + let res_accept = check_file(&ast_accept); + assert_eq!( + res_accept.verdict, + Verdict::Accept, + "iteration {}: accept verdict drifted!", + i + ); + let errs_accept: Vec<_> = res_accept + .diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .collect(); + assert!(errs_accept.is_empty(), "iteration {}: leaked error into accept", i); + + let res_reject = check_file(&ast_reject); + assert_eq!( + res_reject.verdict, + Verdict::Reject, + "iteration {}: reject verdict drifted!", + i + ); + let errs_reject: Vec<_> = res_reject + .diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .collect(); + assert_eq!( + errs_reject.len(), + 1, + "iteration {}: reject error count drifted!", + i + ); + assert_eq!( + errs_reject[0].tier0_rule.as_deref(), + Some("A1c"), + "iteration {}: rule drifted!", + i + ); + } +} + +#[test] +fn concurrent_inprocess_checking_is_thread_safe() { + use std::thread; + + let handles: Vec<_> = (0..8) + .map(|_| { + thread::spawn(|| { + let ast_accept = parse_file("accept.hy", FILE_ACCEPT).expect("parse accept"); + let ast_reject = parse_file("reject.hy", FILE_REJECT).expect("parse reject"); + for _ in 0..200 { + let r1 = check_file(&ast_accept); + assert_eq!(r1.verdict, Verdict::Accept); + let r2 = check_file(&ast_reject); + assert_eq!(r2.verdict, Verdict::Reject); + } + }) + }) + .collect(); + + for h in handles { + h.join().expect("thread join"); + } +} diff --git a/compiler/crates/hy-wasm/Cargo.toml b/compiler/crates/hy-wasm/Cargo.toml new file mode 100644 index 00000000..dc4955e3 --- /dev/null +++ b/compiler/crates/hy-wasm/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "hy-wasm" +version = "0.0.1" +edition = "2021" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +hy-syntax = { path = "../hy-syntax" } +hy-check = { path = "../hy-check" } +hy-diag = { path = "../hy-diag" } diff --git a/compiler/crates/hy-wasm/src/lib.rs b/compiler/crates/hy-wasm/src/lib.rs new file mode 100644 index 00000000..c6ccfe13 --- /dev/null +++ b/compiler/crates/hy-wasm/src/lib.rs @@ -0,0 +1,113 @@ +use std::alloc::{alloc, dealloc, Layout}; +use std::ptr; +use std::slice; +use std::str; + +#[no_mangle] +pub extern "C" fn hy_wasm_alloc(len: usize) -> *mut u8 { + if len == 0 { + return ptr::NonNull::::dangling().as_ptr(); + } + + let layout = match Layout::array::(len) { + Ok(layout) => layout, + Err(_) => return ptr::null_mut(), + }; + unsafe { alloc(layout) } +} + +#[no_mangle] +pub unsafe extern "C" fn hy_wasm_free(ptr: *mut u8, len: usize) { + if ptr.is_null() || len == 0 { + return; + } + + if let Ok(layout) = Layout::array::(len) { + dealloc(ptr, layout); + } +} + +unsafe fn pack_check_output(diags: &[hy_diag::Diagnostic], summary: &hy_diag::CheckSummary) -> u64 { + let json = hy_diag::check_output_json(diags, summary); + let bytes = json.as_bytes(); + let len = bytes.len(); + let ptr = hy_wasm_alloc(len); + + if ptr.is_null() { + return 0; + } + + ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, len); + ((ptr as u32 as u64) << 32) | (len as u64) +} + +/// Takes source code, checks it, returns a pointer to a JSON envelope in memory. +/// Returns (ptr, len) packed as u64. +#[no_mangle] +pub unsafe extern "C" fn hy_wasm_check(src_ptr: *const u8, src_len: usize) -> u64 { + let slice = if src_len == 0 { + &[] + } else { + if src_ptr.is_null() { + return 0; + } + slice::from_raw_parts(src_ptr, src_len) + }; + + let src = match str::from_utf8(slice) { + Ok(s) => s, + Err(_) => { + let diag = hy_diag::Diagnostic::error( + "HYB-E0002", + "input.hy", + hy_diag::Span::point(hy_diag::Pos::new(1, 1)), + "source is not valid UTF-8", + ); + let diags = vec![diag]; + let summary = hy_diag::CheckSummary { + file: "input.hy".into(), + verdict: "reject".into(), + errors: 1, + warnings: 0, + managed: None, + }; + return pack_check_output(&diags, &summary); + } + }; + + let (diags, summary) = match hy_syntax::parse_file("input.hy", src) { + Ok(ast) => { + let res = hy_check::check_file(&ast); + let errors = res + .diagnostics + .iter() + .filter(|d| d.severity == hy_diag::Severity::Error) + .count(); + let warnings = res + .diagnostics + .iter() + .filter(|d| d.severity == hy_diag::Severity::Warning) + .count(); + let summary = hy_diag::CheckSummary { + file: "input.hy".into(), + verdict: res.verdict.as_str().to_string(), + errors, + warnings, + managed: None, + }; + (res.diagnostics, summary) + } + Err(diags) => { + let summary = hy_diag::CheckSummary { + file: "input.hy".into(), + verdict: "reject".into(), + errors: diags.len(), + warnings: 0, + managed: None, + }; + (diags, summary) + } + }; + + pack_check_output(&diags, &summary) +} diff --git a/spikes/w012-analyzer/analyzer.go b/spikes/w012-analyzer/analyzer.go new file mode 100644 index 00000000..8675efc0 --- /dev/null +++ b/spikes/w012-analyzer/analyzer.go @@ -0,0 +1,393 @@ +package analyzer + +import ( + "context" + "fmt" + "go/token" + "os" + "path/filepath" + "slices" + "sort" + "sync" + "time" + "unicode/utf8" + + "github.com/tetratelabs/wazero" + "golang.org/x/tools/go/analysis" +) + +type Pos struct { + Line int `json:"line"` + Col int `json:"col"` +} + +type Span struct { + Start Pos `json:"start"` + End Pos `json:"end"` +} + +type Diagnostic struct { + Code string `json:"code"` + Severity string `json:"severity"` + Message string `json:"message"` + Span Span `json:"span"` + Tier0Rule string `json:"tier0_rule,omitempty"` +} + +type Summary struct { + File string `json:"file"` + Verdict string `json:"verdict"` + Errors int `json:"errors"` + Warnings int `json:"warnings"` +} + +type CheckOutput struct { + Diagnostics []Diagnostic `json:"diagnostics"` + Summary Summary `json:"summary"` + Version int `json:"version"` +} + +type Engine struct { + ctx context.Context + runtime wazero.Runtime + compiled wazero.CompiledModule + cache wazero.CompilationCache // optional, held until Close +} + +var ( + defaultWasmPath = wasmArtifactPath() + globalEngine *Engine + globalEngineOnce sync.Once + globalEngineErr error +) + +func wasmArtifactPath() string { + if path := os.Getenv("HYBRID_WASM_PATH"); path != "" { + return path + } + return filepath.Join("..", "..", "compiler", "target", "wasm32-unknown-unknown", "release", "hy_wasm.wasm") +} + +// newEngine loads the same checker as GetEngine with an optional filesystem +// compilation cache. A cache is not an analysis-result cache: every request +// still executes the checker and fail-closed diagnostic validation. +func newEngine(wasmPath, cacheDir string) (*Engine, error) { + wasmBytes, err := os.ReadFile(wasmPath) + if err != nil { + return nil, fmt.Errorf("failed to read wasm binary: %w", err) + } + ctx := context.Background() + config := wazero.NewRuntimeConfig().WithCloseOnContextDone(true) + var cache wazero.CompilationCache + if cacheDir != "" { + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return nil, fmt.Errorf("prepare WASM compilation cache: %w", err) + } + cache, err = wazero.NewCompilationCacheWithDir(cacheDir) + if err != nil { + return nil, fmt.Errorf("open WASM compilation cache: %w", err) + } + config = config.WithCompilationCache(cache) + } + r := wazero.NewRuntimeWithConfig(ctx, config) + compiled, err := r.CompileModule(ctx, wasmBytes) + if err != nil { + _ = r.Close(ctx) + if cache != nil { + _ = cache.Close(ctx) + } + return nil, fmt.Errorf("failed to compile wasm module: %w", err) + } + return &Engine{ctx: ctx, runtime: r, compiled: compiled, cache: cache}, nil +} + +func (e *Engine) Close() error { + err := e.runtime.Close(e.ctx) + if e.cache != nil { + cacheErr := e.cache.Close(e.ctx) + if err == nil { + err = cacheErr + } + } + return err +} + +func GetEngine(wasmPath string) (*Engine, error) { + globalEngineOnce.Do(func() { + if wasmPath == "" { + wasmPath = defaultWasmPath + } + // Opt-in: unset preserves the original in-memory-only runtime. The + // cache stores compiled WASM machine code, not checker verdicts. + globalEngine, globalEngineErr = newEngine(wasmPath, os.Getenv("HYBRID_W012_CACHE_DIR")) + }) + return globalEngine, globalEngineErr +} + +// Check bounds a single analysis even when the go/analysis driver does not +// provide a cancellation context. Infrastructure timeout remains fail-closed. +func (e *Engine) Check(src string) (*CheckOutput, error) { + ctx, cancel := context.WithTimeout(e.ctx, 30*time.Second) + defer cancel() + return e.CheckContext(ctx, src) +} + +// CheckContext permits an IDE or other embedding host to cancel one request +// without closing the shared wazero runtime or poisoning later checks. +func (e *Engine) CheckContext(ctx context.Context, src string) (*CheckOutput, error) { + if ctx == nil { + return nil, fmt.Errorf("nil analysis context") + } + engine := &WasmEngine{ctx: e.ctx, runtime: e.runtime, compiled: e.compiled} + res := engine.CheckSource(ctx, src) + if res.Verdict == VerdictAnalysisFailure { + return nil, res.Failure + } + return &CheckOutput{Version: 1, Summary: *res.Summary, Diagnostics: res.Diagnostics}, nil +} + +// PassContextLog records empirical pass inspection for research +type PassContextLog struct { + PkgPath string + FilesCount int + GoFiles []string + OtherFiles []string + IgnoredFiles []string + HasReadFile bool + HasTypesInfo bool + FoundHyFiles []string + PassReadFileTested bool + PassReadFileErr string + PassReadFileLen int +} + +var ( + lastContextLogMu sync.RWMutex + lastContextLog PassContextLog +) + +// Clone the slices as well as the struct: copying only its slice headers would +// expose mutable internal arrays to callers even while protecting assignments +// to lastContextLog with a mutex. +func clonePassContextLog(log PassContextLog) PassContextLog { + log.GoFiles = slices.Clone(log.GoFiles) + log.OtherFiles = slices.Clone(log.OtherFiles) + log.IgnoredFiles = slices.Clone(log.IgnoredFiles) + log.FoundHyFiles = slices.Clone(log.FoundHyFiles) + return log +} + +func setLastContextLog(log PassContextLog) { + lastContextLogMu.Lock() + lastContextLog = clonePassContextLog(log) + lastContextLogMu.Unlock() +} + +// LastContextLogSnapshot returns an independent snapshot. Callers may modify +// its slices without racing with later analyzer runs or other snapshots. +func LastContextLogSnapshot() PassContextLog { + lastContextLogMu.RLock() + defer lastContextLogMu.RUnlock() + return clonePassContextLog(lastContextLog) +} + +var Analyzer = &analysis.Analyzer{ + Name: "hybrid", + Doc: "Spike: inspects Go package and invokes in-process WASM checker on sibling .hy files", + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + // The cache-safe vet wrapper supplies a pre-run digest snapshot. Malformed + // snapshots fail closed before analyzing any package. + snapshot, err := hybridSnapshotFromEnv() + if err != nil { + return nil, fmt.Errorf("read Hybrid input snapshot: %w", err) + } + log := PassContextLog{ + PkgPath: pass.Pkg.Path(), + FilesCount: len(pass.Files), + OtherFiles: pass.OtherFiles, + IgnoredFiles: pass.IgnoredFiles, + HasReadFile: pass.ReadFile != nil, + HasTypesInfo: pass.TypesInfo != nil, + } + + for _, f := range pass.Files { + if pass.Fset != nil { + pos := pass.Fset.File(f.Pos()) + if pos != nil { + log.GoFiles = append(log.GoFiles, pos.Name()) + } + } + } + + // Sibling .hy discovery: + // A. Check pass.OtherFiles + // B. Check directory containing Go files + hyFiles := make(map[string]string) // name -> full path or content + + // Declared files are a trusted input set, not an optional hint: failing to + // read one must fail this analysis, never silently produce an all-clear. + for _, of := range pass.OtherFiles { + if filepath.Ext(of) != ".hy" { + continue + } + log.FoundHyFiles = append(log.FoundHyFiles, "OtherFiles:"+of) + var data []byte + var readErr error + if pass.ReadFile != nil { + data, readErr = pass.ReadFile(of) + } else { + data, readErr = os.ReadFile(of) + } + if readErr != nil { + return nil, fmt.Errorf("read declared .hy file %q: %w", of, readErr) + } + name, err := filepath.Abs(of) + if err != nil { + return nil, fmt.Errorf("resolve declared .hy path %q: %w", of, err) + } + if err := snapshot.VerifyFile(name, data); err != nil { + return nil, fmt.Errorf("verify declared .hy input: %w", err) + } + if previous, exists := hyFiles[name]; exists && previous != string(data) { + return nil, fmt.Errorf("conflicting content for declared .hy file %q", name) + } + hyFiles[name] = string(data) + } + + // Also inspect disk directory of first Go file to test if sibling .hy can be discovered via dir scan + if len(pass.Files) > 0 && pass.Fset != nil { + firstFile := pass.Fset.File(pass.Files[0].Pos()) + if firstFile != nil { + dir := filepath.Dir(firstFile.Name()) + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("discover .hy files: %w", err) + } + for _, e := range entries { + if filepath.Ext(e.Name()) == ".hy" { + fullPath, err := filepath.Abs(filepath.Join(dir, e.Name())) + if err != nil { + return nil, fmt.Errorf("resolve sibling .hy path %q: %w", e.Name(), err) + } + if _, already := hyFiles[fullPath]; !already { + log.FoundHyFiles = append(log.FoundHyFiles, "DirScan:"+e.Name()) + + // .hy files found only by directory scan are NOT valid inputs + // to Pass.ReadFile: strict drivers may reject or panic on + // undeclared paths. For this explicitly on-disk vet adapter, + // read from disk and verify the pre-run Hybrid snapshot. + // Only listed OtherFiles are read through Pass.ReadFile. + data, err := os.ReadFile(fullPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", fullPath, err) + } + if err := snapshot.VerifyFile(fullPath, data); err != nil { + return nil, fmt.Errorf("verify scanned .hy input: %w", err) + } + hyFiles[fullPath] = string(data) + } + } + } + } + } + + setLastContextLog(log) + + // Go-only packages need neither a WASM artifact nor a live runtime. + if len(hyFiles) == 0 { + return nil, nil + } + engine, err := GetEngine("") + if err != nil { + return nil, fmt.Errorf("failed to init wazero engine: %w", err) + } + + // Map iteration order is unstable: report diagnostics in canonical filename + // order so identical source trees yield reproducible go vet output. + filenames := make([]string, 0, len(hyFiles)) + for filename := range hyFiles { + filenames = append(filenames, filename) + } + sort.Strings(filenames) + for _, filename := range filenames { + content := hyFiles[filename] + out, err := engine.Check(content) + if err != nil { + return nil, fmt.Errorf("hybrid check on %s failed: %w", filename, err) + } + + var fileToken *token.File + if pass.Fset != nil { + fileToken = newHybridTokenFile(pass.Fset, filename, content) + } + + // The structured engine preserves nonfatal warnings. go/analysis has + // no severity channel: passing warnings to vet would turn an accepted + // managed program into a failing build. Report only errors here. + for _, d := range out.Diagnostics { + if d.Severity != "error" { + continue + } + var exactPos token.Pos = token.NoPos + if fileToken != nil { + exactPos, err = hybridTokenPos(fileToken, content, d.Span.Start.Line, d.Span.Start.Col) + if err != nil { + return nil, fmt.Errorf("invalid diagnostic position in %s: %w", filename, err) + } + } + + pass.Report(analysis.Diagnostic{ + Pos: exactPos, + Message: fmt.Sprintf("[%s] %s", d.Code, d.Message), + Category: "hybrid", + }) + } + } + + return nil, nil +} + +// Go's line table omits an empty final line, but the Hybrid lexer can report +// EOF at 1:1 in an empty file or after a trailing newline. Reserve one byte +// only in the token table so these positions retain their actual line and +// column. The checker and hybridTokenPos still use the unmodified content. +func newHybridTokenFile(fset *token.FileSet, filename, content string) *token.File { + file := fset.AddFile(filename, -1, len(content)+1) + file.SetLinesForContent(append([]byte(content), 0)) + return file +} + +// hybridTokenPos translates Rust lexer Unicode-scalar columns into Go's +// byte-based token.Pos without allowing a diagnostic to escape its own line. +func hybridTokenPos(file *token.File, content string, line, col int) (token.Pos, error) { + if line < 1 || line > file.LineCount() || col < 1 { + return token.NoPos, fmt.Errorf("invalid position %d:%d", line, col) + } + start := file.Offset(file.LineStart(line)) + end := len(content) + if line < file.LineCount() { + end = file.Offset(file.LineStart(line + 1)) + } + if end > start && content[end-1] == '\n' { + end-- + } + if end > start && content[end-1] == '\r' { + end-- + } + runeCol := 1 + for offset := start; offset < end; { + if runeCol == col { + return file.Pos(offset), nil + } + _, size := utf8.DecodeRuneInString(content[offset:end]) + offset += size + runeCol++ + } + if runeCol == col { + return file.Pos(end), nil + } + return token.NoPos, fmt.Errorf("column %d beyond line %d (maximum %d)", col, line, runeCol) +} diff --git a/spikes/w012-analyzer/analyzer_test.go b/spikes/w012-analyzer/analyzer_test.go new file mode 100644 index 00000000..4facbab6 --- /dev/null +++ b/spikes/w012-analyzer/analyzer_test.go @@ -0,0 +1,381 @@ +package analyzer + +import ( + "context" + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/packages" +) + +func TestAnalyzerEmpiricalPassContext(t *testing.T) { + testdata, err := filepath.Abs("testdata/src/samplepkg") + if err != nil { + t.Fatal(err) + } + + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | + packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | + packages.NeedSyntax | packages.NeedTypesInfo, + Dir: testdata, + } + + pkgs, err := packages.Load(cfg, ".") + if err != nil { + t.Fatalf("failed to load packages: %v", err) + } + if len(pkgs) == 0 { + t.Fatalf("no packages loaded") + } + + pkg := pkgs[0] + var reportedDiags []analysis.Diagnostic + + pass := &analysis.Pass{ + Analyzer: Analyzer, + Fset: pkg.Fset, + Files: pkg.Syntax, + OtherFiles: pkg.OtherFiles, + IgnoredFiles: pkg.IgnoredFiles, + Pkg: pkg.Types, + TypesInfo: pkg.TypesInfo, + TypesSizes: pkg.TypesSizes, + ReadFile: os.ReadFile, + Report: func(d analysis.Diagnostic) { + reportedDiags = append(reportedDiags, d) + }, + } + + _, err = run(pass) + if err != nil { + t.Fatalf("analyzer run failed: %v", err) + } + + // 1. Empirical Question A: Do .hy files enter pass.Files? + // Expectation: NO. Only Go AST files enter pass.Files. + if len(pass.Files) != 1 { + t.Errorf("Expected exactly 1 Go file in pass.Files, got %d", len(pass.Files)) + } + t.Logf("A. pass.Files count: %d (Only .go files parsed into Go AST)", len(pass.Files)) + + // 2. Empirical Question B: Can .hy be discovered via OtherFiles or IgnoredFiles? + // Finding: In stock Go packages, OtherFiles and IgnoredFiles do NOT include .hy files. + t.Logf("B1. pass.OtherFiles: %v (len=%d)", pass.OtherFiles, len(pass.OtherFiles)) + t.Logf("B2. pass.IgnoredFiles: %v (len=%d)", pass.IgnoredFiles, len(pass.IgnoredFiles)) + if len(pass.OtherFiles) > 0 { + t.Errorf("Expected empty OtherFiles for non-Go files, got %v", pass.OtherFiles) + } + + // 2b. Empirical Question B3: Can pass.ReadFile read .hy once path is known? + badHyPath := filepath.Join(testdata, "bad.hy") + rfData, rfErr := pass.ReadFile(badHyPath) + t.Logf("B3. pass.ReadFile(%q): err=%v, bytes=%d", badHyPath, rfErr, len(rfData)) + if rfErr != nil || len(rfData) == 0 { + t.Fatalf("pass.ReadFile failed on .hy file: %v", rfErr) + } + + // 3. Sibling discovery requires directory scan anchor: + contextLog := LastContextLogSnapshot() + t.Logf("C. Discovered .hy files via DirScan: %v", contextLog.FoundHyFiles) + if len(contextLog.FoundHyFiles) == 0 { + t.Errorf("Expected to discover sibling .hy file via directory scan") + } + + // 4. Empirical Question D: Available Go context: + t.Logf("D. PkgPath: %s, HasTypesInfo: %v", pass.Pkg.Path(), pass.TypesInfo != nil) + + // 5. Empirical Question E: Diagnostic reporting and source position contract: + t.Logf("E. Reported %d diagnostic(s) from sibling .hy analysis:", len(reportedDiags)) + for _, d := range reportedDiags { + position := pass.Fset.Position(d.Pos) + t.Logf(" - [%s] Pos=%d (%s): %s", d.Category, d.Pos, position, d.Message) + if position.Line == 0 || position.Column == 0 { + t.Errorf("Expected valid Line and Column from exact position, got %s", position) + } + } + if len(reportedDiags) == 0 { + t.Errorf("Expected diagnostics from invalid .hy, got 0") + } +} + +func TestContextLogConcurrentAccess(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(2) + go func(i int) { + defer wg.Done() + setLastContextLog(PassContextLog{PkgPath: "pkg", FilesCount: i}) + }(i) + go func() { + defer wg.Done() + _ = LastContextLogSnapshot() + }() + } + wg.Wait() +} + +func TestContextLogSnapshotDoesNotAliasCallerOrStoredSlices(t *testing.T) { + original := PassContextLog{ + PkgPath: "example.org/test", + GoFiles: []string{"main.go"}, + OtherFiles: []string{"other.hy"}, + IgnoredFiles: []string{"ignored.hy"}, + FoundHyFiles: []string{"DirScan:other.hy"}, + } + setLastContextLog(original) + + // The setter must not retain the input slices (which are owned by a pass). + original.GoFiles[0] = "caller-mutated.go" + original.OtherFiles[0] = "caller-mutated.hy" + original.IgnoredFiles[0] = "caller-mutated.hy" + original.FoundHyFiles[0] = "caller-mutated.hy" + + first := LastContextLogSnapshot() + if first.GoFiles[0] != "main.go" || first.OtherFiles[0] != "other.hy" || + first.IgnoredFiles[0] != "ignored.hy" || first.FoundHyFiles[0] != "DirScan:other.hy" { + t.Fatalf("input slices leaked into saved observation: %+v", first) + } + + // The snapshot must not expose shared backing arrays, even to other readers. + first.GoFiles[0] = "snapshot-mutated.go" + first.OtherFiles[0] = "snapshot-mutated.hy" + first.IgnoredFiles[0] = "snapshot-mutated.hy" + first.FoundHyFiles[0] = "snapshot-mutated.hy" + second := LastContextLogSnapshot() + if second.GoFiles[0] != "main.go" || second.OtherFiles[0] != "other.hy" || + second.IgnoredFiles[0] != "ignored.hy" || second.FoundHyFiles[0] != "DirScan:other.hy" { + t.Fatalf("snapshot mutated shared observation: %+v", second) + } +} + +func TestEngineCheckContextIsCancellableAndDoesNotPoisonRuntime(t *testing.T) { + engine, err := GetEngine("") + if err != nil { + t.Fatal(err) + } + source, err := os.ReadFile("../../tests/checker-corpus/02-shared-map-managed.hy") + if err != nil { + t.Fatal(err) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := engine.CheckContext(cancelled, string(source)); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled request must fail closed: %v", err) + } + if _, err := engine.CheckContext(nil, string(source)); err == nil { + t.Fatal("nil request context should return an error, not panic") + } + + // A completed request must still be possible using the same compiled runtime. + out, err := engine.CheckContext(context.Background(), string(source)) + if err != nil { + t.Fatalf("cancelled request poisoned shared runtime: %v", err) + } + if out.Summary.Verdict != "accept" || out.Summary.Warnings < 1 { + t.Fatalf("unexpected result after cancelling previous check: %+v", out) + } +} + +func TestDeclaredHybridReadFailuresFailClosed(t *testing.T) { + unreadable := filepath.Join(t.TempDir(), "unreadable.hy") + for _, tc := range []struct { + name string + readFile func(string) ([]byte, error) + }{ + {"driver_read_denied", func(string) ([]byte, error) { + return nil, errors.New("virtual file unavailable") + }}, + {"no_driver_reader", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + pass := &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + OtherFiles: []string{unreadable}, + ReadFile: tc.readFile, + Report: func(analysis.Diagnostic) { + t.Error("no diagnostics should be emitted when a declared input cannot be read") + }, + } + _, err := run(pass) + if err == nil || !strings.Contains(err.Error(), "read declared .hy file") { + t.Fatalf("unreadable declared input must fail closed, got %v", err) + } + }) + } +} + +func TestDeclaredAndScannedHybridPathsAreCanonicalAndReportedOnce(t *testing.T) { + dir := t.TempDir() + hyPath := filepath.Join(dir, "bad.hy") + src := "package sample\nfn check() { let x = 7; let y = move x; let z = x }\n" + if err := os.WriteFile(hyPath, []byte(src), 0600); err != nil { + t.Fatal(err) + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + relative, err := filepath.Rel(cwd, hyPath) + if err != nil { + t.Fatal(err) + } + engine, err := GetEngine("") + if err != nil { + t.Fatal(err) + } + out, err := engine.Check(src) + if err != nil { + t.Fatal(err) + } + if len(out.Diagnostics) == 0 { + t.Fatal("regression fixture must emit a diagnostic") + } + + for _, tc := range []struct { + name string + readFile func(string) ([]byte, error) + }{ + {"driver_file_reader", os.ReadFile}, + {"filesystem_fallback", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + fset := token.NewFileSet() + goFile, err := parser.ParseFile(fset, filepath.Join(dir, "anchor.go"), "package sample\n", 0) + if err != nil { + t.Fatal(err) + } + var reported []analysis.Diagnostic + pass := &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + Fset: fset, + Files: []*ast.File{goFile}, + OtherFiles: []string{relative, hyPath}, + ReadFile: tc.readFile, + Report: func(d analysis.Diagnostic) { reported = append(reported, d) }, + } + if _, err := run(pass); err != nil { + t.Fatal(err) + } + if len(reported) != len(out.Diagnostics) { + t.Fatalf("expected %d diagnostics for one canonical file, got %d", len(out.Diagnostics), len(reported)) + } + for _, d := range reported { + if pos := fset.Position(d.Pos); pos.Filename != hyPath { + t.Fatalf("diagnostic must use canonical source path: %s", pos) + } + } + }) + } +} + +func TestSiblingDiagnosticsHaveStableFileOrder(t *testing.T) { + dir := t.TempDir() + src := "package sample\nfn check() { let x = 7; let y = move x; let z = x }\n" + for _, name := range []string{"b.hy", "a.hy"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0600); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 4; i++ { + fset := token.NewFileSet() + goFile, err := parser.ParseFile(fset, filepath.Join(dir, "anchor.go"), "package sample\n", 0) + if err != nil { + t.Fatal(err) + } + var filenames []string + pass := &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + Fset: fset, + Files: []*ast.File{goFile}, + Report: func(d analysis.Diagnostic) { + filenames = append(filenames, filepath.Base(fset.Position(d.Pos).Filename)) + }, + } + if _, err := run(pass); err != nil { + t.Fatal(err) + } + if len(filenames) < 2 { + t.Fatalf("want both source files to produce diagnostics, got %v", filenames) + } + firstB := -1 + for index, name := range filenames { + if name == "b.hy" { + firstB = index + break + } + } + if firstB < 1 { + t.Fatalf("diagnostics not sorted by canonical source name: %v", filenames) + } + for _, name := range filenames[firstB:] { + if name != "b.hy" { + t.Fatalf("a.hy emitted after b.hy: %v", filenames) + } + } + } +} + +func TestAnalyzerDirectInvocation(t *testing.T) { + engine, err := GetEngine("") + if err != nil { + t.Fatalf("failed to get engine: %v", err) + } + + badHy := `// EXPECTED-VERDICT: reject HYB-E3003 +package samplepkg + +fn bad_use_after_move() { + let x = 42 + let y = move x + let z = x +} +` + out, err := engine.Check(badHy) + if err != nil { + t.Fatalf("WASM engine check failed: %v", err) + } + + if out.Summary.Verdict != "reject" { + t.Fatalf("Expected verdict reject, got %s", out.Summary.Verdict) + } + + var diags []analysis.Diagnostic + pass := &analysis.Pass{ + Fset: token.NewFileSet(), + Report: func(d analysis.Diagnostic) { + diags = append(diags, d) + }, + } + + // Map to analysis.Diagnostic + for _, d := range out.Diagnostics { + pass.Report(analysis.Diagnostic{ + Pos: token.NoPos, + Message: d.Message, + Category: d.Code, + }) + } + + if len(diags) == 0 { + t.Fatalf("Expected reported diagnostics, got 0") + } + + t.Logf("Successfully mapped %d diagnostics to analysis.Pass:", len(diags)) + for _, d := range diags { + t.Logf(" - [%s] %s", d.Category, d.Message) + } +} diff --git a/spikes/w012-analyzer/bench/bench_wazero.go b/spikes/w012-analyzer/bench/bench_wazero.go new file mode 100644 index 00000000..f32fa9d6 --- /dev/null +++ b/spikes/w012-analyzer/bench/bench_wazero.go @@ -0,0 +1,519 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "time" + + analyzer "github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer" + "github.com/tetratelabs/wazero" +) + +type LatencyStats struct { + Min float64 + P50 float64 + P90 float64 + P99 float64 + Mean float64 +} + +func calcStats(samples []float64) LatencyStats { + if len(samples) == 0 { + return LatencyStats{} + } + sorted := make([]float64, len(samples)) + copy(sorted, samples) + sort.Float64s(sorted) + + var sum float64 + for _, v := range sorted { + sum += v + } + + p := func(percent float64) float64 { + idx := int(math.Ceil(percent*float64(len(sorted)))) - 1 + if idx < 0 { + idx = 0 + } + if idx >= len(sorted) { + idx = len(sorted) - 1 + } + return sorted[idx] + } + + return LatencyStats{ + Min: sorted[0], + P50: p(0.50), + P90: p(0.90), + P99: p(0.99), + Mean: sum / float64(len(sorted)), + } +} + +// Paths default to the repository containing the current working directory. +// Environment overrides match the parity and analyzer test harnesses. +func repoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "compiler", "Cargo.toml")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", errors.New("repository root not found; run inside the checkout or set -root") + } + dir = parent + } +} + +// Exit 1 is expected only when the CLI also returns a valid rejection envelope. +// Missing binaries, signals, unexpected exit codes, and malformed output abort +// the benchmark instead of being counted as artificially fast checks. +func nativeReject(ctx context.Context, binary, fixture string) error { + output, err := exec.CommandContext(ctx, binary, "check", fixture, "--json").Output() + return validateNativeRejection(output, err) +} + +func validateNativeRejection(output []byte, runErr error) error { + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) || exitErr.ExitCode() != 1 { + return fmt.Errorf("native CLI must reject with exit 1; got %v", runErr) + } + out, err := analyzer.DecodeCheckOutput(output) + if err != nil { + return fmt.Errorf("native CLI returned an invalid diagnostic envelope: %w", err) + } + if out.Summary.Verdict != "reject" { + return fmt.Errorf("native CLI exit 1 disagrees with verdict %q", out.Summary.Verdict) + } + return nil +} + +// A slower warm path has no finite amortization point; do not print a negative +// or infinite check count. This is an estimate for compilation alone. +func compileBreakEven(overhead, savings float64) string { + if savings <= 0 { + return "no finite break-even (warm path is not faster)" + } + n := overhead / savings + return fmt.Sprintf("%.1f checks (ceil: %.0f checks)", n, math.Ceil(n)) +} + +func main() { + ctx := context.Background() + defaultRoot, _ := repoRoot() + root := flag.String("root", defaultRoot, "repository root") + wasmFlag := flag.String("wasm", os.Getenv("HYBRID_WASM_PATH"), "WASM artifact (or HYBRID_WASM_PATH)") + nativeFlag := flag.String("native", os.Getenv("HYBRID_NATIVE_PATH"), "native CLI (or HYBRID_NATIVE_PATH)") + fixtureFlag := flag.String("fixture", "", "reject fixture") + iterationsFlag := flag.Int("iterations", 100, "samples per case") + outputFlag := flag.String("output", "", "write raw samples and environment metadata as JSON") + flag.Parse() + if *root == "" { + panic("repository root not found; use -root") + } + if *iterationsFlag < 1 { + panic("iterations must be positive") + } + wasmPath := *wasmFlag + if wasmPath == "" { + wasmPath = filepath.Join(*root, "compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm") + } + hybPath := *nativeFlag + if hybPath == "" { + hybPath = filepath.Join(*root, "compiler/target/release/hyb") + } + testHyPath := *fixtureFlag + if testHyPath == "" { + testHyPath = filepath.Join(*root, "tests/checker-corpus/08-a3-map-slot-invalidated.hy") + } + wasmBytes, err := os.ReadFile(wasmPath) + if err != nil { + panic(err) + } + testHyBytes, err := os.ReadFile(testHyPath) + if err != nil { + panic(err) + } + testHyStr := string(testHyBytes) + iterations := *iterationsFlag + fmt.Printf("Go=%s OS=%s arch=%s GOMAXPROCS=%d\nWASM=%s\nNative=%s\nFixture=%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH, runtime.GOMAXPROCS(0), wasmPath, hybPath, testHyPath) + + fmt.Println("================================================================================") + fmt.Printf("W-012 HARDENING BENCHMARK: Wazero vs Native Subprocess (%d iterations/case)\n", iterations) + fmt.Println("================================================================================") + + // 0. Native Subprocess Benchmark + var subprocSamples []float64 + for i := 0; i < iterations; i++ { + t0 := time.Now() + if err := nativeReject(ctx, hybPath, testHyPath); err != nil { + panic(err) + } + subprocSamples = append(subprocSamples, float64(time.Since(t0).Microseconds())/1000.0) + } + subprocStats := calcStats(subprocSamples) + fmt.Printf("[Baseline] Native CLI Subprocess: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", + subprocStats.Min, subprocStats.P50, subprocStats.P90, subprocStats.P99, subprocStats.Mean) + + // CASE A: Same process / fresh runtime / no compilation cache (not process startup) + var caseACompile, caseAInstantiate, caseACheck, caseATotal []float64 + for i := 0; i < iterations; i++ { + t0 := time.Now() + r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) + + tCompileStart := time.Now() + compiled, err := r.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + tCompile := float64(time.Since(tCompileStart).Microseconds()) / 1000.0 + + tInstStart := time.Now() + mod, err := r.InstantiateModule(ctx, compiled, wazero.NewModuleConfig()) + if err != nil { + panic(err) + } + tInst := float64(time.Since(tInstStart).Microseconds()) / 1000.0 + + abi, err := analyzer.NewWazeroInstanceABI(mod) + if err != nil { + panic(err) + } + + tCheckStart := time.Now() + res := analyzer.CheckSourceWithABI(ctx, abi, testHyStr) + if res.Verdict != analyzer.VerdictReject { + panic("expected reject") + } + tCheck := float64(time.Since(tCheckStart).Microseconds()) / 1000.0 + + _ = mod.Close(ctx) + _ = r.Close(ctx) + + tTotal := float64(time.Since(t0).Microseconds()) / 1000.0 + + caseACompile = append(caseACompile, tCompile) + caseAInstantiate = append(caseAInstantiate, tInst) + caseACheck = append(caseACheck, tCheck) + caseATotal = append(caseATotal, tTotal) + } + sACompile := calcStats(caseACompile) + sAInst := calcStats(caseAInstantiate) + sACheck := calcStats(caseACheck) + sATotal := calcStats(caseATotal) + + fmt.Printf("[CASE A] Fresh Runtime (No Cache): total_mean=%.3fms (compile=%.3fms, inst=%.3fms, check=%.3fms)\n", + sATotal.Mean, sACompile.Mean, sAInst.Mean, sACheck.Mean) + fmt.Printf(" Compile Stats: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", + sACompile.Min, sACompile.P50, sACompile.P90, sACompile.P99, sACompile.Mean) + + // CASE B: Same process / same wazero.Runtime / compiled module reused + var caseBInst, caseBCheck, caseBTotal []float64 + rB := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) + compiledB, err := rB.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + + for i := 0; i < iterations; i++ { + t0 := time.Now() + mod, err := rB.InstantiateModule(ctx, compiledB, wazero.NewModuleConfig()) + if err != nil { + panic(err) + } + tInst := float64(time.Since(t0).Microseconds()) / 1000.0 + + abi, err := analyzer.NewWazeroInstanceABI(mod) + if err != nil { + panic(err) + } + + tCheckStart := time.Now() + res := analyzer.CheckSourceWithABI(ctx, abi, testHyStr) + if res.Verdict != analyzer.VerdictReject { + panic("expected reject") + } + tCheck := float64(time.Since(tCheckStart).Microseconds()) / 1000.0 + + _ = mod.Close(ctx) + tTotal := float64(time.Since(t0).Microseconds()) / 1000.0 + + caseBInst = append(caseBInst, tInst) + caseBCheck = append(caseBCheck, tCheck) + caseBTotal = append(caseBTotal, tTotal) + } + _ = rB.Close(ctx) + sBInst := calcStats(caseBInst) + sBCheck := calcStats(caseBCheck) + sBTotal := calcStats(caseBTotal) + + fmt.Printf("[CASE B] Reused CompiledModule: total_mean=%.3fms (inst=%.3fms, check=%.3fms)\n", + sBTotal.Mean, sBInst.Mean, sBCheck.Mean) + fmt.Printf(" Total Stats: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", + sBTotal.Min, sBTotal.P50, sBTotal.P90, sBTotal.P99, sBTotal.Mean) + + // CASE C: New runtime / shared in-memory compilation cache + memCache := wazero.NewCompilationCache() + // Warm cache once + rWarm := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithCompilationCache(memCache)) + cWarm, err := rWarm.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + _ = cWarm.Close(ctx) + _ = rWarm.Close(ctx) + + var caseCCompile, caseCInst, caseCCheck, caseCTotal []float64 + for i := 0; i < iterations; i++ { + t0 := time.Now() + rC := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithCompilationCache(memCache)) + + tCompStart := time.Now() + compiledC, err := rC.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + tComp := float64(time.Since(tCompStart).Microseconds()) / 1000.0 + + tInstStart := time.Now() + mod, err := rC.InstantiateModule(ctx, compiledC, wazero.NewModuleConfig()) + if err != nil { + panic(err) + } + tInst := float64(time.Since(tInstStart).Microseconds()) / 1000.0 + + abi, err := analyzer.NewWazeroInstanceABI(mod) + if err != nil { + panic(err) + } + + tCheckStart := time.Now() + res := analyzer.CheckSourceWithABI(ctx, abi, testHyStr) + if res.Verdict != analyzer.VerdictReject { + panic("expected reject") + } + tCheck := float64(time.Since(tCheckStart).Microseconds()) / 1000.0 + + _ = mod.Close(ctx) + _ = rC.Close(ctx) + tTotal := float64(time.Since(t0).Microseconds()) / 1000.0 + + caseCCompile = append(caseCCompile, tComp) + caseCInst = append(caseCInst, tInst) + caseCCheck = append(caseCCheck, tCheck) + caseCTotal = append(caseCTotal, tTotal) + } + _ = memCache.Close(ctx) + sCComp := calcStats(caseCCompile) + sCInst := calcStats(caseCInst) + sCCheck := calcStats(caseCCheck) + sCTotal := calcStats(caseCTotal) + + fmt.Printf("[CASE C] New Runtime (In-Memory Cache): total_mean=%.3fms (compile=%.3fms, inst=%.3fms, check=%.3fms)\n", + sCTotal.Mean, sCComp.Mean, sCInst.Mean, sCCheck.Mean) + fmt.Printf(" Compile Stats: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", + sCComp.Min, sCComp.P50, sCComp.P90, sCComp.P99, sCComp.Mean) + + // CASE D: Filesystem compilation cache + var caseDCompile, caseDInst, caseDCheck, caseDTotal []float64 + cacheDir, err := os.MkdirTemp("", "w012-wazero-cache-*") + if err != nil { + panic(err) + } + defer os.RemoveAll(cacheDir) + + fsCache, err := wazero.NewCompilationCacheWithDir(cacheDir) + if err != nil { + fmt.Printf("[CASE D] Filesystem cache NOT TESTED: %v\n", err) + } else { + // Warm fs cache once + rWarmFS := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithCompilationCache(fsCache)) + cWarmFS, err := rWarmFS.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + _ = cWarmFS.Close(ctx) + _ = rWarmFS.Close(ctx) + _ = fsCache.Close(ctx) + + for i := 0; i < iterations; i++ { + t0 := time.Now() + // Open a new cache handle in the same process; this is not process startup. + fsHandle, err := wazero.NewCompilationCacheWithDir(cacheDir) + if err != nil { + panic(err) + } + + rD := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithCompilationCache(fsHandle)) + + tCompStart := time.Now() + compiledD, err := rD.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + tComp := float64(time.Since(tCompStart).Microseconds()) / 1000.0 + + tInstStart := time.Now() + mod, err := rD.InstantiateModule(ctx, compiledD, wazero.NewModuleConfig()) + if err != nil { + panic(err) + } + tInst := float64(time.Since(tInstStart).Microseconds()) / 1000.0 + + abi, err := analyzer.NewWazeroInstanceABI(mod) + if err != nil { + panic(err) + } + + tCheckStart := time.Now() + res := analyzer.CheckSourceWithABI(ctx, abi, testHyStr) + if res.Verdict != analyzer.VerdictReject { + panic("expected reject") + } + tCheck := float64(time.Since(tCheckStart).Microseconds()) / 1000.0 + + _ = mod.Close(ctx) + _ = rD.Close(ctx) + _ = fsHandle.Close(ctx) + tTotal := float64(time.Since(t0).Microseconds()) / 1000.0 + + caseDCompile = append(caseDCompile, tComp) + caseDInst = append(caseDInst, tInst) + caseDCheck = append(caseDCheck, tCheck) + caseDTotal = append(caseDTotal, tTotal) + } + sDComp := calcStats(caseDCompile) + sDInst := calcStats(caseDInst) + sDCheck := calcStats(caseDCheck) + sDTotal := calcStats(caseDTotal) + + fmt.Printf("[CASE D] New Runtime (Filesystem Cache): total_mean=%.3fms (compile=%.3fms, inst=%.3fms, check=%.3fms)\n", + sDTotal.Mean, sDComp.Mean, sDInst.Mean, sDCheck.Mean) + fmt.Printf(" Compile Stats: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", + sDComp.Min, sDComp.P50, sDComp.P90, sDComp.P99, sDComp.Mean) + } + + // CASE E: Already-compiled and instantiated module / repeated checks (warm check latency) + rE := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) + compiledE, err := rE.CompileModule(ctx, wasmBytes) + if err != nil { + panic(err) + } + modE, err := rE.InstantiateModule(ctx, compiledE, wazero.NewModuleConfig()) + if err != nil { + panic(err) + } + abiE, err := analyzer.NewWazeroInstanceABI(modE) + if err != nil { + panic(err) + } + + var caseEWarm []float64 + for i := 0; i < iterations; i++ { + t0 := time.Now() + res := analyzer.CheckSourceWithABI(ctx, abiE, testHyStr) + if res.Verdict != analyzer.VerdictReject { + panic("expected reject") + } + caseEWarm = append(caseEWarm, float64(time.Since(t0).Microseconds())/1000.0) + } + _ = modE.Close(ctx) + _ = rE.Close(ctx) + sEWarm := calcStats(caseEWarm) + fmt.Printf("[CASE E] Validated Reused Instance: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", sEWarm.Min, sEWarm.P50, sEWarm.P90, sEWarm.P99, sEWarm.Mean) + + // CASE F: Actual public Engine.Check used by the analysis.Analyzer. + // Compile once outside the timed loop; unlike case E, each call creates and + // closes a fresh instance through the production wrapper. + engine, err := analyzer.GetEngine(wasmPath) + if err != nil { + panic(err) + } + var caseFEngine []float64 + for i := 0; i < iterations; i++ { + t0 := time.Now() + out, err := engine.Check(testHyStr) + if err != nil { + panic(err) + } + if out.Summary.Verdict != "reject" { + panic("expected reject") + } + caseFEngine = append(caseFEngine, float64(time.Since(t0).Microseconds())/1000.0) + } + sFEngine := calcStats(caseFEngine) + fmt.Printf("[CASE F] Public Engine.Check: min=%.3fms, p50=%.3fms, p90=%.3fms, p99=%.3fms, mean=%.3fms\n", sFEngine.Min, sFEngine.P50, sFEngine.P90, sFEngine.P99, sFEngine.Mean) + fmt.Printf(" Native/Public Engine.Check p50 ratio: %.2fx (below 1 means native is faster)\n", subprocStats.P50/sFEngine.P50) + + // Break-Even Calculations: + // Primary: p50-based comparison (robust to scheduler outliers) + // Secondary: mean-based comparison (outlier-sensitive) + + savingsP50 := subprocStats.P50 - sEWarm.P50 + + savingsMean := subprocStats.Mean - sEWarm.Mean + + speedupP50 := subprocStats.P50 / sEWarm.P50 + speedupMean := subprocStats.Mean / sEWarm.Mean + + fmt.Println("================================================================================") + fmt.Printf("COMPILE-ONLY BREAK-EVEN ESTIMATES (%d ITERATIONS; excludes instantiation/startup):\n", iterations) + fmt.Printf(" [Primary: p50-based Metrics]\n") + fmt.Printf(" • Native CLI Subprocess p50: %.3f ms\n", subprocStats.P50) + fmt.Printf(" • Validated Reused-Instance WASM p50: %.3f ms\n", sEWarm.P50) + fmt.Printf(" • Representative Steady-State Speedup (p50): %.1fx\n", speedupP50) + fmt.Printf(" • Latency Advantage per Check (delta_T p50): %.3f ms\n", savingsP50) + fmt.Printf(" • Cold Compile Overhead (T_compile_cold p50):%.3f ms\n", sACompile.P50) + fmt.Printf(" -> Break-Even (Cold / No Cache, p50): %s\n", compileBreakEven(sACompile.P50, savingsP50)) + + if len(caseDCompile) > 0 { + sDComp := calcStats(caseDCompile) + fmt.Printf(" • FS Cached Compile (T_compile_fs p50): %.3f ms\n", sDComp.P50) + fmt.Printf(" -> Break-Even (Filesystem Cache, p50): %s\n", compileBreakEven(sDComp.P50, savingsP50)) + } + + fmt.Println("\n [Secondary: Outlier-Sensitive Mean Metrics]") + fmt.Printf(" • Native CLI Subprocess Mean: %.3f ms\n", subprocStats.Mean) + fmt.Printf(" • Validated Reused-Instance WASM Mean: %.3f ms\n", sEWarm.Mean) + fmt.Printf(" • Ratio of Means (Mean Speedup): %.1fx\n", speedupMean) + fmt.Printf(" -> Break-Even (Cold / No Cache, Mean): %s\n", compileBreakEven(sACompile.Mean, savingsMean)) + if len(caseDCompile) > 0 { + sDComp := calcStats(caseDCompile) + fmt.Printf(" -> Break-Even (Filesystem Cache, Mean): %s\n", compileBreakEven(sDComp.Mean, savingsMean)) + } + if *outputFlag != "" { + report := map[string]any{ + "go": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH, "gomaxprocs": runtime.GOMAXPROCS(0), + "wasm_path": wasmPath, "native_path": hybPath, "fixture": testHyPath, "iterations": iterations, "unit": "milliseconds", + "native": subprocSamples, + "fresh_runtime": map[string]any{"compile": caseACompile, "instantiate": caseAInstantiate, "check": caseACheck, "total": caseATotal}, + "reused_compiled_module": map[string]any{"instantiate": caseBInst, "check": caseBCheck, "total": caseBTotal}, + "memory_cache": map[string]any{"compile": caseCCompile, "instantiate": caseCInst, "check": caseCCheck, "total": caseCTotal}, + "filesystem_cache_same_process": map[string]any{"compile": caseDCompile, "instantiate": caseDInst, "check": caseDCheck, "total": caseDTotal}, + "public_engine_check": caseFEngine, + "reused_instance": caseEWarm, + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + panic(err) + } + if err := os.WriteFile(*outputFlag, append(data, '\n'), 0600); err != nil { + panic(err) + } + } + + fmt.Println("================================================================================") +} diff --git a/spikes/w012-analyzer/bench/bench_wazero_test.go b/spikes/w012-analyzer/bench/bench_wazero_test.go new file mode 100644 index 00000000..467a78cd --- /dev/null +++ b/spikes/w012-analyzer/bench/bench_wazero_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const validRejection = `{"version":1,"diagnostics":[{"code":"HYB-E3003","severity":"error","message":"bad","span":{"start":{"line":1,"col":1},"end":{"line":1,"col":2}}}],"summary":{"verdict":"reject","errors":1,"warnings":0}}` + +func TestNativeExitHelper(t *testing.T) { + if os.Getenv("W012_TEST_NATIVE_HELPER") != "1" { + return + } + fmt.Print(os.Getenv("W012_TEST_NATIVE_OUTPUT")) + code, _ := strconv.Atoi(os.Getenv("W012_TEST_NATIVE_EXIT")) + os.Exit(code) +} + +func TestNativeRejectionValidation(t *testing.T) { + for _, tc := range []struct { + name, output string + code int + wantErr bool + }{ + {"valid_rejection", validRejection, 1, false}, + {"missing_output", "", 1, true}, + {"invalid_json", "oops", 1, true}, + {"missing_warning_count", strings.Replace(validRejection, `,"warnings":0`, "", 1), 1, true}, + {"missing_code", strings.Replace(validRejection, `"code":"HYB-E3003",`, "", 1), 1, true}, + {"missing_message", strings.Replace(validRejection, `"message":"bad",`, "", 1), 1, true}, + {"invalid_span", strings.Replace(validRejection, `"line":1`, `"line":0`, 1), 1, true}, + {"invalid_severity", strings.Replace(validRejection, `"severity":"error"`, `"severity":"mystery"`, 1), 1, true}, + {"unexpected_success", validRejection, 0, true}, + {"unexpected_failure", validRejection, 2, true}, + {"false_reject", `{"version":1,"diagnostics":[],"summary":{"verdict":"reject","errors":1,"warnings":0}}`, 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestNativeExitHelper$") + cmd.Env = append(os.Environ(), "W012_TEST_NATIVE_HELPER=1", "W012_TEST_NATIVE_EXIT="+strconv.Itoa(tc.code), "W012_TEST_NATIVE_OUTPUT="+tc.output) + output, err := cmd.Output() + got := validateNativeRejection(output, err) + if (got != nil) != tc.wantErr { + t.Fatalf("error=%v, wantErr=%v", got, tc.wantErr) + } + }) + } +} + +func TestMissingNativeBinaryFails(t *testing.T) { + if err := nativeReject(context.Background(), filepath.Join(t.TempDir(), "missing-hyb"), "fixture.hy"); err == nil { + t.Fatal("missing native binary accepted") + } +} + +func TestStatistics(t *testing.T) { + samples := []float64{4, 1, 3, 2} + stats := calcStats(samples) + if stats.Min != 1 || stats.P50 != 2 || stats.P90 != 4 || stats.P99 != 4 || stats.Mean != 2.5 { + t.Fatalf("wrong stats: %+v", stats) + } + if samples[0] != 4 { + t.Fatal("mutated raw samples") + } +} + +func TestNoFiniteBreakEvenWhenWarmPathIsSlower(t *testing.T) { + for _, savings := range []float64{0, -1} { + if got := compileBreakEven(10, savings); got != "no finite break-even (warm path is not faster)" { + t.Fatal(got) + } + } + if got := compileBreakEven(10, 3); got != "3.3 checks (ceil: 4 checks)" { + t.Fatal(got) + } +} diff --git a/spikes/w012-analyzer/cache_invalidation_test.go b/spikes/w012-analyzer/cache_invalidation_test.go new file mode 100644 index 00000000..90014c6a --- /dev/null +++ b/spikes/w012-analyzer/cache_invalidation_test.go @@ -0,0 +1,197 @@ +package analyzer + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/analysis" +) + +func TestVetWrapperInvalidatesEveryHybridInput(t *testing.T) { + moduleDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + fixtureParent := filepath.Join(moduleDir, "testdata") + if err := os.MkdirAll(fixtureParent, 0o700); err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp(fixtureParent, "vet-cache-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + + write := func(path, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(root, path), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + write("go.mod", "module example.com/w012-cache\n\ngo 1.26.9\n") + write("foo.go", "package samplepkg\n\nvar N = 1\n") + + valid, err := os.ReadFile("../../tests/checker-corpus/09-a3-map-slot-valid.hy") + if err != nil { + t.Fatal(err) + } + invalid, err := os.ReadFile("../../tests/checker-corpus/08-a3-map-slot-invalidated.hy") + if err != nil { + t.Fatal(err) + } + write("a.hy", string(valid)) + + wasmPath, err := filepath.Abs(defaultWasmPath) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(wasmPath); err != nil { + t.Fatalf("fresh WASM artifact required: %v", err) + } + + bin := filepath.Join(root, "hybvet") + build := exec.Command("go", "build", "-o", bin, "./cmd/hybvet") + build.Dir = moduleDir + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build hybvet: %v\n%s", err, out) + } + + run := func() (string, error) { + t.Helper() + cmd := exec.Command(bin, "vet", ".") + cmd.Dir = root + cmd.Env = append(os.Environ(), "HYBRID_WASM_PATH="+wasmPath) + out, err := cmd.CombinedOutput() + return string(out), err + } + + if out, err := run(); err != nil { + t.Fatalf("initial valid input should pass: %v\n%s", err, out) + } + + // Change only .hy bytes. The supported wrapper must not reuse the prior + // successful go vet action even though all Go inputs are unchanged. + write("a.hy", string(invalid)) + if out, err := run(); err == nil { + t.Fatalf("invalid .hy edit was hidden by cache; output=%s", out) + } else if !strings.Contains(out, "HYB-E") { + t.Fatalf("expected Hybrid diagnostic after invalid .hy edit: %v\n%s", err, out) + } + + write("a.hy", string(valid)) + if out, err := run(); err != nil { + t.Fatalf("valid restoration should pass: %v\n%s", err, out) + } + + // A second sibling must participate independently in freshness. + write("b.hy", string(invalid)) + if out, err := run(); err == nil { + t.Fatalf("second invalid .hy file was not analyzed; output=%s", out) + } + write("b.hy", string(valid)) + if out, err := run(); err != nil { + t.Fatalf("two valid siblings should pass: %v\n%s", err, out) + } + + // Unrelated Go edits must not break wrapper freshness semantics. + write("foo.go", "package samplepkg\n\nvar N = 2\n") + if out, err := run(); err != nil { + t.Fatalf("unrelated Go edit should remain valid: %v\n%s", err, out) + } +} + +func TestHybridSnapshotDetectsMutationAdditionAndDeletion(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "a.hy") + if err := os.WriteFile(path, []byte("package p\n"), 0o600); err != nil { + t.Fatal(err) + } + snapshot, err := CaptureHybridSnapshot(root) + if err != nil { + t.Fatal(err) + } + if err := snapshot.VerifyCurrent(); err != nil { + t.Fatalf("fresh snapshot rejected: %v", err) + } + + if err := os.WriteFile(path, []byte("package q\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := snapshot.VerifyCurrent(); err == nil { + t.Fatal("mutation must invalidate snapshot") + } + + if err := os.WriteFile(path, []byte("package p\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "b.hy"), []byte("package p\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := snapshot.VerifyCurrent(); err == nil { + t.Fatal("addition must invalidate snapshot") + } + + if err := os.Remove(filepath.Join(root, "b.hy")); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := snapshot.VerifyCurrent(); err == nil { + t.Fatal("deletion must invalidate snapshot") + } +} + +func TestHybridSnapshotRejectsChangedFileAtAnalysisRead(t *testing.T) { + for _, declared := range []bool{false, true} { + label := "scanned" + if declared { + label = "declared" + } + t.Run(label, func(t *testing.T) { + root := t.TempDir() + file := filepath.Join(root, "source.hy") + if err := os.WriteFile(file, []byte("package sample\n"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := CaptureHybridSnapshot(root) + if err != nil { + t.Fatal(err) + } + encoded, err := snapshot.Encode() + if err != nil { + t.Fatal(err) + } + t.Setenv(hybridSnapshotEnv, encoded) + if err := os.WriteFile(file, []byte("package changed\n"), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + anchor, err := parser.ParseFile(fset, filepath.Join(root, "anchor.go"), "package sample\n", 0) + if err != nil { + t.Fatal(err) + } + pass := &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + Fset: fset, + Files: []*ast.File{anchor}, + ReadFile: os.ReadFile, + Report: func(analysis.Diagnostic) { t.Fatal("changed input must not emit stale diagnostics") }, + } + if declared { + pass.OtherFiles = []string{file} + } + if _, err := run(pass); err == nil || !strings.Contains(err.Error(), "snapshot") { + t.Fatalf("%s .hy input changed after snapshot must fail closed, got %v", label, err) + } + }) + } +} diff --git a/spikes/w012-analyzer/cmd/editbench/main.go b/spikes/w012-analyzer/cmd/editbench/main.go new file mode 100644 index 00000000..2c4f6d7a --- /dev/null +++ b/spikes/w012-analyzer/cmd/editbench/main.go @@ -0,0 +1,378 @@ +// editbench measures after-write request latency. It is NOT an LSP or gopls benchmark. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + analyzer "github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer" +) + +type statistics struct { + Min, P50, P90, P95, P99, Mean float64 +} + +func summarize(values []float64) statistics { + if len(values) == 0 { + return statistics{} + } + sorted := append([]float64(nil), values...) + sort.Float64s(sorted) + percentile := func(p float64) float64 { + i := int(math.Ceil(p*float64(len(sorted)))) - 1 + if i < 0 { + i = 0 + } + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] + } + var sum float64 + for _, value := range sorted { + sum += value + } + return statistics{Min: sorted[0], P50: percentile(.5), P90: percentile(.9), P95: percentile(.95), P99: percentile(.99), Mean: sum / float64(len(sorted))} +} + +type sample struct { + Edit int + File string + First string + NativeMS float64 + InProcessMS float64 + ExperimentalReusedMS float64 +} + +type report struct { + Kind string + TimestampUTC string + OS, Arch, Go string + CPUs, GOMAXPROCS, Files, Edits, Warmup int + FixtureSHA256, WASMSHA256, NativeSHA256 string + WASMBytes, NativeBytes int + ColdWASMInitMS, ColdReusedInitMS float64 + Native, InProcess, ReusedGuest statistics + NativeOverInProcessP50, NativeOverReusedP50 float64 + Samples []sample + Limitations []string +} + +type settings struct { + root, wasm, native, fixture, output string + files, edits, warmup int +} + +func editedSource(template string, iteration int) string { + // Fixed-width, non-cumulative changes preserve the registered oracle. + return template + fmt.Sprintf("\n// W012-EDIT-BENCH revision=%06d\n", iteration) +} + +func expectedVerdict(template string) (string, error) { + if strings.Contains(template, "EXPECTED-VERDICT: REJECT") { + return "reject", nil + } + if strings.Contains(template, "EXPECTED-VERDICT: ACCEPT") { + return "accept", nil + } + return "", errors.New("missing preregistered EXPECTED-VERDICT in benchmark fixture") +} + +func artifactInfo(path string) (string, int, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", 0, err + } + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]), len(data), nil +} + +func nativeCheck(binary, file string) (*analyzer.CheckOutput, float64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + start := time.Now() + data, runErr := exec.CommandContext(ctx, binary, "check", file, "--json").Output() + var exitErr *exec.ExitError + if runErr != nil && (!errors.As(runErr, &exitErr) || exitErr.ExitCode() != 1) { + return nil, 0, fmt.Errorf("unexpected native process failure: %w", runErr) + } + out, err := analyzer.DecodeCheckOutput(data) + if err != nil { + return nil, 0, fmt.Errorf("malformed native diagnostics: %w", err) + } + if (runErr == nil) != (out.Summary.Verdict == "accept") { + return nil, 0, fmt.Errorf("native exit code inconsistent with %s", out.Summary.Verdict) + } + return out, float64(time.Since(start).Nanoseconds()) / 1e6, nil +} + +func wasmCheck(engine *analyzer.Engine, source string) (*analyzer.CheckOutput, float64, error) { + start := time.Now() + out, err := engine.Check(source) + if err != nil { + return nil, 0, err + } + return out, float64(time.Since(start).Nanoseconds()) / 1e6, nil +} + +func comparable(native, embedded *analyzer.CheckOutput, expected string) error { + if native.Summary.Verdict != expected || embedded.Summary.Verdict != expected { + return fmt.Errorf("oracle %s: native=%s embedded=%s", expected, native.Summary.Verdict, embedded.Summary.Verdict) + } + if native.Summary.Errors != embedded.Summary.Errors || native.Summary.Warnings != embedded.Summary.Warnings { + return fmt.Errorf("diagnostic count mismatch: %+v %+v", native.Summary, embedded.Summary) + } + if len(native.Diagnostics) != len(embedded.Diagnostics) { + return fmt.Errorf("diagnostic length mismatch %d != %d", len(native.Diagnostics), len(embedded.Diagnostics)) + } + for i, d := range native.Diagnostics { + w := embedded.Diagnostics[i] + if d.Code != w.Code || d.Severity != w.Severity || d.Tier0Rule != w.Tier0Rule || d.Span != w.Span { + return fmt.Errorf("diagnostic %d differs: native=%+v wasm=%+v", i, d, w) + } + } + return nil +} + +func execute(cfg settings) (*report, error) { + if cfg.files <= 0 || cfg.edits <= 0 || cfg.warmup < 0 { + return nil, errors.New("invalid files, edits or warmup count") + } + templateBytes, err := os.ReadFile(cfg.fixture) + if err != nil { + return nil, fmt.Errorf("fixture: %w", err) + } + template := string(templateBytes) + expected, err := expectedVerdict(template) + if err != nil { + return nil, err + } + fixtureHash, _, err := artifactInfo(cfg.fixture) + if err != nil { + return nil, err + } + wasmHash, wasmSize, err := artifactInfo(cfg.wasm) + if err != nil { + return nil, err + } + nativeHash, nativeSize, err := artifactInfo(cfg.native) + if err != nil { + return nil, err + } + workspace, err := os.MkdirTemp("", "w012-editbench-") + if err != nil { + return nil, err + } + defer os.RemoveAll(workspace) + if err := os.WriteFile(filepath.Join(workspace, "go.mod"), []byte("module example.com/w012-editbench\n\ngo 1.26.9\n"), 0600); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(workspace, "anchor.go"), []byte("package editbench\n"), 0600); err != nil { + return nil, err + } + paths := make([]string, cfg.files) + for i := range paths { + paths[i] = filepath.Join(workspace, fmt.Sprintf("file-%03d.hy", i)) + if err := os.WriteFile(paths[i], []byte(editedSource(template, 0)), 0600); err != nil { + return nil, err + } + } + coldStart := time.Now() + engine, err := analyzer.GetEngine(cfg.wasm) + if err != nil { + return nil, fmt.Errorf("cold WASM init: %w", err) + } + coldMS := float64(time.Since(coldStart).Nanoseconds()) / 1e6 + reused, coldReusedMS, err := newExperimentalGuest(cfg.wasm) + if err != nil { + return nil, fmt.Errorf("initialize experimental guest: %w", err) + } + defer reused.Close() + // Mixed-verdict controls expose simple cross-request state residue. + checkOpposite := func() error { + other := "tests/checker-corpus/09-a3-map-slot-valid.hy" + if expected == "accept" { + other = "tests/checker-corpus/08-a3-map-slot-invalidated.hy" + } + file := filepath.Join(cfg.root, other) + data, err := os.ReadFile(file) + if err != nil { + return err + } + want, err := expectedVerdict(string(data)) + if err != nil { + return err + } + n, _, err := nativeCheck(cfg.native, file) + if err != nil { + return err + } + w, _, err := wasmCheck(engine, string(data)) + if err != nil { + return err + } + e, _, err := reused.Check(string(data)) + if err != nil { + return err + } + if err := comparable(n, w, want); err != nil { + return err + } + return comparable(n, e, want) + } + if err := checkOpposite(); err != nil { + return nil, fmt.Errorf("before-run cross-verdict control: %w", err) + } + for i := 0; i < cfg.warmup; i++ { + src := editedSource(template, -i-1) + if err := os.WriteFile(paths[0], []byte(src), 0600); err != nil { + return nil, err + } + n, _, err := nativeCheck(cfg.native, paths[0]) + if err != nil { + return nil, err + } + w, _, err := wasmCheck(engine, src) + if err != nil { + return nil, err + } + e, _, err := reused.Check(src) + if err != nil { + return nil, err + } + if err := comparable(n, w, expected); err != nil { + return nil, fmt.Errorf("warmup %d: %w", i, err) + } + if err := comparable(n, e, expected); err != nil { + return nil, fmt.Errorf("reused warmup %d: %w", i, err) + } + } + samples := make([]sample, 0, cfg.edits) + nativeTimes := make([]float64, 0, cfg.edits) + wasmTimes := make([]float64, 0, cfg.edits) + reusedTimes := make([]float64, 0, cfg.edits) + for i := 0; i < cfg.edits; i++ { + index := i % cfg.files + src := editedSource(template, i+1) + if err := os.WriteFile(paths[index], []byte(src), 0600); err != nil { + return nil, err + } + var n, w, e *analyzer.CheckOutput + var nt, wt, et float64 + order := [][3]int{{0, 1, 2}, {1, 2, 0}, {2, 0, 1}}[i%3] + first := []string{"native", "embedded", "experimental-reused"}[order[0]] + for _, mode := range order { + switch mode { + case 0: + n, nt, err = nativeCheck(cfg.native, paths[index]) + case 1: + w, wt, err = wasmCheck(engine, src) + case 2: + e, et, err = reused.Check(src) + } + if err != nil { + return nil, fmt.Errorf("edit %d request path %d: %w", i+1, mode, err) + } + } + if err := comparable(n, w, expected); err != nil { + return nil, fmt.Errorf("edit %d public path: %w", i+1, err) + } + if err := comparable(n, e, expected); err != nil { + return nil, fmt.Errorf("edit %d reused guest: %w", i+1, err) + } + samples = append(samples, sample{Edit: i + 1, File: filepath.Base(paths[index]), First: first, NativeMS: nt, InProcessMS: wt, ExperimentalReusedMS: et}) + nativeTimes = append(nativeTimes, nt) + wasmTimes = append(wasmTimes, wt) + reusedTimes = append(reusedTimes, et) + } + if err := checkOpposite(); err != nil { + return nil, fmt.Errorf("after-run cross-verdict control: %w", err) + } + r := &report{ + Kind: "single-file-after-write request proxy, NOT gopls/LSP", + TimestampUTC: time.Now().UTC().Format(time.RFC3339), + OS: runtime.GOOS, Arch: runtime.GOARCH, Go: runtime.Version(), CPUs: runtime.NumCPU(), GOMAXPROCS: runtime.GOMAXPROCS(0), + Files: cfg.files, Edits: cfg.edits, Warmup: cfg.warmup, + FixtureSHA256: fixtureHash, WASMSHA256: wasmHash, NativeSHA256: nativeHash, WASMBytes: wasmSize, NativeBytes: nativeSize, + ColdWASMInitMS: coldMS, ColdReusedInitMS: coldReusedMS, Native: summarize(nativeTimes), InProcess: summarize(wasmTimes), ReusedGuest: summarize(reusedTimes), Samples: samples, + Limitations: []string{ + "Not stock gopls, not unsaved buffers, not editor keystroke-to-diagnostic latency.", + "Each request checks the edited file only, not all 25 workspace files.", + "Durations begin after the write; native includes subprocess, file read and envelope decode; embedded receives in-memory source bytes.", + "Cold WASM initialization is separate from warm Engine.Check timings.", + "The experimental reused guest is strictly sequential and not a production instance pool; cancellation, concurrency and lifetime safety remain unproven.", + "One fixture and one CI machine are insufficient for population-level performance or memory claims.", + }, + } + if r.InProcess.P50 > 0 { + r.NativeOverInProcessP50 = r.Native.P50 / r.InProcess.P50 + } + if r.ReusedGuest.P50 > 0 { + r.NativeOverReusedP50 = r.Native.P50 / r.ReusedGuest.P50 + } + if cfg.output != "" { + b, err := json.MarshalIndent(r, "", " ") + if err != nil { + return nil, err + } + if err := os.WriteFile(cfg.output, append(b, '\n'), 0600); err != nil { + return nil, err + } + } + return r, nil +} + +func main() { + root := flag.String("root", ".", "repository root") + wasm := flag.String("wasm", os.Getenv("HYBRID_WASM_PATH"), "WASM artifact") + native := flag.String("native", os.Getenv("HYBRID_NATIVE_PATH"), "native CLI") + fixture := flag.String("fixture", "", "checker corpus .hy source") + files := flag.Int("files", 25, "workspace file count") + edits := flag.Int("edits", 100, "sequential edits") + warmup := flag.Int("warmup", 10, "untimed checks per path") + output := flag.String("output", "", "raw JSON output") + flag.Parse() + if *wasm == "" { + *wasm = filepath.Join(*root, "compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm") + } + if *native == "" { + *native = filepath.Join(*root, "compiler/target/release/hyb") + } + if *fixture == "" { + *fixture = filepath.Join(*root, "tests/checker-corpus/08-a3-map-slot-invalidated.hy") + } + for _, p := range []*string{wasm, native, fixture} { + path, err := filepath.Abs(*p) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + *p = path + } + r, err := execute(settings{root: *root, wasm: *wasm, native: *native, fixture: *fixture, output: *output, files: *files, edits: *edits, warmup: *warmup}) + if err != nil { + fmt.Fprintln(os.Stderr, "W-012 edit benchmark FAILED:", err) + os.Exit(1) + } + fmt.Printf("W-012 after-write request proxy (NOT gopls): %d edits, %d files\n", r.Edits, r.Files) + fmt.Printf("WASM cold initialization %.3f ms; bytes WASM=%d native=%d\n", r.ColdWASMInitMS, r.WASMBytes, r.NativeBytes) + fmt.Printf("Native subprocess+decode: p50 %.3f ms p95 %.3f ms p99 %.3f ms\n", r.Native.P50, r.Native.P95, r.Native.P99) + fmt.Printf("Real public Engine.Check: p50 %.3f ms p95 %.3f ms p99 %.3f ms\n", r.InProcess.P50, r.InProcess.P95, r.InProcess.P99) + fmt.Printf("Native/embedded p50 ratio %.3fx (above 1: embedded faster)\n", r.NativeOverInProcessP50) + fmt.Printf("EXPERIMENTAL sequential reused guest: p50 %.3f ms p95 %.3f ms p99 %.3f ms; cold init %.3f ms\n", r.ReusedGuest.P50, r.ReusedGuest.P95, r.ReusedGuest.P99, r.ColdReusedInitMS) + fmt.Printf("Native/experimental reused p50 %.3fx (research only)\n", r.NativeOverReusedP50) + fmt.Println("No stock gopls, whole-package latency, or production memory footprint claim.") +} diff --git a/spikes/w012-analyzer/cmd/editbench/main_test.go b/spikes/w012-analyzer/cmd/editbench/main_test.go new file mode 100644 index 00000000..35b49e65 --- /dev/null +++ b/spikes/w012-analyzer/cmd/editbench/main_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" + + analyzer "github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer" +) + +func TestStatisticsAreStable(t *testing.T) { + values := []float64{4, 1, 3, 2} + s := summarize(values) + if s.Min != 1 || s.P50 != 2 || s.P90 != 4 || s.P95 != 4 || s.P99 != 4 || s.Mean != 2.5 { + t.Fatalf("unexpected statistics: %+v", s) + } + if values[0] != 4 { + t.Fatal("sample array mutated by summarize") + } +} + +func TestEditsAreNonCumulative(t *testing.T) { + const src = "package example\n" + first := editedSource(src, 1) + second := editedSource(src, 2) + if first == second || !strings.Contains(second, "revision=000002") || strings.Contains(second, "revision=000001") { + t.Fatalf("not a fixed-width, non-cumulative edit: %q / %q", first, second) + } + if src != "package example\n" { + t.Fatal("template unexpectedly changed") + } +} + +func TestExpectedVerdictRequiresCorpusOracle(t *testing.T) { + for _, c := range []struct{ src, want string }{ + {"// EXPECTED-VERDICT: ACCEPT\n", "accept"}, + {"// EXPECTED-VERDICT: ACCEPT-MANAGED\n", "accept"}, + {"// EXPECTED-VERDICT: REJECT A3\n", "reject"}, + } { + got, err := expectedVerdict(c.src) + if err != nil || got != c.want { + t.Fatalf("src=%s expected=%s got=%s err=%v", c.src, c.want, got, err) + } + } + if _, err := expectedVerdict("package x\n"); err == nil { + t.Fatal("unregistered fixture accepted") + } +} + +func TestDiagnosticParityRequiresFullModeledSpan(t *testing.T) { + first := &analyzer.CheckOutput{ + Version: 1, + Summary: analyzer.Summary{Verdict: "reject", Errors: 1, Warnings: 0}, + Diagnostics: []analyzer.Diagnostic{ + {Code: "HYB-E3003", Severity: "error", Tier0Rule: "A3", Span: analyzer.Span{ + Start: analyzer.Pos{Line: 3, Col: 2}, End: analyzer.Pos{Line: 3, Col: 4}, + }}, + }, + } + second := *first + second.Diagnostics = append([]analyzer.Diagnostic(nil), first.Diagnostics...) + if err := comparable(first, &second, "reject"); err != nil { + t.Fatalf("identical outputs rejected: %v", err) + } + second.Diagnostics[0].Span.Start.Col++ + if err := comparable(first, &second, "reject"); err == nil { + t.Fatal("changed source span accepted") + } + if err := comparable(first, first, "accept"); err == nil { + t.Fatal("incorrect registered verdict accepted") + } +} + +func TestInvalidRunCounts(t *testing.T) { + for _, cfg := range []settings{ + {files: 0, edits: 100, warmup: 0}, + {files: 25, edits: 0, warmup: 0}, + {files: 25, edits: 100, warmup: -1}, + } { + if _, err := execute(cfg); err == nil { + t.Fatalf("invalid settings accepted: %+v", cfg) + } + } +} diff --git a/spikes/w012-analyzer/cmd/editbench/reused.go b/spikes/w012-analyzer/cmd/editbench/reused.go new file mode 100644 index 00000000..666f61e2 --- /dev/null +++ b/spikes/w012-analyzer/cmd/editbench/reused.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "errors" + "os" + "time" + + analyzer "github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer" + "github.com/tetratelabs/wazero" +) + +// experimentalGuest is deliberately sequential and never used by the public +// Analyzer. It is a timing control, not a proposal to deploy shared mutable +// guest memory without proven state and cancellation isolation. +type experimentalGuest struct { + ctx context.Context + runtime wazero.Runtime + abi *analyzer.WazeroInstanceABI +} + +func newExperimentalGuest(path string) (*experimentalGuest, float64, error) { + start := time.Now() + wasm, err := os.ReadFile(path) + if err != nil { + return nil, 0, err + } + ctx := context.Background() + rt := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) + compiled, err := rt.CompileModule(ctx, wasm) + if err != nil { + _ = rt.Close(ctx) + return nil, 0, err + } + mod, err := rt.InstantiateModule(ctx, compiled, wazero.NewModuleConfig().WithName("")) + if err != nil { + _ = rt.Close(ctx) + return nil, 0, err + } + abi, err := analyzer.NewWazeroInstanceABI(mod) + if err != nil { + _ = rt.Close(ctx) + return nil, 0, err + } + return &experimentalGuest{ctx: ctx, runtime: rt, abi: abi}, float64(time.Since(start).Nanoseconds()) / 1e6, nil +} + +func (g *experimentalGuest) Close() error { return g.runtime.Close(g.ctx) } + +func (g *experimentalGuest) Check(src string) (*analyzer.CheckOutput, float64, error) { + ctx, cancel := context.WithTimeout(g.ctx, 30*time.Second) + defer cancel() + start := time.Now() + res := analyzer.CheckSourceWithABI(ctx, g.abi, src) + if err := res.CheckInvariant(); err != nil { + return nil, 0, err + } + if res.Verdict == analyzer.VerdictAnalysisFailure { + return nil, 0, res.Failure + } + if res.Summary == nil { + return nil, 0, errors.New("reused WASM guest returned nil summary") + } + return &analyzer.CheckOutput{Version: 1, Summary: *res.Summary, Diagnostics: res.Diagnostics}, float64(time.Since(start).Nanoseconds()) / 1e6, nil +} diff --git a/spikes/w012-analyzer/cmd/hybvet/main.go b/spikes/w012-analyzer/cmd/hybvet/main.go new file mode 100644 index 00000000..6b1dcc9c --- /dev/null +++ b/spikes/w012-analyzer/cmd/hybvet/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "fmt" + "os" + + analyzer "github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer" + "golang.org/x/tools/go/analysis/singlechecker" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "vet" { + if err := analyzer.RunVet(context.Background(), os.Args[2:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + singlechecker.Main(analyzer.Analyzer) +} diff --git a/spikes/w012-analyzer/compilation_cache_test.go b/spikes/w012-analyzer/compilation_cache_test.go new file mode 100644 index 00000000..1f30daf5 --- /dev/null +++ b/spikes/w012-analyzer/compilation_cache_test.go @@ -0,0 +1,54 @@ +package analyzer + +import ( + "os" + "path/filepath" + "testing" +) + +// This test exercises two independent runtimes sharing only a disk compilation +// cache. It validates semantics and cache error handling, not a timing target. +func TestWasmCompilationCachePreservesCheckerVerdicts(t *testing.T) { + cacheDir := t.TempDir() + accept, err := os.ReadFile("../../tests/checker-corpus/09-a3-map-slot-valid.hy") + if err != nil { + t.Fatal(err) + } + reject, err := os.ReadFile("../../tests/checker-corpus/08-a3-map-slot-invalidated.hy") + if err != nil { + t.Fatal(err) + } + for run := 0; run < 2; run++ { + engine, err := newEngine(defaultWasmPath, cacheDir) + if err != nil { + t.Fatalf("runtime #%d open: %v", run, err) + } + for _, tc := range []struct { + src string + verdict string + }{ + {string(accept), "accept"}, {string(reject), "reject"}, + } { + out, checkErr := engine.Check(tc.src) + if checkErr != nil || out.Summary.Verdict != tc.verdict { + t.Fatalf("runtime #%d verdict expected %s, output=%+v, err=%v", run, tc.verdict, out, checkErr) + } + } + if err := engine.Close(); err != nil { + t.Fatalf("close runtime #%d: %v", run, err) + } + } +} + +func TestWasmCompilationCacheDirectoryFailureIsFailClosed(t *testing.T) { + file := filepath.Join(t.TempDir(), "not-a-cache-directory") + if err := os.WriteFile(file, []byte("occupied"), 0600); err != nil { + t.Fatal(err) + } + if engine, err := newEngine(defaultWasmPath, file); err == nil { + if engine != nil { + _ = engine.Close() + } + t.Fatal("cache configuration error silently fell back to noncached runtime") + } +} diff --git a/spikes/w012-analyzer/engine.go b/spikes/w012-analyzer/engine.go new file mode 100644 index 00000000..6a72e8d4 --- /dev/null +++ b/spikes/w012-analyzer/engine.go @@ -0,0 +1,333 @@ +package analyzer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" +) + +type AnalysisVerdict string + +const ( + VerdictAccept AnalysisVerdict = "Accept" + VerdictReject AnalysisVerdict = "Reject" + VerdictAnalysisFailure AnalysisVerdict = "AnalysisFailure" +) + +type AnalysisResult struct { + Verdict AnalysisVerdict + Diagnostics []Diagnostic + Failure error + Summary *Summary +} + +// Invariant: AnalysisFailure MUST NEVER become Accept, nor masquerade as Reject. +func (r *AnalysisResult) CheckInvariant() error { + switch r.Verdict { + case VerdictAccept: + if r.Failure != nil { + return errors.New("INVARIANT VIOLATION: Failure present in Accept verdict") + } + // Accepted source may still carry managed-safety warnings or notes. + for _, d := range r.Diagnostics { + if d.Severity != "warning" && d.Severity != "note" { + return errors.New("INVARIANT VIOLATION: error or invalid diagnostic in Accept verdict") + } + } + case VerdictReject: + if r.Failure != nil { + return errors.New("INVARIANT VIOLATION: Failure present in Reject verdict (infrastructure failure masquerading as user rejection)") + } + case VerdictAnalysisFailure: + if r.Failure == nil { + return errors.New("INVARIANT VIOLATION: AnalysisFailure without concrete error") + } + default: + return fmt.Errorf("INVARIANT VIOLATION: unknown verdict %s", r.Verdict) + } + return nil +} + +// WasmABI abstracts the low-level WASM memory and function boundary +type WasmABI interface { + Alloc(ctx context.Context, byteLen uint64) (uint32, error) + Free(ctx context.Context, ptr, byteLen uint64) error + Check(ctx context.Context, ptr, byteLen uint64) (uint64, error) + WriteMemory(offset uint32, b []byte) bool + ReadMemory(offset, byteCount uint32) ([]byte, bool) +} + +// WazeroInstanceABI implements WasmABI backed by a real Wazero api.Module +type WazeroInstanceABI struct { + mod api.Module + allocFn api.Function + freeFn api.Function + checkFn api.Function +} + +func NewWazeroInstanceABI(mod api.Module) (*WazeroInstanceABI, error) { + allocFn := mod.ExportedFunction("hy_wasm_alloc") + freeFn := mod.ExportedFunction("hy_wasm_free") + checkFn := mod.ExportedFunction("hy_wasm_check") + + if allocFn == nil || freeFn == nil || checkFn == nil { + return nil, errors.New("missing required exports (hy_wasm_alloc, hy_wasm_free, hy_wasm_check)") + } + if mod.Memory() == nil { + return nil, errors.New("missing exported memory") + } + + return &WazeroInstanceABI{ + mod: mod, + allocFn: allocFn, + freeFn: freeFn, + checkFn: checkFn, + }, nil +} + +func (w *WazeroInstanceABI) Alloc(ctx context.Context, byteLen uint64) (uint32, error) { + res, err := w.allocFn.Call(ctx, byteLen) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, errors.New("alloc returned empty result") + } + ptr := uint32(res[0]) + if ptr == 0 && byteLen > 0 { + return 0, errors.New("alloc returned null pointer") + } + return ptr, nil +} + +func (w *WazeroInstanceABI) Free(ctx context.Context, ptr, byteLen uint64) error { + _, err := w.freeFn.Call(ctx, ptr, byteLen) + return err +} + +func (w *WazeroInstanceABI) Check(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + res, err := w.checkFn.Call(ctx, ptr, byteLen) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, errors.New("check returned empty result") + } + return res[0], nil +} + +func (w *WazeroInstanceABI) WriteMemory(offset uint32, b []byte) bool { + return w.mod.Memory().Write(offset, b) +} + +func (w *WazeroInstanceABI) ReadMemory(offset, byteCount uint32) ([]byte, bool) { + return w.mod.Memory().Read(offset, byteCount) +} + +type WasmEngine struct { + ctx context.Context + runtime wazero.Runtime + compiled wazero.CompiledModule +} + +func NewWasmEngine(ctx context.Context, wasmBytes []byte) (*WasmEngine, error) { + r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) + compiled, err := r.CompileModule(ctx, wasmBytes) + if err != nil { + r.Close(ctx) + return nil, fmt.Errorf("compilation failed: %w", err) + } + return &WasmEngine{ + ctx: ctx, + runtime: r, + compiled: compiled, + }, nil +} + +func (e *WasmEngine) Close() error { + return e.runtime.Close(e.ctx) +} + +func (e *WasmEngine) CheckSource(ctx context.Context, src string) *AnalysisResult { + if err := ctx.Err(); err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: context timeout before instantiation: %w", err), + } + } + + mod, err := e.runtime.InstantiateModule(ctx, e.compiled, wazero.NewModuleConfig().WithName("")) + if err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: module instantiation failed: %w", err), + } + } + defer mod.Close(ctx) + + abi, err := NewWazeroInstanceABI(mod) + if err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: ABI initialization failed: %w", err), + } + } + + return CheckSourceWithABI(ctx, abi, src) +} + +// CheckSourceWithABI runs analysis against any WasmABI implementation +func CheckSourceWithABI(ctx context.Context, abi WasmABI, src string) *AnalysisResult { + if err := ctx.Err(); err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: execution timeout: %w", err), + } + } + + srcBytes := []byte(src) + srcLen := uint64(len(srcBytes)) + + srcPtr, err := abi.Alloc(ctx, srcLen) + if err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: guest memory alloc failed: %w", err), + } + } + // This flag means that cleanup is still due, not that a previous free + // returned success. A trapped free may already have deallocated memory. + sourceFreePending := true + defer func() { + if sourceFreePending { + _ = abi.Free(ctx, uint64(srcPtr), srcLen) + } + }() + + if !abi.WriteMemory(srcPtr, srcBytes) { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: errors.New("infrastructure error: failed writing source to guest linear memory"), + } + } + + packed, err := abi.Check(ctx, uint64(srcPtr), srcLen) + if err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: WASM trap/panic during execution: %w", err), + } + } + + // Mark the release as attempted BEFORE invoking guest code. On a trap, + // retrying from the defer could double-free a partially released buffer. + sourceFreePending = false + if err := abi.Free(ctx, uint64(srcPtr), srcLen); err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: failed freeing guest source buffer: %w", err), + } + } + + if err := ctx.Err(); err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: context cancelled post-call: %w", err), + } + } + + outPtr := uint32(packed >> 32) + outLen := uint32(packed & 0xFFFFFFFF) + if outPtr == 0 || outLen == 0 { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: invalid guest output buffer: ptr=%d, len=%d", outPtr, outLen), + } + } + + outBytes, ok := abi.ReadMemory(outPtr, outLen) + if !ok { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: out-of-bounds guest memory read: ptr=%d, len=%d", outPtr, outLen), + } + } + // Wazero exposes a view into guest memory. Copy it before returning ownership + // of the guest buffer so decoding never observes freed linear memory. + outCopy := append([]byte(nil), outBytes...) + if err := abi.Free(ctx, uint64(outPtr), uint64(outLen)); err != nil { + return &AnalysisResult{ + Verdict: VerdictAnalysisFailure, + Failure: fmt.Errorf("infrastructure error: failed freeing guest output buffer: %w", err), + } + } + + out, err := DecodeCheckOutput(outCopy) + if err != nil { + return &AnalysisResult{Verdict: VerdictAnalysisFailure, Failure: fmt.Errorf("infrastructure error: malformed diagnostic envelope: %w", err)} + } + if out.Summary.Verdict == "accept" { + return &AnalysisResult{Verdict: VerdictAccept, Diagnostics: out.Diagnostics, Summary: &out.Summary} + } + return &AnalysisResult{Verdict: VerdictReject, Diagnostics: out.Diagnostics, Summary: &out.Summary} +} + +// DecodeCheckOutput validates semantic structure, not merely JSON syntax. +// Unknown additive fields remain allowed; missing fields and contradictory +// summaries are infrastructure failures rather than user-code verdicts. +func DecodeCheckOutput(data []byte) (*CheckOutput, error) { + var wire struct { + Version *int `json:"version"` + Diagnostics *[]Diagnostic `json:"diagnostics"` + Summary *struct { + Verdict string `json:"verdict"` + Errors *int `json:"errors"` + Warnings *int `json:"warnings"` + } `json:"summary"` + } + if err := json.Unmarshal(data, &wire); err != nil { + return nil, err + } + if wire.Version == nil || *wire.Version != 1 || wire.Diagnostics == nil || wire.Summary == nil || wire.Summary.Errors == nil || wire.Summary.Warnings == nil { + return nil, errors.New("missing required fields or unsupported envelope version") + } + var out CheckOutput + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + errorsCount, warningsCount := 0, 0 + for _, d := range out.Diagnostics { + if d.Code == "" || d.Message == "" || d.Span.Start.Line < 1 || d.Span.Start.Col < 1 || d.Span.End.Line < d.Span.Start.Line || d.Span.End.Col < 1 || (d.Span.End.Line == d.Span.Start.Line && d.Span.End.Col < d.Span.Start.Col) { + return nil, errors.New("invalid diagnostic code, message, or span") + } + switch d.Severity { + case "error": + errorsCount++ + case "warning": + warningsCount++ + case "note": + default: + return nil, fmt.Errorf("unknown diagnostic severity %q", d.Severity) + } + } + if out.Summary.Errors != errorsCount || out.Summary.Warnings != warningsCount { + return nil, errors.New("summary counts disagree with diagnostics") + } + switch out.Summary.Verdict { + case "accept": + if errorsCount != 0 { + return nil, errors.New("accept verdict with error diagnostics") + } + case "reject": + if errorsCount == 0 { + return nil, errors.New("reject verdict without error diagnostics") + } + default: + return nil, fmt.Errorf("unknown verdict %q", out.Summary.Verdict) + } + return &out, nil +} diff --git a/spikes/w012-analyzer/engine_concurrency_test.go b/spikes/w012-analyzer/engine_concurrency_test.go new file mode 100644 index 00000000..976bf340 --- /dev/null +++ b/spikes/w012-analyzer/engine_concurrency_test.go @@ -0,0 +1,94 @@ +package analyzer + +import ( + "context" + "fmt" + "go/token" + "os" + "sync" + "testing" +) + +// A single Wazero Runtime can serve concurrent checks only if each disposable +// guest has an anonymous module name. A named module fails on overlapping +// instantiation even though sequential checks and the race detector may pass. +func TestConcurrentDisposableGuests(t *testing.T) { + wasmBytes, err := os.ReadFile(defaultWasmPath) + if err != nil { + t.Fatalf("fresh WASM artifact required: %v", err) + } + engine, err := NewWasmEngine(context.Background(), wasmBytes) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + acceptBytes, err := os.ReadFile("../../tests/checker-corpus/09-a3-map-slot-valid.hy") + if err != nil { + t.Fatal(err) + } + rejectBytes, err := os.ReadFile("../../tests/checker-corpus/08-a3-map-slot-invalidated.hy") + if err != nil { + t.Fatal(err) + } + const workers, iterations = 12, 40 + start := make(chan struct{}) + failures := make(chan error, workers) + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + <-start + for i := 0; i < iterations; i++ { + for _, tc := range []struct { + source string + want AnalysisVerdict + }{ + {string(acceptBytes), VerdictAccept}, + {string(rejectBytes), VerdictReject}, + } { + res := engine.CheckSource(context.Background(), tc.source) + if err := res.CheckInvariant(); err != nil { + failures <- fmt.Errorf("worker %d iteration %d: %w", worker, i, err) + return + } + if res.Verdict != tc.want { + failures <- fmt.Errorf("worker %d iteration %d: got %s, want %s (%v)", worker, i, res.Verdict, tc.want, res.Failure) + return + } + } + } + }(worker) + } + close(start) + wg.Wait() + close(failures) + for err := range failures { + t.Error(err) + } +} + +func TestHybridUnicodeDiagnosticPosition(t *testing.T) { + content := "αβx\nplain\r\n" + fset := token.NewFileSet() + file := fset.AddFile("unicode.hy", -1, len(content)) + file.SetLinesForContent([]byte(content)) + for _, tc := range []struct{ line, col, want int }{ + {1, 1, 0}, {1, 2, len("α")}, {1, 3, len("αβ")}, + {1, 4, len("αβx")}, {2, 3, len("αβx\npl")}, + {2, 6, len("αβx\nplain")}, + } { + pos, err := hybridTokenPos(file, content, tc.line, tc.col) + if err != nil { + t.Fatalf("%d:%d: %v", tc.line, tc.col, err) + } + if got := file.Offset(pos); got != tc.want { + t.Fatalf("%d:%d mapped to byte %d; want %d", tc.line, tc.col, got, tc.want) + } + } + for _, tc := range []struct{ line, col int }{{1, 5}, {2, 7}, {3, 2}, {0, 1}, {1, 0}} { + if _, err := hybridTokenPos(file, content, tc.line, tc.col); err == nil { + t.Errorf("out-of-range %d:%d was accepted", tc.line, tc.col) + } + } +} diff --git a/spikes/w012-analyzer/envelope_test.go b/spikes/w012-analyzer/envelope_test.go new file mode 100644 index 00000000..16be72b3 --- /dev/null +++ b/spikes/w012-analyzer/envelope_test.go @@ -0,0 +1,81 @@ +package analyzer + +import ( + "context" + "encoding/hex" + "encoding/json" + "testing" + "time" +) + +func TestMalformedEnvelopeFailsClosed(t *testing.T) { + valid := `{"version":1,"diagnostics":[],"summary":{"verdict":"accept","errors":0,"warnings":0}}` + for _, payload := range []string{ + `{}`, `null`, `[]`, `{"summary":{"verdict":"accept"}}`, + `{"version":1,"diagnostics":[],"summary":{"verdict":"mystery","errors":0,"warnings":0}}`, + `{"version":1,"diagnostics":[],"summary":{"verdict":"reject","errors":0,"warnings":0}}`, + `{"version":1,"diagnostics":[],"summary":{"verdict":"accept","errors":1,"warnings":0}}`, + `{"version":1,"diagnostics":null,"summary":{"verdict":"accept","errors":0,"warnings":0}}`, + `{"version":1,"diagnostics":[],"summary":{"verdict":"accept","errors":-1,"warnings":0}}`, + `{"version":2,"diagnostics":[],"summary":{"verdict":"accept","errors":0,"warnings":0}}`, + } { + t.Run(payload, func(t *testing.T) { + abi := &MockWasmABI{Memory: make([]byte, 4096), CheckFn: func(context.Context, uint64, uint64) (uint64, error) { + return uint64(1024)<<32 | uint64(len(payload)), nil + }} + copy(abi.Memory[1024:], payload) + result := CheckSourceWithABI(context.Background(), abi, "source") + if result.Verdict != VerdictAnalysisFailure || result.Failure == nil || len(result.Diagnostics) != 0 { + t.Fatalf("not fail-closed: %+v", result) + } + }) + } + var envelope map[string]any + if err := json.Unmarshal([]byte(valid), &envelope); err != nil { + t.Fatal(err) + } + envelope["future_field"] = "allowed" + data, _ := json.Marshal(envelope) + if _, err := DecodeCheckOutput(data); err != nil { + t.Fatalf("additive field rejected: %v", err) + } +} + +// Minimal real WASM guest: memory, allocator, free, and check exports. The +// check body is either unreachable or an infinite loop; no mocked Go error. +func TestRealGuestTrapAndActiveTimeout(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"trap", "00000042000b"}, + {"active_timeout", "0003400c000b42000b"}, + } { + t.Run(tc.name, func(t *testing.T) { + wasm, err := hex.DecodeString(guestModuleHex(tc.body)) + if err != nil { + t.Fatal(err) + } + engine, err := NewWasmEngine(context.Background(), wasm) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + result := engine.CheckSource(ctx, "input") + if result.Verdict != VerdictAnalysisFailure || result.Failure == nil { + t.Fatalf("not fail-closed: %+v", result) + } + if tc.name == "active_timeout" && ctx.Err() == nil { + t.Fatal("guest returned before deadline") + } + }) + } +} + +func guestModuleHex(checkBody string) string { + prefix := "0061736d0100000001110360017f017f60027f7f0060027f7f017e0304030001020503010001073904066d656d6f727902000d68795f7761736d5f616c6c6f6300000c68795f7761736d5f6672656500010d68795f7761736d5f636865636b0002" + body, _ := hex.DecodeString(checkBody) + code := []byte{3, 4, 0, 0x41, 8, 0x0b, 2, 0, 0x0b, byte(len(body))} + code = append(code, body...) + section := append([]byte{10, byte(len(code))}, code...) + return prefix + hex.EncodeToString(section) +} diff --git a/spikes/w012-analyzer/eof_position_test.go b/spikes/w012-analyzer/eof_position_test.go new file mode 100644 index 00000000..e45547ac --- /dev/null +++ b/spikes/w012-analyzer/eof_position_test.go @@ -0,0 +1,123 @@ +package analyzer + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/analysis" +) + +// Unterminated blocks produce a real HYB-E0002 at EOF. A final newline must +// not turn that source error into an infrastructure/position-mapping failure. +func TestAnalyzerReportsEOFParseErrors(t *testing.T) { + tests := []struct { + name string + source string + line int + col int // Unicode-scalar column reported by the Hybrid lexer. + }{ + {"lf", "package sample\nfn broken() {\n", 3, 1}, + {"crlf", "package sample\r\nfn broken() {\r\n", 3, 1}, + {"blank_final_line", "package sample\nfn broken() {\n\n", 4, 1}, + {"no_final_newline", "package sample\nfn broken() {", 2, 14}, + {"unicode_at_eof", "package sample\nfn broken() { // 雪", 2, 19}, + } + engine, err := GetEngine("") + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := engine.Check(tt.source) + if err != nil { + t.Fatal(err) + } + if out.Summary.Verdict != "reject" || len(out.Diagnostics) != 1 { + t.Fatalf("fixture must produce one parse error: %+v", out) + } + d := out.Diagnostics[0] + if d.Code != "HYB-E0002" || d.Span.Start != (Pos{Line: tt.line, Col: tt.col}) { + t.Fatalf("fixture must report its actual EOF position: %+v", d) + } + + pass, hyPath := eofTestPass(t, tt.source) + var reported []analysis.Diagnostic + pass.Report = func(d analysis.Diagnostic) { reported = append(reported, d) } + if _, err := run(pass); err != nil { + t.Fatalf("source parse error became an analyzer failure: %v", err) + } + if len(reported) != 1 || !strings.Contains(reported[0].Message, "HYB-E0002") { + t.Fatalf("expected the parse diagnostic, got %+v", reported) + } + pos := pass.Fset.Position(reported[0].Pos) + byteCol := len(tt.source) - strings.LastIndex(tt.source, "\n") + if pos.Filename != hyPath || pos.Offset != len(tt.source) || pos.Line != tt.line || pos.Column != byteCol { + t.Fatalf("wrong EOF position: %+v; want %s:%d:%d offset %d", pos, hyPath, tt.line, byteCol, len(tt.source)) + } + + file := pass.Fset.File(reported[0].Pos) + if _, err := hybridTokenPos(file, tt.source, tt.line, tt.col+1); err == nil { + t.Fatal("token-table padding must not accept a column after EOF") + } + if _, err := hybridTokenPos(file, tt.source, tt.line+1, 1); err == nil { + t.Fatal("token-table padding must not accept a line after EOF") + } + }) + } +} + +func TestAnalyzerEmptySourceHasEOFPosition(t *testing.T) { + pass, hyPath := eofTestPass(t, "") + pass.Report = func(d analysis.Diagnostic) { t.Errorf("empty source should be accepted: %+v", d) } + if _, err := run(pass); err != nil { + t.Fatal(err) + } + var file *token.File + pass.Fset.Iterate(func(candidate *token.File) bool { + if candidate.Name() == hyPath { + file = candidate + return false + } + return true + }) + if file == nil { + t.Fatal("empty Hybrid source missing from the token file set") + } + pos, err := hybridTokenPos(file, "", 1, 1) + if err != nil { + t.Fatalf("empty source EOF must map to 1:1: %v", err) + } + got := pass.Fset.Position(pos) + if got.Filename != hyPath || got.Offset != 0 || got.Line != 1 || got.Column != 1 { + t.Fatalf("wrong empty-source EOF position: %+v", got) + } + if _, err := hybridTokenPos(file, "", 1, 2); err == nil { + t.Fatal("empty source must not acquire a synthetic second column") + } +} + +func eofTestPass(t *testing.T, source string) (*analysis.Pass, string) { + t.Helper() + dir := t.TempDir() + hyPath := filepath.Join(dir, "incomplete.hy") + if err := os.WriteFile(hyPath, []byte(source), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + goFile, err := parser.ParseFile(fset, filepath.Join(dir, "anchor.go"), "package sample\n", 0) + if err != nil { + t.Fatal(err) + } + return &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + Fset: fset, + Files: []*ast.File{goFile}, + }, hyPath +} diff --git a/spikes/w012-analyzer/fault_injection_test.go b/spikes/w012-analyzer/fault_injection_test.go new file mode 100644 index 00000000..5893b00b --- /dev/null +++ b/spikes/w012-analyzer/fault_injection_test.go @@ -0,0 +1,272 @@ +package analyzer + +import ( + "context" + "errors" + "os" + "testing" + "time" +) + +type MockWasmABI struct { + AllocFn func(ctx context.Context, byteLen uint64) (uint32, error) + FreeFn func(ctx context.Context, ptr, byteLen uint64) error + CheckFn func(ctx context.Context, ptr, byteLen uint64) (uint64, error) + Memory []byte + FailRead bool + FailWrite bool +} + +func (m *MockWasmABI) Alloc(ctx context.Context, byteLen uint64) (uint32, error) { + if m.AllocFn != nil { + return m.AllocFn(ctx, byteLen) + } + return 0, nil +} + +func (m *MockWasmABI) Free(ctx context.Context, ptr, byteLen uint64) error { + if m.FreeFn != nil { + return m.FreeFn(ctx, ptr, byteLen) + } + return nil +} + +func (m *MockWasmABI) Check(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + if m.CheckFn != nil { + return m.CheckFn(ctx, ptr, byteLen) + } + return 0, nil +} + +func (m *MockWasmABI) WriteMemory(offset uint32, b []byte) bool { + if m.FailWrite { + return false + } + if int(offset)+len(b) > len(m.Memory) { + return false + } + copy(m.Memory[offset:], b) + return true +} + +func (m *MockWasmABI) ReadMemory(offset, length uint32) ([]byte, bool) { + if m.FailRead { + return nil, false + } + if int(offset)+int(length) > len(m.Memory) { + return nil, false + } + return m.Memory[offset : offset+length], true +} + +// Releasing a guest allocation may fail after the guest actually freed it. +// Repeating the call from deferred cleanup risks a double-free or guest trap. +func TestSourceReleaseFailureIsNotRetried(t *testing.T) { + freeCalls := 0 + mock := &MockWasmABI{ + Memory: make([]byte, 128), + AllocFn: func(context.Context, uint64) (uint32, error) { + return 8, nil + }, + CheckFn: func(context.Context, uint64, uint64) (uint64, error) { + return 0, nil + }, + FreeFn: func(context.Context, uint64, uint64) error { + freeCalls++ + return errors.New("guest free trapped after deallocation") + }, + } + + res := CheckSourceWithABI(context.Background(), mock, "package sample") + if res.Verdict != VerdictAnalysisFailure || res.Failure == nil { + t.Fatalf("source release failure must fail closed: %+v", res) + } + if freeCalls != 1 { + t.Fatalf("source release must be attempted exactly once, got %d calls", freeCalls) + } +} + +func TestFailureDomainSeparation(t *testing.T) { + ctx := context.Background() + wasmBytes, err := os.ReadFile(defaultWasmPath) + if err != nil { + t.Fatalf("failed to read wasm binary: %v", err) + } + + engine, err := NewWasmEngine(ctx, wasmBytes) + if err != nil { + t.Fatalf("failed to init engine: %v", err) + } + defer engine.Close() + + // 1. Positive Control: Valid Program -> VerdictAccept + validSrcBytes, err := os.ReadFile("../../tests/checker-corpus/09-a3-map-slot-valid.hy") + if err != nil { + t.Fatalf("failed to read valid corpus file: %v", err) + } + resValid := engine.CheckSource(ctx, string(validSrcBytes)) + if err := resValid.CheckInvariant(); err != nil { + t.Fatal(err) + } + if resValid.Verdict != VerdictAccept { + t.Fatalf("Expected VerdictAccept, got %v (%+v)", resValid.Verdict, resValid.Diagnostics) + } + t.Logf("[PASS] Valid program -> %s", resValid.Verdict) + + // 2. Negative Control: Hybrid Rule Violation -> VerdictReject (User rejection) + invalidSrcBytes, err := os.ReadFile("../../tests/checker-corpus/08-a3-map-slot-invalidated.hy") + if err != nil { + t.Fatalf("failed to read invalid corpus file: %v", err) + } + resInvalid := engine.CheckSource(ctx, string(invalidSrcBytes)) + if err := resInvalid.CheckInvariant(); err != nil { + t.Fatal(err) + } + if resInvalid.Verdict != VerdictReject { + t.Fatalf("Expected VerdictReject, got %v", resInvalid.Verdict) + } + if len(resInvalid.Diagnostics) == 0 { + t.Fatalf("Expected diagnostics on reject, got 0") + } + t.Logf("[PASS] Rule violation -> %s (%d diagnostics)", resInvalid.Verdict, len(resInvalid.Diagnostics)) + + // 2b. Invalid UTF-8 is a user-source rejection, not an infrastructure failure. + t.Run("InvalidUTF8_IsReject", func(t *testing.T) { + res := engine.CheckSource(ctx, string([]byte{0xff, 0xfe})) + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictReject { + t.Fatalf("Expected VerdictReject for invalid UTF-8, got %v (%v)", res.Verdict, res.Failure) + } + if len(res.Diagnostics) != 1 || res.Diagnostics[0].Code != "HYB-E0002" { + t.Fatalf("Expected one HYB-E0002 diagnostic, got %+v", res.Diagnostics) + } + }) + + // 3. Fault Injection 1: WASM Trap / Panic during execution + t.Run("Fault_WasmTrap", func(t *testing.T) { + mock := &MockWasmABI{ + Memory: make([]byte, 1024), + CheckFn: func(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + return 0, errors.New("wasm trap: unreachable instruction executed") + }, + } + + res := CheckSourceWithABI(ctx, mock, "struct test {}") + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictAnalysisFailure { + t.Fatalf("Expected VerdictAnalysisFailure on WASM trap, got %v", res.Verdict) + } + t.Logf("[PASS] Fault Injection 1 (WASM Trap) -> %s: %v", res.Verdict, res.Failure) + }) + + // 4. Fault Injection 2: Malformed JSON diagnostic envelope + t.Run("Fault_MalformedJSON", func(t *testing.T) { + corrupt := []byte(`{"diagnostics": [{"code": "HYB-E3003", "severity": TRUNCATED_MALFORMED...`) + mock := &MockWasmABI{ + Memory: make([]byte, 1024), + CheckFn: func(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + return (uint64(64) << 32) | uint64(len(corrupt)), nil + }, + } + copy(mock.Memory[64:], corrupt) + + res := CheckSourceWithABI(ctx, mock, "struct test {}") + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictAnalysisFailure { + t.Fatalf("Expected VerdictAnalysisFailure on malformed JSON, got %v", res.Verdict) + } + t.Logf("[PASS] Fault Injection 2 (Malformed JSON) -> %s: %v", res.Verdict, res.Failure) + }) + + // 5. Fault Injection 3: Invalid pointer / memory out-of-bounds + t.Run("Fault_InvalidPointerBounds", func(t *testing.T) { + mock := &MockWasmABI{ + Memory: make([]byte, 1024), + CheckFn: func(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + // 50000 exceeds 1024-byte memory + return (uint64(50000) << 32) | uint64(100), nil + }, + } + + res := CheckSourceWithABI(ctx, mock, "struct test {}") + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictAnalysisFailure { + t.Fatalf("Expected VerdictAnalysisFailure on invalid bounds, got %v", res.Verdict) + } + t.Logf("[PASS] Fault Injection 3 (Invalid Memory Bounds) -> %s: %v", res.Verdict, res.Failure) + }) + + // 6. Fault Injection 4: Host allocation failure + t.Run("Fault_AllocFailure", func(t *testing.T) { + mock := &MockWasmABI{ + Memory: make([]byte, 1024), + AllocFn: func(ctx context.Context, byteLen uint64) (uint32, error) { + return 0, errors.New("out of memory in guest wasm allocator") + }, + } + + res := CheckSourceWithABI(ctx, mock, "struct test {}") + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictAnalysisFailure { + t.Fatalf("Expected VerdictAnalysisFailure on alloc failure, got %v", res.Verdict) + } + t.Logf("[PASS] Fault Injection 4 (Alloc Failure) -> %s: %v", res.Verdict, res.Failure) + }) + + // 7. Fault Injection 5: Execution Timeout + t.Run("Fault_Timeout", func(t *testing.T) { + timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancel() + time.Sleep(15 * time.Millisecond) // expire context + + res := engine.CheckSource(timeoutCtx, "struct test {}") + if err := res.CheckInvariant(); err != nil { + t.Fatal(err) + } + if res.Verdict != VerdictAnalysisFailure { + t.Fatalf("Expected VerdictAnalysisFailure on timeout, got %v", res.Verdict) + } + t.Logf("[PASS] Fault Injection 5 (Execution Timeout) -> %s: %v", res.Verdict, res.Failure) + }) + + // 8. Successful calls release both the input and diagnostic output buffers. + t.Run("GuestBuffersAreFreed", func(t *testing.T) { + envelope := []byte(`{"diagnostics":[],"summary":{"file":"input.hy","verdict":"accept","errors":0,"warnings":0},"version":1}`) + var frees [][2]uint64 + mock := &MockWasmABI{ + Memory: make([]byte, 1024), + AllocFn: func(ctx context.Context, byteLen uint64) (uint32, error) { + return 8, nil + }, + CheckFn: func(ctx context.Context, ptr, byteLen uint64) (uint64, error) { + return (uint64(128) << 32) | uint64(len(envelope)), nil + }, + FreeFn: func(ctx context.Context, ptr, byteLen uint64) error { + frees = append(frees, [2]uint64{ptr, byteLen}) + return nil + }, + } + copy(mock.Memory[128:], envelope) + + res := CheckSourceWithABI(ctx, mock, "package p") + if res.Verdict != VerdictAccept { + t.Fatalf("Expected VerdictAccept, got %v (%v)", res.Verdict, res.Failure) + } + if len(frees) != 2 { + t.Fatalf("Expected source and output frees, got %v", frees) + } + if frees[0][0] != 8 || frees[1][0] != 128 { + t.Fatalf("Unexpected freed pointers: %v", frees) + } + }) +} diff --git a/spikes/w012-analyzer/go.mod b/spikes/w012-analyzer/go.mod new file mode 100644 index 00000000..99bc1c9e --- /dev/null +++ b/spikes/w012-analyzer/go.mod @@ -0,0 +1,13 @@ +module github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer + +go 1.26.9 + +require ( + github.com/tetratelabs/wazero v1.9.0 + golang.org/x/tools v0.52.0 +) + +require ( + golang.org/x/mod v0.42.0 // indirect + golang.org/x/sync v0.24.0 // indirect +) diff --git a/spikes/w012-analyzer/go.sum b/spikes/w012-analyzer/go.sum new file mode 100644 index 00000000..d4079564 --- /dev/null +++ b/spikes/w012-analyzer/go.sum @@ -0,0 +1,10 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= diff --git a/spikes/w012-analyzer/go_only_test.go b/spikes/w012-analyzer/go_only_test.go new file mode 100644 index 00000000..a8b8b79a --- /dev/null +++ b/spikes/w012-analyzer/go_only_test.go @@ -0,0 +1,30 @@ +package analyzer + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "path/filepath" + "testing" + + "golang.org/x/tools/go/analysis" +) + +// An unrelated Go package must not depend on the Hybrid WASM artifact. +func TestGoOnlyPackageRequiresNoWasm(t *testing.T) { + dir := t.TempDir() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filepath.Join(dir, "only.go"), "package only\n", 0) + if err != nil { + t.Fatal(err) + } + pass := &analysis.Pass{ + Analyzer: Analyzer, Pkg: types.NewPackage("only", "only"), + Fset: fset, Files: []*ast.File{file}, + Report: func(analysis.Diagnostic) { t.Error("go-only package should not report Hybrid diagnostics") }, + } + if _, err := run(pass); err != nil { + t.Fatalf("go-only package failed: %v", err) + } +} diff --git a/spikes/w012-analyzer/native_parity_test.go b/spikes/w012-analyzer/native_parity_test.go new file mode 100644 index 00000000..a4b1f052 --- /dev/null +++ b/spikes/w012-analyzer/native_parity_test.go @@ -0,0 +1,122 @@ +package analyzer + +import ( + "context" + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "reflect" + "testing" + + "golang.org/x/tools/go/analysis" +) + +// This covers the fields modeled by the Go wrapper. The Node parity harness +// separately compares complete native/WASM JSON envelopes, including fields +// such as notes and fixits that CheckOutput does not model. +func TestNativeCorpusModeledFieldParity(t *testing.T) { + nativePath := os.Getenv("HYBRID_NATIVE_PATH") + if nativePath == "" { + nativePath = filepath.Join("..", "..", "compiler", "target", "release", "hyb") + } + engine, err := GetEngine("") + if err != nil { + t.Fatal(err) + } + files, err := filepath.Glob("../../tests/checker-corpus/*.hy") + if err != nil || len(files) == 0 { + t.Fatalf("empty or unreadable corpus: %v", err) + } + passed := 0 + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + path, err := filepath.Abs(file) + if err != nil { + t.Fatal(err) + } + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + actual, err := engine.Check(string(source)) + if err != nil { + t.Fatal(err) + } + raw, err := exec.Command(nativePath, "check", path, "--json").Output() + var exitErr *exec.ExitError + if err != nil && (!errors.As(err, &exitErr) || exitErr.ExitCode() != 1) { + t.Fatalf("native CLI failed: %v", err) + } + var expected CheckOutput + if err := json.Unmarshal(raw, &expected); err != nil { + t.Fatal(err) + } + if (err == nil) != (expected.Summary.Verdict == "accept") { + t.Fatal("native exit status disagrees with verdict") + } + expected.Summary.File = "input.hy" + // nil and empty slices represent the same absence of diagnostics. + if len(expected.Diagnostics) == 0 { + expected.Diagnostics = nil + } + if len(actual.Diagnostics) == 0 { + actual.Diagnostics = nil + } + if !reflect.DeepEqual(actual, &expected) { + t.Fatalf("Go wrapper mismatch\nactual: %+v\nexpected: %+v", actual, expected) + } + passed++ + }) + } + t.Logf("Go wrapper/native modeled-field parity: %d/%d", passed, len(files)) +} + +func TestAcceptedWarningsPreserved(t *testing.T) { + payload := `{"version":1,"diagnostics":[{"code":"HYB-E8001","severity":"warning","message":"managed share","span":{"start":{"line":1,"col":1},"end":{"line":1,"col":2}}}],"summary":{"file":"input.hy","verdict":"accept","errors":0,"warnings":1}}` + abi := &MockWasmABI{Memory: make([]byte, 4096), CheckFn: func(context.Context, uint64, uint64) (uint64, error) { + return uint64(1024)<<32 | uint64(len(payload)), nil + }} + copy(abi.Memory[1024:], payload) + result := CheckSourceWithABI(context.Background(), abi, "source") + if result.Verdict != VerdictAccept || len(result.Diagnostics) != 1 || result.Diagnostics[0].Severity != "warning" || result.Summary.Errors != 0 || result.Summary.Warnings != 1 { + t.Fatalf("accepted warning lost: %+v", result) + } + if err := result.CheckInvariant(); err != nil { + t.Fatal(err) + } + result.Diagnostics[0].Severity = "error" + if err := result.CheckInvariant(); err == nil { + t.Fatal("error diagnostic accepted") + } +} + +func TestAnalyzerKeepsManagedWarningsNonfatal(t *testing.T) { + source, err := os.ReadFile("../../tests/checker-corpus/02-shared-map-managed.hy") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + hyFile := filepath.Join(dir, "managed.hy") + if err := os.WriteFile(hyFile, source, 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filepath.Join(dir, "sample.go"), "package sample", 0) + if err != nil { + t.Fatal(err) + } + var diagnostics []analysis.Diagnostic + pass := &analysis.Pass{Analyzer: Analyzer, Fset: fset, Files: []*ast.File{file}, Pkg: types.NewPackage("sample", "sample"), ReadFile: os.ReadFile, Report: func(d analysis.Diagnostic) { diagnostics = append(diagnostics, d) }} + if _, err := run(pass); err != nil { + t.Fatal(err) + } + if len(diagnostics) != 0 { + t.Fatalf("accepted managed warnings must not fail go vet: %+v", diagnostics) + } +} diff --git a/spikes/w012-analyzer/readfile_contract_test.go b/spikes/w012-analyzer/readfile_contract_test.go new file mode 100644 index 00000000..0bec99c5 --- /dev/null +++ b/spikes/w012-analyzer/readfile_contract_test.go @@ -0,0 +1,49 @@ +package analyzer + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "testing" + + "golang.org/x/tools/go/analysis" +) + +// The documented Pass.ReadFile contract allows only Files, OtherFiles and +// IgnoredFiles. Stock go vet does not declare .hy siblings; our explicitly +// on-disk adapter must never probe such paths through a strict driver. +func TestScannedSiblingDoesNotViolateReadFileContract(t *testing.T) { + dir := t.TempDir() + hy := filepath.Join(dir, "source.hy") + content := "package sample\nfn broken() { let x = 7; let y = move x; let z = x }\n" + if err := os.WriteFile(hy, []byte(content), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + goFile, err := parser.ParseFile(fset, filepath.Join(dir, "anchor.go"), "package sample\n", 0) + if err != nil { + t.Fatal(err) + } + count := 0 + pass := &analysis.Pass{ + Analyzer: Analyzer, + Pkg: types.NewPackage("sample", "sample"), + Fset: fset, + Files: []*ast.File{goFile}, + ReadFile: func(name string) ([]byte, error) { + t.Errorf("undeclared Hybrid sibling %q must not be passed to ReadFile", name) + return nil, errors.New("file is not declared to analyzer") + }, + Report: func(analysis.Diagnostic) { count++ }, + } + if _, err := run(pass); err != nil { + t.Fatal(err) + } + if count == 0 { + t.Fatal("on-disk Hybrid source was skipped instead of checked") + } +} diff --git a/spikes/w012-analyzer/snapshot.go b/spikes/w012-analyzer/snapshot.go new file mode 100644 index 00000000..cfe7b714 --- /dev/null +++ b/spikes/w012-analyzer/snapshot.go @@ -0,0 +1,133 @@ +package analyzer + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" +) + +const hybridSnapshotEnv = "HYBRID_W012_SNAPSHOT" + +// HybridSnapshot is the immutable input contract used by the supported +// `hybvet vet` wrapper. Keys are absolute cleaned .hy paths and values are +// SHA-256 digests of the bytes captured before go vet starts. +type HybridSnapshot struct { + Version int `json:"version"` + Root string `json:"root"` + Files map[string]string `json:"files"` +} + +func CaptureHybridSnapshot(root string) (*HybridSnapshot, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, err + } + absRoot = filepath.Clean(absRoot) + s := &HybridSnapshot{Version: 1, Root: absRoot, Files: make(map[string]string)} + + err = filepath.WalkDir(absRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + switch d.Name() { + case ".git", "target", "node_modules": + if path != absRoot { + return filepath.SkipDir + } + } + return nil + } + if filepath.Ext(d.Name()) != ".hy" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + abs, err := filepath.Abs(path) + if err != nil { + return err + } + s.Files[filepath.Clean(abs)] = digestBytes(data) + return nil + }) + if err != nil { + return nil, err + } + return s, nil +} + +func (s *HybridSnapshot) Encode() (string, error) { + if s == nil || s.Version != 1 || s.Root == "" || s.Files == nil { + return "", errors.New("invalid hybrid snapshot") + } + data, err := json.Marshal(s) + if err != nil { + return "", err + } + return string(data), nil +} + +func hybridSnapshotFromEnv() (*HybridSnapshot, error) { + raw := os.Getenv(hybridSnapshotEnv) + if raw == "" { + return nil, nil + } + var s HybridSnapshot + if err := json.Unmarshal([]byte(raw), &s); err != nil { + return nil, fmt.Errorf("decode %s: %w", hybridSnapshotEnv, err) + } + if s.Version != 1 || s.Root == "" || s.Files == nil { + return nil, fmt.Errorf("unsupported or incomplete snapshot") + } + return &s, nil +} + +func (s *HybridSnapshot) VerifyFile(path string, data []byte) error { + if s == nil { + return nil + } + abs, err := filepath.Abs(path) + if err != nil { + return err + } + abs = filepath.Clean(abs) + want, ok := s.Files[abs] + if !ok { + return fmt.Errorf("%s was not present in the pre-analysis .hy snapshot", abs) + } + got := digestBytes(data) + if got != want { + return fmt.Errorf("%s changed after the pre-analysis .hy snapshot", abs) + } + return nil +} + +// VerifyCurrent detects mutation, deletion, or addition anywhere under the +// captured workspace root. This is intentionally conservative: a concurrent +// .hy edit makes the wrapper fail closed rather than certify mixed inputs. +func (s *HybridSnapshot) VerifyCurrent() error { + if s == nil { + return errors.New("nil hybrid snapshot") + } + current, err := CaptureHybridSnapshot(s.Root) + if err != nil { + return err + } + if !reflect.DeepEqual(s.Files, current.Files) { + return errors.New(".hy workspace changed while analysis was running") + } + return nil +} + +func digestBytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/spikes/w012-analyzer/testdata/src/samplepkg/bad.hy b/spikes/w012-analyzer/testdata/src/samplepkg/bad.hy new file mode 100644 index 00000000..edb43180 --- /dev/null +++ b/spikes/w012-analyzer/testdata/src/samplepkg/bad.hy @@ -0,0 +1,8 @@ +// EXPECTED-VERDICT: reject HYB-E3003 +package samplepkg + +fn bad_use_after_move() { + let x = 42 + let y = move x + let z = x +} diff --git a/spikes/w012-analyzer/testdata/src/samplepkg/sample.go b/spikes/w012-analyzer/testdata/src/samplepkg/sample.go new file mode 100644 index 00000000..c3934eed --- /dev/null +++ b/spikes/w012-analyzer/testdata/src/samplepkg/sample.go @@ -0,0 +1,6 @@ +package samplepkg + +// Sample Go file in package +func Add(a, b int) int { + return a + b +} diff --git a/spikes/w012-analyzer/testpkg/foo.go b/spikes/w012-analyzer/testpkg/foo.go new file mode 100644 index 00000000..0cd2ac60 --- /dev/null +++ b/spikes/w012-analyzer/testpkg/foo.go @@ -0,0 +1,5 @@ +package testpkg + +func Hello() string { + return "hello" +} diff --git a/spikes/w012-analyzer/testpkg/sibling.hy b/spikes/w012-analyzer/testpkg/sibling.hy new file mode 100644 index 00000000..06035e2b --- /dev/null +++ b/spikes/w012-analyzer/testpkg/sibling.hy @@ -0,0 +1,8 @@ +// EXPECTED-VERDICT: reject HYB-E3003 +package testpkg + +fn bad_borrow() { + let x = 42 + let y = move x + let z = x +} diff --git a/spikes/w012-analyzer/vet.go b/spikes/w012-analyzer/vet.go new file mode 100644 index 00000000..f1707555 --- /dev/null +++ b/spikes/w012-analyzer/vet.go @@ -0,0 +1,71 @@ +package analyzer + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" +) + +// RunVet is the supported cache-safe entry point for the W-012 go/analysis +// spike. Stock `go vet -vettool=...` does not know that arbitrary sibling .hy +// files are analyzer inputs, so its action cache cannot key on them. The +// wrapper therefore forces a fresh vet action (-a) and supplies an immutable +// .hy snapshot that the analyzer validates while reading each source. +func RunVet(ctx context.Context, args []string, stdout, stderr io.Writer) error { + // The wrapper owns both the analyzer binary and Go vet's forced rebuild. + // Go accepts -flag and --flag (also -flag=value). A caller-supplied + // override later in the argv would defeat the .hy snapshot/cache contract. + for _, arg := range args { + name, _, _ := strings.Cut(arg, "=") + switch name { + case "-vettool", "--vettool": + return errorsVet("do not pass -vettool to hybvet vet; the wrapper owns the analyzer binary") + case "-a", "--a": + return errorsVet("do not override -a in hybvet vet; the wrapper requires an uncached vet action") + } + } + + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("resolve workspace root: %w", err) + } + snapshot, err := CaptureHybridSnapshot(cwd) + if err != nil { + return fmt.Errorf("capture .hy snapshot: %w", err) + } + encoded, err := snapshot.Encode() + if err != nil { + return fmt.Errorf("encode .hy snapshot: %w", err) + } + + exe, err := os.Executable() + if err != nil { + return fmt.Errorf("resolve hybvet executable: %w", err) + } + + vetArgs := []string{"vet", "-a", "-vettool=" + exe} + vetArgs = append(vetArgs, args...) + cmd := exec.CommandContext(ctx, "go", vetArgs...) + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Env = append(os.Environ(), hybridSnapshotEnv+"="+encoded) + + runErr := cmd.Run() + verifyErr := snapshot.VerifyCurrent() + if verifyErr != nil { + return fmt.Errorf("hybrid input snapshot invalidated: %w", verifyErr) + } + if runErr != nil { + return runErr + } + return nil +} + +type vetUsageError string + +func (e vetUsageError) Error() string { return string(e) } + +func errorsVet(msg string) error { return vetUsageError(msg) } diff --git a/spikes/w012-analyzer/vet_flag_guard_test.go b/spikes/w012-analyzer/vet_flag_guard_test.go new file mode 100644 index 00000000..e9ef661b --- /dev/null +++ b/spikes/w012-analyzer/vet_flag_guard_test.go @@ -0,0 +1,36 @@ +package analyzer + +import ( + "context" + "io" + "strings" + "testing" +) + +func TestRunVetRejectsOwnedFlagOverrides(t *testing.T) { + tests := []struct { + name string + args []string + want string + }{ + {"vettool short equals", []string{"-vettool=/tmp/other-analyzer", "./..."}, "owns the analyzer binary"}, + {"vettool long equals", []string{"--vettool=/tmp/other-analyzer", "./..."}, "owns the analyzer binary"}, + {"vettool short separated", []string{"-vettool", "/tmp/other-analyzer", "./..."}, "owns the analyzer binary"}, + {"vettool long separated", []string{"--vettool", "/tmp/other-analyzer", "./..."}, "owns the analyzer binary"}, + {"cache rebuild short false", []string{"-a=false", "./..."}, "uncached vet action"}, + {"cache rebuild long false", []string{"--a=false", "./..."}, "uncached vet action"}, + {"cache rebuild short zero", []string{"-a=0", "./..."}, "uncached vet action"}, + {"cache rebuild long zero", []string{"--a=0", "./..."}, "uncached vet action"}, + {"cache rebuild short alone", []string{"-a", "./..."}, "uncached vet action"}, + {"cache rebuild long alone", []string{"--a", "./..."}, "uncached vet action"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RunVet(context.Background(), tt.args, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("RunVet(%q) = %v; want an immediate owned-flag rejection containing %q", tt.args, err, tt.want) + } + }) + } +} diff --git a/spikes/w012-inprocess/RESULTS.md b/spikes/w012-inprocess/RESULTS.md new file mode 100644 index 00000000..3103d00f --- /dev/null +++ b/spikes/w012-inprocess/RESULTS.md @@ -0,0 +1,167 @@ +> Historical spike record. For the current corrected Go-wrapper checks and freshly measured latency, use [validation-2026-10-18](validation-2026-10-18/README.md). The old speedup figures below do not describe the current analyzer path. + +# Confirmatory Replication Protocol & Results Ledger: W-012 In-Process Analyzer Spike + +## 0. Preregistration Notice & Protocol Status +> [!IMPORTANT] +> **Exploratory measurements were already observed before this preregistration.** +> +> In accordance with Go-Hybrid spike measurement protocol, this document registers a formal **confirmatory replication**. The hypotheses, targets, sample counts, and execution commands below were locked in at commit `58ad41ae78f240680525a634aad8309fc9ad7281` prior to executing the 100-iteration confirmatory benchmark and test ledger. + +- **Baseline Upstream Commit**: `e2a5f7d5eb90becb210df274765adfa50c99b21b` (`upstream/main`) +- **Preregistration Commit SHA**: `58ad41ae78f240680525a634aad8309fc9ad7281` +- **Target Branch**: `spike/w012-inprocess-analyzer` +- **Protocol Class**: Confirmatory Replication (N=100 iterations per benchmark condition) +- **Primary Representative Metric**: Median (p50) and 90th percentile (p90) latency; mean reported as secondary outlier-sensitive metric. + +--- + +## 1. Preregistered Targets & Validation Criteria + +| Target # | Target Scope | Target Hypothesis | Pass Criterion | Confirmatory Result | +|---|---|---|---|---| +| **Target 1** | Corpus Parity | 55/55 fixtures match native CLI as normalized parsed JSON envelopes (input filenames normalized only) | 100.0% parity (0 diffs) | **CONFIRMED (55/55, 100.0%)** | +| **Target 2** | Native CLI Subprocess | Original 15–40 ms assumption overstated; p50 < 8 ms | Documented baseline | **FALSIFIED / OVERSTATED** (p50: 5.038 ms) | +| **Target 3** | Steady-State WASM Speedup | Warm in-process check p50 < 1.0 ms; speedup $\ge 10\times$ | p50 ratio $\ge 10\times$ | **CONFIRMED** (p50: 0.269 ms, **18.7x speedup**) | +| **Target 4** | Filesystem Cache Break-Even | FS cache reduces compile by $>90\%$; break-even $\le 5$ checks | $N_{\text{break-even}} \le 5$ | **CONFIRMED** (Compile: 12.919 ms, **3 checks**) | +| **Target 5** | Failure-Domain Separation | Failures fail closed to `AnalysisFailure`, disjoint from Reject | 100% pass on 5 fault injections | **CONFIRMED** (5/5 injected faults caught) | +| **Target 6** | On-Disk Analyzer & Position | On-disk discovery works via `os.ReadDir`; exact positions via `Fset.AddFile` | Exact `line:col` in `go vet` | **CONFIRMED** (`sibling.hy:7:13`) | + +--- + +## 2. Confirmatory Test Results (Executed at Preregistered Commit) + +### 2.1 Corpus Parity Verification (`node test_wasm_parity.js`) +```text +Loaded WASM: 925098 bytes (903.4 KB) +WASM instantiation time: 2.55 ms +Verifying all 55 corpus files against WASM... +Corpus parity: 55/55 (100.0%) +WASM Check Latency per file: + mean: 0.823 ms + p50: 0.528 ms + p90: 1.298 ms + p99: 10.456 ms +``` +- **Verdict**: **PASSED (100.0% parity)**. + +### 2.2 Analyzer Contract & Source Position (`go test -v -run TestAnalyzerEmpiricalPassContext`) +```text +=== RUN TestAnalyzerEmpiricalPassContext + analyzer_test.go:62: A. pass.Files count: 1 (Only .go files parsed into Go AST) + analyzer_test.go:66: B1. pass.OtherFiles: [] (len=0) + analyzer_test.go:67: B2. pass.IgnoredFiles: [] (len=0) + analyzer_test.go:73: C. Discovered .hy files via DirScan: [DirScan:bad.hy] + analyzer_test.go:79: D. PkgPath: github.com/limitlessprojects/go-hybrid/spikes/w012-analyzer/testdata/src/samplepkg, HasTypesInfo: true + analyzer_test.go:82: E. Reported 3 diagnostic(s) from sibling .hy analysis: + analyzer_test.go:85: - [hybrid] Pos=193 (.../bad.hy:6:5): [HYB-E1002] `y` is declared and never used + analyzer_test.go:85: - [hybrid] Pos=212 (.../bad.hy:7:5): [HYB-E1002] `z` is declared and never used + analyzer_test.go:85: - [hybrid] Pos=220 (.../bad.hy:7:13): [HYB-E3003] `x` is read after ownership was moved away +--- PASS: TestAnalyzerEmpiricalPassContext (0.44s) +PASS +``` + +### 2.3 On-Disk `go vet -vettool` Execution +```text +$ go vet -vettool=./cmd/hybvet/hybvet ./testpkg +testpkg/sibling.hy:6:5: [HYB-E1002] `y` is declared and never used — Go refuses to build the emitted `y := ...` (declared and not used) (HYB-E1002) +testpkg/sibling.hy:7:5: [HYB-E1002] `z` is declared and never used — Go refuses to build the emitted `z := ...` (declared and not used) (HYB-E1002) +testpkg/sibling.hy:7:13: [HYB-E3003] `x` is read after ownership was moved away +exit status 1 +``` +- **Verdict**: **Exact on-disk .hy positions: CONFIRMED**. + +### 2.4 Failure-Domain Separation & Fault Injection (`go test -v -run TestFailureDomainSeparation`) +```text +=== RUN TestFailureDomainSeparation + fault_injection_test.go:87: [PASS] Valid program -> Accept + fault_injection_test.go:104: [PASS] Rule violation -> Reject (1 diagnostics) +=== RUN TestFailureDomainSeparation/Fault_WasmTrap + fault_injection_test.go:122: [PASS] Fault Injection 1 (WASM Trap) -> AnalysisFailure: infrastructure error: WASM trap/panic during execution: wasm trap: unreachable instruction executed +=== RUN TestFailureDomainSeparation/Fault_MalformedJSON + fault_injection_test.go:143: [PASS] Fault Injection 2 (Malformed JSON) -> AnalysisFailure: infrastructure error: malformed diagnostic envelope: invalid character 'T' looking for beginning of value +=== RUN TestFailureDomainSeparation/Fault_InvalidPointerBounds + fault_injection_test.go:163: [PASS] Fault Injection 3 (Invalid Memory Bounds) -> AnalysisFailure: infrastructure error: out-of-bounds guest memory read: ptr=50000, len=100 +=== RUN TestFailureDomainSeparation/Fault_AllocFailure + fault_injection_test.go:182: [PASS] Fault Injection 4 (Alloc Failure) -> AnalysisFailure: infrastructure error: guest memory alloc failed: out of memory in guest wasm allocator +=== RUN TestFailureDomainSeparation/Fault_Timeout + fault_injection_test.go:198: [PASS] Fault Injection 5 (Execution Timeout) -> AnalysisFailure: infrastructure error: context timeout before instantiation: context deadline exceeded +--- PASS: TestFailureDomainSeparation (0.37s) +PASS +``` + +### 2.5 Confirmatory 100-Iteration Benchmark Raw Output +```text +================================================================================ +W-012 HARDENING BENCHMARK: Wazero vs Native Subprocess (100 iterations/case) +================================================================================ +[Baseline] Native CLI Subprocess: min=4.721ms, p50=5.038ms, p90=5.476ms, p99=5.935ms, mean=5.122ms +[CASE A] Fresh Runtime (No Cache): total_mean=354.967ms (compile=353.878ms, inst=0.216ms, check=0.772ms) + Compile Stats: min=339.287ms, p50=346.119ms, p90=371.130ms, p99=417.249ms, mean=353.878ms +[CASE B] Reused CompiledModule: total_mean=0.723ms (inst=0.078ms, check=0.640ms) + Total Stats: min=0.513ms, p50=0.694ms, p90=0.947ms, p99=1.392ms, mean=0.723ms +[CASE C] New Runtime (In-Memory Cache): total_mean=14.437ms (compile=13.384ms, inst=0.237ms, check=0.803ms) + Compile Stats: min=9.583ms, p50=9.887ms, p90=10.356ms, p99=21.417ms, mean=13.384ms +[CASE D] New Runtime (Filesystem Cache): total_mean=14.356ms (compile=12.974ms, inst=0.172ms, check=0.923ms) + Compile Stats: min=12.337ms, p50=12.919ms, p90=13.400ms, p99=14.090ms, mean=12.974ms +================================================================================ +BREAK-EVEN QUANTITATIVE ANALYSIS (100 ITERATIONS CONFIRMATORY): + [Primary: p50-based Metrics] + • Native CLI Subprocess p50: 5.038 ms + • Pure Warm WASM In-Process p50: 0.269 ms + • Representative Steady-State Speedup (p50): 18.7x + • Latency Advantage per Check (delta_T p50): 4.769 ms + • Cold Compile Overhead (T_compile_cold p50):346.119 ms + -> Break-Even (Cold / No Cache, p50): N_break_even ≈ 72.6 checks (ceil: 73 checks) + • FS Cached Compile (T_compile_fs p50): 12.919 ms + -> Break-Even (Filesystem Cache, p50): N_break_even ≈ 2.7 checks (ceil: 3 checks) + + [Secondary: Outlier-Sensitive Mean Metrics] + • Native CLI Subprocess Mean: 5.122 ms + • Pure Warm WASM In-Process Mean: 0.278 ms + • Ratio of Means (Mean Speedup): 18.4x + -> Break-Even (Cold / No Cache, Mean): N_break_even ≈ 73.1 checks (ceil: 74 checks) + -> Break-Even (Filesystem Cache, Mean): N_break_even ≈ 2.7 checks (ceil: 3 checks) +================================================================================ +``` + +--- + +## 3. Protocol Verification Sign-Off +All preregistered targets have completed confirmatory execution: +1. Corpus parity: **100.0% confirmed** (55/55 fixtures matching as normalized parsed JSON envelopes (input filenames normalized only)). +2. Baseline CLI latency: **5.038 ms p50** (Original 15–40ms normal CLI latency estimate was overstated on this host). +3. Representative steady-state speedup: **≈ 17–19x** (18.7x observed at p50: 5.038 ms vs 0.269 ms). +4. Break-even threshold: **73 checks (cold uncached)** vs **3 checks (filesystem-cached)** by p50. +5. Failure-domain isolation: **100% verified across all 5 fault-injection scenarios** (fails closed to `AnalysisFailure`). +6. On-disk Analyzer execution: **Exact source line/col positions confirmed** (`file.hy:line:col` emitted under `go vet -vettool`). Sibling `.hy` discovery relies on host filesystem scanning (`os.ReadDir`). Current spike proves on-disk go-vet/singlechecker integration. It does NOT prove unsaved editor-buffer integration. + + +## Closeout validation — 2026-10-18 + +The earlier corpus script checked expected fixture verdicts and the first error rule; it did **not** establish complete native/WASM diagnostic equality. The updated script now invokes the native CLI for every fixture and compares complete parsed JSON envelopes, normalizing only the input filename (including its occurrences in diagnostic notes). This is semantic JSON equality, not raw byte equality. + +Executed against the existing native/WASM artifacts from the reviewed baseline: + +```text +Corpus fixture agreement: 55/55 +Native/WASM normalized envelope parity: 55/55 +``` + +`go test ./...` passes for the analyzer module, including ten malformed-envelope cases, acceptance of additive fields, the original five boundary fault injections, a real minimal guest trap, and a real infinite-loop guest interrupted by a 50 ms context deadline. The allocator-failure test uses a mock error; it does not exhaust a real guest heap. The malformed-output tests now include valid JSON with missing/unknown verdicts and contradictory counts. The Analyzer invokes the same validated boundary as the failure harness and propagates infrastructure errors rather than reporting a code diagnostic. Wazero context cancellation is enabled explicitly. + +A rebuilt singlechecker driver reports the expected `sibling.hy:6:5`, `:7:5`, and `:7:13` diagnostics under `go vet -vettool`; exit 1 is the expected rejection of the invalid fixture. + +Reproduction (from repository root; build both artifacts from the reviewed baseline first): + +```sh +export HYBRID_WASM_PATH="$PWD/compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm" +export HYBRID_NATIVE_PATH="$PWD/compiler/target/release/hyb" +node test_wasm_parity.js +cd spikes/w012-analyzer +go test ./... +go build -o /tmp/hybvet ./cmd/hybvet +go vet -vettool=/tmp/hybvet ./testpkg +``` + +Benchmark provenance: all canonical 100-run timing figures above are the recorded snapshot at `b3d73c2c3f211f69e856abd089adc4cad9a21f50`, before this closeout added envelope validation. They were not rerun or relabeled as measurements of the changed wrapper. The added validation cost and end-to-end editor latency remain unmeasured. The parity script's per-file timings are separate and are not the canonical benchmark. Long-duration memory behavior remains unproven. Stock-gopls plugins, unsaved `.hy` buffers, and a completed standalone LSP are not demonstrated. diff --git a/spikes/w012-inprocess/WORKSPACE_EDIT_BENCHMARK.md b/spikes/w012-inprocess/WORKSPACE_EDIT_BENCHMARK.md new file mode 100644 index 00000000..10905e7a --- /dev/null +++ b/spikes/w012-inprocess/WORKSPACE_EDIT_BENCHMARK.md @@ -0,0 +1,56 @@ +# W-012: 25-file, 100-edit request-path comparison + +Purpose: independently measure the corrected public Go/Wazero request path against the native hyb CLI on the *same machine*, using real file modifications and the existing pre-registered corpus oracle. + +**This is not a gopls or LSP benchmark.** It measures an on-disk single-file edit-to-check request proxy. It cannot establish an end-to-end keystroke-to-diagnostic time or successful integration of unsaved Hybrid buffers into stock gopls. + +## Method + +1. Create a temporary Go module with one ordinary Go anchor and 25 .hy source files copied from one pre-registered checker-corpus fixture. +2. Warm both check paths equally. Modify a different .hy file for every edit (round-robin through all 25), with 100 sequential, constant-width, non-cumulative edits. Start the request timer only after writing modified bytes. +3. Evaluate the changed file with (A) native hyb check PATH --json (new subprocess and JSON decoder each request), and (B) public Engine.Check(source) (in-process Wazero with already-compiled module and disposable guest per request). Alternate request order on odd and even edits. +4. Fail if the native exit code is inconsistent with a valid parsed envelope; if the registered expected verdict fails; or if modeled diagnostic codes, counts, severities, Tier-0 rules or source spans differ. Timing never overrides correctness. +5. Record all 200 durations plus p50/p90/p95/p99/mean, separate cold WASM compilation/init, SHA-256 hashes and sizes of the native and WASM artifacts, and basic host metadata. +6. CI saves the raw JSON as an upload-artifact attachment for the exact workflow run, with 30-day retention for request samples and cross-process cache timings. Performance figures must remain associated with their exact tested commit and host. + +## Reproduction + +From the root of the repository, first build both artifacts with the toolchain versions pinned in tools/env.sh: + + (cd compiler && cargo build --locked --release -p hyb) + (cd compiler && cargo build --locked --release -p hy-wasm --target wasm32-unknown-unknown) + (cd spikes/w012-analyzer && go run ./cmd/editbench -root ../.. -files 25 -edits 100 -warmup 10 -output /tmp/w012-edit-benchmark.json) + +## Interpretation and falsification + +The benchmark models only the request made *after* an edit. The native path must create a subprocess and read the file; Engine.Check receives source bytes in-process as an embedding host could. The paths are not identical user interfaces and no complete workspace reanalysis is performed. Cold compile time is isolated, never hidden in or misrepresented as warm request latency. + +No assertion is made about editor startup, stock gopls, LSP, live type bridging, Docker image size, binary release packaging, or production RSS/long-duration memory use. The original M2 goal requires additional, independently verifiable experiments for these costs. The public Engine.Check may remain slower than the native subprocess; if so, report that falsification instead of substituting a different (faster) reused-instance prototype. + + +## Observed on hosted CI — 2026-10-19 JST + +All runs built their own checker artifacts from the tested pull-request checkout; the raw samples are attached to each linked workflow run. These are different runs on hosted Linux workers, not a performance guarantee across machines. + +| Validated checkout / run | Native p50 | Public Engine.Check p50 | Sequential reused-guest p50 | Interpretation | +| --- | ---: | ---: | ---: | --- | +| 7d0a942 / [39172638567](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39172638567) | 1.466 ms | 2.902 ms | not measured | Public WASM path slower | +| 9c04ee6 / [39173168608](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39173168608) | 1.458 ms | 2.648 ms | 1.789 ms | Reuse reduces guest creation overhead, but remains slower than native | +| 2e0a2b8 / [39174111506](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39174111506) | 1.485 ms | 2.613 ms | 1.792 ms | Earlier validated request-proxy sample favored native | +| 6719fa6 / [39175130478](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39175130478) | 1.493 ms | 2.620 ms | 1.799 ms | Earlier validated 25-file/100-edit sample: public path still slower | +| Source 6dafd497, tested merge b49e060 / [39195170609](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609) | 1.150 ms | 2.096 ms | 1.451 ms | Preserved pre-EOF-fix hosted snapshot; public path still slower | + +For the 9c04ee6 100-edit sample, the native subprocess finished faster in **all 100 paired trials** than both the public in-process path and the experimental reused guest. The experimental reused guest finished faster than the public in-process path in all 100 pairs. This is limited to one fixture, one Linux job and sequential execution, not a representative IDE workload. + +**Cold WASM compilation cache:** The 2e0a2b8 run executed two *different Go processes* using the same optional HYBRID_W012_CACHE_DIR and matching WASM/native hashes. Public GetEngine cold initialization measured **616.613 ms for the first process** and **19.585 ms for the second**. This is a real cross-process compilation-cache benefit; it does **not** alter Engine.Check's hot request speed. The cache is opt-in; a directory controlled by the user must be protected from external modification. The control never caches source analysis results. The latest [6719fa6 run 39175130478](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39175130478) likewise recorded **623.789 ms → 20.105 ms** for a first versus second process with a protected shared compilation cache. This is a cold-start observation only. + +**Go consumer without a Rust compiler invocation:** In [run 39174111506](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39174111506), a separate workflow job downloaded the precompiled WASM from the Rust-capable build job, compiled the Go hybvet executable with no Rust installation step, and executed failing-then-passing .hy source tests while cargo and rustc command names were explicitly poisoned. This verifies an execution boundary without *invoking* the Rust toolchain. It is not proof that the hosted VM contains no Rust installation, nor that a ready-to-install distribution has been shipped. + +**Decision consequence:** Do not replace the production per-check disposable instance on these measurements. The proposed long-lived reused guest remains an isolated sequential research control; cancellation and concurrent ownership must be validated before any production pooling. For batch runs, native CLI is currently the faster measured path. A packaged Go host plus shipped WASM and a protected compilation cache is a plausible deployment benefit, but gopls integration, CI Docker footprint, broad-workload latency and RSS remain unproven. + +## Durable review evidence + +The [2026-10-19 archive](validation-2026-10-19/README.md) contains the byte-identical 100-edit JSON and both two-process compilation-cache JSONs from run 39195170609, with source/merge/base identities and verified artifact/file SHA-256 digests. The original attachments expire on October 26, before the November 1 M1 review. The committed archive remains available after that expiration. New workflow samples and cache timings retain artifacts for 30 days; this setting does not extend existing attachments. + +These preserved measurements precede the EOF diagnostic-position fix. They are not a rebenchmark or correctness evidence for a later source revision. The EOF fix changes token-table mapping, not the checker input or the public Engine.Check request boundary; its correctness is verified separately by the analyzer regressions. + diff --git a/spikes/w012-inprocess/validation-2026-10-18/README.md b/spikes/w012-inprocess/validation-2026-10-18/README.md new file mode 100644 index 00000000..c71b7ce1 --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-18/README.md @@ -0,0 +1,96 @@ +# W012 validation snapshots — 2026-10-18 + +> **Latest hosted correctness evidence, not a new benchmark:** [W012 validation run +> 39169356538](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39169356538) +> checked PR head `e029d661d8ce7f79e7f362a7ee97bf3251d6a274` against the +> then-current main: **113/113 full native/WASM envelope parity**, Go race suite, +> and Go vet passed; the [compiler CI](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39169356527) +> passed too. This head also guards declared `.hy` read failures and duplicate +> paths. Earlier hosted run 39143438366 passed 79/79 and is superseded only as +> a corpus-count snapshot. This **60-fixture** benchmark and its original raw +> samples remain unchanged for reproducibility. No new public-path latency +> benchmark or editor/LSP integration result is implied. + +## Scope and provenance + +- Prior PR #556 head: `aa2d9bdd598d32cb6fd19a30b9fb500f9c96563f`. +- Tested correction: `34c759f91e4ff76c8b6d587208cbf4bea6a51ca0` (code tree `d084a402e25d02ded364d7747fc7f776aa2c50af`). This report is a documentation-only follow-on to that tested commit. +- This branch contains **60** checker-corpus `.hy` fixtures. These results do not claim coverage of a later upstream corpus. +- Cloud Linux x86-64; kernel 6.18.55; Go 1.26.9; Rust 1.99.0; Node 24.19.0; wazero 1.9.0; `GOMAXPROCS=9`. The host reports AMD EPYC 9V74 80-Core Processor, with 9 CPUs visible to this process; the model name does not imply 80 dedicated CPUs. Toolchains were installed from their official distribution sites; Go archive SHA-256 was checked against the official download manifest. +- Native and WASM artifacts were freshly built from the prior PR head using Cargo's locked dependencies and release profile. The correction changes no Rust source, so those artifacts are also the correction's Rust artifacts. +- WASM: 1,138,172 bytes, SHA-256 `5e5c7feb5599e134cb5a21eaf68b4e7bec674e05967b4667a15db3c3d6c18b81`. +- Native binary SHA-256: `256696d7de0fe8fe83e27cf04fab61af68f6c50290f70b17b0b9889d534d149c`. +- [Raw benchmark samples](benchmark-samples.json): 100 samples per case; SHA-256 `2a8d694f9b5cc0b5ffa6ac4617c1f65d15d296a091d82e1532c05c40c8dd85da`. + +## Correctness: a real wrapper discrepancy found and fixed + +At the prior head, the raw native/WASM envelope comparison passed **60/60**, and the existing Go race suite passed. An additional comparison through the actual Go `Engine.Check` found **58/60** agreement for the fields represented by `CheckOutput`. + +The discrepancies were `02-shared-map-managed.hy` and `18-pool-shared-struct-managed.hy`. Both native results had `verdict=accept`, `errors=0`, `warnings=1`, and one `HYB-E8001` managed-safety warning. The Go wrapper kept `warnings=1` but returned zero diagnostics. `CheckSourceWithABI` dropped accepted diagnostics, and `Analyzer.Run` separately reported diagnostics only for rejected source. + +The correction retains warnings/notes for accepted source and reports them through the analyzer. Acceptance still means no error diagnostics; infrastructure failures remain distinct. New tests cover the complete branch corpus through the public wrapper, a valid accepted-warning envelope, rejection of an error under the accept invariant, and an actual analyzer pass reporting the managed warning at its exact source position. + +After correction: + +- Native/WASM complete normalized JSON-envelope equality: **60/60**. +- Go wrapper/native equality for modeled fields: **60/60**. +- `go test -race -count=1 -v ./...`: **pass**, including the existing real trap, active timeout, malformed-envelope, and boundary-failure tests. +- `go vet ./...`: **pass**. +- Real `go vet -vettool` smoke: expected exit **1**, with `sibling.hy:6:5`, `:7:5`, and `:7:13`. +- Benchmark native output now uses the same strict envelope decoder as WASM. Negative tests reject a missing binary, malformed JSON, missing required warning count, missing code/message, invalid span, unknown severity, contradictory rejection, and unexpected exit codes. +- JS parity now frees its guest output buffer after decoding. This removes a harness leak; it does not prove long-duration memory stability. + +**Driver caveat:** Go `analysis.Diagnostic` has no separate warning severity. The semantic result remains `accept` with zero errors, but reporting the retained warning makes `go vet -vettool` exit 1. This was checked on a temporary package containing fixture 02: `managed.hy:76:13` reports `HYB-E8001`, exit 1. Thus the host driver exit policy differs from native `hyb check` on warning-only input; it is not full CLI exit-code equivalence. + +The Go model deliberately does not represent every native envelope field (for example, diagnostic notes/fixits). Its parity test is therefore explicitly modeled-field parity. The separate JS test compares complete envelopes after filename normalization only. Neither result establishes unsaved-buffer, stock-gopls, or end-to-end LSP integration. + +## Current benchmark: actual analyzer method is slower here + +Single reject fixture: `08-a3-map-slot-invalidated.hy`; 100 sequential samples per case. Times are milliseconds. Native measurements include JSON capture and validation. Every WASM case enables the same context-cancellation support used by the production wrapper. + +| Path | p50 | p99 | Mean | +| --- | ---: | ---: | ---: | +| Native `hyb check --json`, subprocess + validation | 1.204 | 1.796 | 1.256 | +| Public `Engine.Check`, compiled module reused, fresh instance per call | 2.143 | 7.176 | 2.658 | +| Reused-instance boundary check, experimental comparison | 1.097 | 1.521 | 1.151 | + +The implemented public wrapper is about **1.78× the native p50 latency** on this host. The already-instantiated comparison is only about **1.10× faster than native** and is not the analyzer's implemented instance-lifetime strategy. It is not evidence of an 18.7× current speedup or sub-millisecond IDE diagnostics. + +Cold compilation p50 is **388.557 ms**; filesystem-cached compilation p50 is **11.626 ms**. The filesystem-cache case opens new cache/runtime handles in the same process; it is not a fresh-process startup benchmark. The harness labels its compile-only amortization estimates as excluding instantiation and process startup, and emits no finite break-even when the warm path is slower. + +These are **cross-host, changed-harness measurements**, not a direct performance regression comparison with the historical snapshot. The old snapshot used a different machine and predated the currently validated/cancellation-enabled path. No tuning was performed to recover its ratio. One fixture and one batch per case are insufficient to characterize all workloads or timing variance. + +## Reproduction + +Use the pinned Go/Rust toolchains on PATH. Run from the checkout containing the tested correction; `CARGO_BUILD_JOBS=2` and `GOFLAGS=-p=2` bound build parallelism. Do not set native linker flags on the wasm target. + +```sh +export CARGO_BUILD_JOBS=2 +export GOFLAGS=-p=2 +(cd compiler && cargo build --locked --release -p hyb) +(cd compiler && cargo build --locked --release -p hy-wasm --target wasm32-unknown-unknown) +export HYBRID_WASM_PATH="$PWD/compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm" +export HYBRID_NATIVE_PATH="$PWD/compiler/target/release/hyb" +node test_wasm_parity.js +cd spikes/w012-analyzer +go test -race -count=1 -v ./... +go vet ./... +go build -o /tmp/hybvet ./cmd/hybvet +go vet -vettool=/tmp/hybvet ./testpkg # expected exit 1 and three documented diagnostics +go build -o /tmp/w012-bench ./bench +/tmp/w012-bench -iterations 100 -output /tmp/w012-benchmark.json +``` + +The benchmark also accepts `-root`, `-wasm`, `-native`, and `-fixture` for explicit paths. Run the binary without the race detector for timing. The captured run used `flock /tmp/heavy.lock nice -n 10 timeout 600` around the benchmark. Existing Rust builds emitted an unreachable-pattern warning in `hy-check/src/model.rs`; no Rust warning was changed or suppressed. + +## Continuous validation + +`.github/workflows/w012-validation.yml` adds a separate, path-filtered job for pull requests and main pushes affecting the compiler, W012 analyzer, corpus, parity script, environment pins, or the workflow itself. It builds native and wasm artifacts, runs full-envelope Node parity, Go race/regression tests, and Go vet. The existing generic CI workflow is unchanged. Actions are commit-pinned; permissions are only `contents: read`; checkout does not persist credentials. There are no secrets, schedules, or performance benchmarks. + +Workflow YAML structure, trigger/permission constraints, action references, and shell syntax were checked locally. Its build/test commands were executed locally as documented above. At the time of the original benchmark this workflow had not yet run in GitHub Actions. It has since passed in the hosted run linked above. The hosted run is correctness/race validation, not a repeat of the latency benchmark. + +## Implication for M1 review + +This strengthens the embedding-boundary evidence by finding and fixing an observable loss of managed-safety warnings. It supports reuse of the Rust checker as an embeddable implementation without a semantic Go rewrite. It does **not** establish an IDE latency win. Keep native CLI as the measured baseline; a next performance experiment should be bounded to a real consumer's request path, with cancellation, diagnostics, concurrency, and state-isolation tests, rather than adopting a pooled/reused instance on timing alone. + +Full editor latency, long-duration memory stability, the proposal's 25-file/100-edit workload, and distribution footprint remain unmeasured here. M1 acceptance remains a maintainer decision; these measurements imply no award probability. diff --git a/spikes/w012-inprocess/validation-2026-10-18/benchmark-samples.json b/spikes/w012-inprocess/validation-2026-10-18/benchmark-samples.json new file mode 100644 index 00000000..98cfaef1 --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-18/benchmark-samples.json @@ -0,0 +1,1867 @@ +{ + "arch": "amd64", + "filesystem_cache_same_process": { + "check": [ + 2.24, + 9.066, + 1.859, + 2.516, + 2.149, + 2.585, + 12.717, + 6.636, + 4.526, + 2.411, + 2.765, + 2.686, + 2.815, + 2.397, + 1.851, + 2.721, + 2.934, + 2.629, + 6.622, + 3.169, + 4.064, + 2.629, + 3.956, + 2.962, + 4.762, + 2.784, + 2.188, + 6.108, + 1.97, + 3.139, + 2.272, + 2.584, + 3.405, + 2.52, + 6.157, + 3.562, + 2.44, + 3.456, + 2.535, + 3.651, + 1.816, + 4.493, + 2.35, + 3.945, + 2.228, + 2.159, + 2.254, + 2.487, + 2.313, + 2.049, + 2.216, + 2.061, + 2.358, + 2.256, + 2.526, + 3.07, + 3.496, + 2.657, + 2.631, + 2.423, + 3.464, + 2.348, + 4.382, + 2.932, + 2.186, + 2.197, + 2.247, + 2.457, + 2.333, + 2.243, + 1.941, + 2.452, + 1.929, + 1.887, + 2.317, + 2.029, + 2.078, + 1.936, + 2.163, + 2.162, + 2.509, + 2.221, + 2.055, + 2.357, + 1.9, + 2.554, + 2.174, + 2.265, + 2.144, + 2.737, + 2.632, + 3.25, + 2.877, + 2.405, + 3.113, + 2.88, + 7.236, + 2.547, + 2.776, + 1.904 + ], + "compile": [ + 11.608, + 11.893, + 11.318, + 12.842, + 11.208, + 13.528, + 12.166, + 10.825, + 10.897, + 10.898, + 11.751, + 12.716, + 11.967, + 11.639, + 11.607, + 14.556, + 13.206, + 12.775, + 10.916, + 11.435, + 10.839, + 10.646, + 10.816, + 10.971, + 11.847, + 13.164, + 11.319, + 12.864, + 10.646, + 12.366, + 11.983, + 14.273, + 11.987, + 12.034, + 10.799, + 10.801, + 10.801, + 11.563, + 10.636, + 11.187, + 10.853, + 12.045, + 10.837, + 11.971, + 11.236, + 12.399, + 11.492, + 11.559, + 12.258, + 12.989, + 11.026, + 10.594, + 12.478, + 11.987, + 12.361, + 11.918, + 11.393, + 11.994, + 12.254, + 11.927, + 11.795, + 12.045, + 10.879, + 10.916, + 12.475, + 13.382, + 10.646, + 12.389, + 12.501, + 11.36, + 11.405, + 11.145, + 11.092, + 10.816, + 10.423, + 10.773, + 10.76, + 12.668, + 11.714, + 11.637, + 11.205, + 11.618, + 11.828, + 11.298, + 12.158, + 12.152, + 11.742, + 10.949, + 10.823, + 11.121, + 10.894, + 11.353, + 12.251, + 12.201, + 11.114, + 11.626, + 12.334, + 18.277, + 11.831, + 14.456 + ], + "instantiate": [ + 0.578, + 0.06, + 0.085, + 0.104, + 0.086, + 0.076, + 0.283, + 0.065, + 0.634, + 0.404, + 0.077, + 0.268, + 0.085, + 0.081, + 0.076, + 1.67, + 0.714, + 0.819, + 0.109, + 0.407, + 0.072, + 0.182, + 0.071, + 0.229, + 0.072, + 0.266, + 0.124, + 0.337, + 0.062, + 0.099, + 0.071, + 2.649, + 0.286, + 0.074, + 0.416, + 0.071, + 0.072, + 0.07, + 0.167, + 0.093, + 0.073, + 0.192, + 0.059, + 0.068, + 0.086, + 1.88, + 0.448, + 0.302, + 2.341, + 1.042, + 0.195, + 0.986, + 0.213, + 0.97, + 0.196, + 0.492, + 0.075, + 0.473, + 0.074, + 0.15, + 0.14, + 0.175, + 0.076, + 0.214, + 0.074, + 0.613, + 0.522, + 0.534, + 0.536, + 1.408, + 2.635, + 0.698, + 0.734, + 0.86, + 0.267, + 0.654, + 1.948, + 2.043, + 0.81, + 0.912, + 1.036, + 1.531, + 0.918, + 0.696, + 0.871, + 0.22, + 0.188, + 0.369, + 1.339, + 0.402, + 0.138, + 0.535, + 0.094, + 0.082, + 0.083, + 0.249, + 0.217, + 0.57, + 0.085, + 0.247 + ], + "total": [ + 14.554, + 21.139, + 13.36, + 15.542, + 13.522, + 16.286, + 25.246, + 17.629, + 16.137, + 13.878, + 14.669, + 15.79, + 14.949, + 14.198, + 13.631, + 19.034, + 17.02, + 16.34, + 17.772, + 15.121, + 15.055, + 13.558, + 14.92, + 14.262, + 16.79, + 16.319, + 13.707, + 19.383, + 12.756, + 15.684, + 14.405, + 19.583, + 15.758, + 14.748, + 17.455, + 14.514, + 13.391, + 15.17, + 13.412, + 15.008, + 12.821, + 16.812, + 13.322, + 16.061, + 13.628, + 16.527, + 14.272, + 14.492, + 17.028, + 16.186, + 13.549, + 13.728, + 15.131, + 15.303, + 15.167, + 15.556, + 15.04, + 15.202, + 15.038, + 14.578, + 15.498, + 14.648, + 15.421, + 14.146, + 14.966, + 16.301, + 13.594, + 15.659, + 15.459, + 15.327, + 16.107, + 14.422, + 13.835, + 13.642, + 13.087, + 13.548, + 14.885, + 16.728, + 14.785, + 14.831, + 14.84, + 15.519, + 14.923, + 14.457, + 15.021, + 15.102, + 14.485, + 13.767, + 14.409, + 14.373, + 13.767, + 15.228, + 15.365, + 14.779, + 14.526, + 14.859, + 19.871, + 21.522, + 14.862, + 17.393 + ] + }, + "fixture": "tests/checker-corpus/08-a3-map-slot-invalidated.hy", + "fresh_runtime": { + "check": [ + 6.8, + 3.002, + 2.277, + 2.185, + 2.598, + 1.836, + 2.395, + 1.902, + 2.191, + 2.323, + 2.649, + 2.681, + 2.651, + 1.776, + 2.296, + 2.397, + 2.281, + 2.429, + 2.393, + 2.256, + 5.358, + 2.321, + 3.491, + 2.666, + 2.519, + 4.223, + 2.927, + 2.465, + 3.181, + 1.775, + 2.536, + 2.483, + 1.762, + 2.026, + 2.117, + 1.794, + 2.554, + 2.23, + 4.422, + 2.555, + 2.527, + 1.817, + 2.44, + 2.411, + 2.26, + 2.417, + 2.499, + 2.323, + 6.633, + 1.831, + 2.486, + 1.7, + 2.576, + 1.676, + 2.581, + 2.426, + 3.715, + 1.808, + 5.557, + 2.173, + 2.416, + 3.298, + 2.461, + 2.605, + 1.871, + 3.329, + 7.391, + 2.585, + 3.085, + 2.099, + 4.699, + 2.725, + 2.81, + 3.401, + 2.641, + 10.193, + 2.67, + 2.732, + 3.081, + 2.456, + 2.064, + 3.249, + 4.447, + 2.642, + 2.08, + 3.028, + 2.679, + 2.184, + 2.576, + 2.723, + 2.371, + 1.83, + 2.563, + 2.456, + 2.559, + 2.92, + 2.79, + 1.939, + 2.743, + 3.561 + ], + "compile": [ + 419.131, + 378.012, + 383.934, + 452.524, + 401.642, + 377.693, + 400.48, + 404.254, + 402.588, + 405.012, + 399.818, + 390.044, + 381.887, + 417.326, + 397.009, + 375.122, + 360.327, + 381.383, + 364.344, + 398.18, + 426.459, + 383.997, + 429.562, + 376.853, + 391.058, + 389.663, + 404.934, + 405.742, + 434.516, + 395.945, + 402.483, + 377.257, + 367.113, + 384.776, + 423.05, + 410.02, + 442.577, + 389.366, + 394.037, + 391.922, + 377.692, + 388.557, + 387.949, + 386.888, + 420.378, + 414.801, + 398.888, + 391.204, + 360.976, + 396.823, + 362.822, + 395.86, + 380.165, + 364.531, + 395.258, + 382.307, + 356.76, + 358.016, + 361.416, + 374.642, + 373.029, + 391.196, + 453.584, + 389.223, + 388.474, + 372.55, + 380.934, + 390.84, + 381.416, + 384.17, + 398.2, + 398.694, + 412.789, + 396.424, + 392.018, + 383.214, + 365.404, + 390.026, + 379.567, + 377.847, + 375.357, + 370.808, + 360.997, + 379.029, + 398.85, + 390.372, + 378.189, + 378.777, + 405.002, + 380.264, + 369.882, + 367.777, + 359.874, + 375.262, + 376.742, + 394.314, + 420.503, + 396.936, + 382.595, + 368.818 + ], + "instantiate": [ + 0.096, + 0.085, + 0.09, + 0.417, + 0.085, + 0.405, + 0.087, + 0.106, + 0.099, + 0.147, + 0.099, + 0.514, + 0.086, + 0.393, + 0.088, + 0.089, + 1.783, + 0.109, + 0.091, + 0.086, + 0.468, + 0.094, + 0.089, + 0.089, + 0.083, + 0.087, + 0.287, + 0.084, + 0.098, + 0.51, + 0.323, + 0.082, + 0.41, + 0.096, + 0.101, + 0.091, + 0.15, + 0.095, + 0.138, + 0.091, + 0.445, + 0.372, + 0.123, + 0.117, + 0.084, + 0.408, + 0.351, + 0.088, + 0.448, + 0.09, + 0.09, + 0.485, + 0.089, + 0.456, + 0.087, + 0.089, + 0.165, + 0.482, + 0.093, + 0.087, + 0.756, + 0.162, + 0.111, + 0.083, + 0.639, + 0.095, + 0.085, + 0.219, + 0.103, + 0.404, + 0.279, + 0.088, + 0.093, + 0.082, + 0.085, + 0.08, + 0.099, + 0.082, + 0.087, + 0.087, + 0.446, + 0.374, + 0.65, + 0.084, + 0.504, + 0.086, + 0.083, + 0.417, + 0.088, + 0.086, + 0.397, + 0.094, + 0.083, + 0.084, + 0.124, + 0.096, + 0.084, + 0.41, + 0.092, + 0.086 + ], + "total": [ + 426.134, + 381.203, + 386.385, + 455.207, + 404.414, + 380.011, + 403.075, + 406.35, + 405.006, + 407.561, + 402.651, + 393.337, + 384.732, + 419.598, + 399.472, + 377.696, + 364.473, + 384.003, + 366.917, + 400.659, + 432.369, + 386.492, + 433.242, + 379.685, + 393.742, + 394.072, + 410.61, + 408.361, + 437.885, + 398.353, + 405.44, + 379.931, + 369.367, + 386.98, + 425.403, + 412.001, + 445.381, + 391.871, + 398.682, + 394.734, + 380.795, + 390.819, + 390.594, + 389.508, + 422.805, + 417.708, + 401.957, + 393.704, + 368.139, + 398.854, + 365.481, + 398.125, + 382.915, + 366.744, + 398.001, + 384.927, + 360.729, + 360.411, + 367.147, + 377.014, + 376.293, + 395.223, + 456.24, + 392.001, + 391.061, + 376.057, + 388.499, + 393.726, + 384.756, + 386.838, + 403.279, + 401.607, + 415.769, + 399.991, + 394.817, + 393.574, + 368.254, + 392.923, + 382.809, + 380.62, + 377.974, + 374.513, + 366.201, + 381.843, + 401.51, + 393.57, + 381.036, + 381.824, + 407.767, + 383.218, + 372.756, + 369.787, + 362.595, + 377.882, + 379.505, + 397.433, + 423.59, + 399.363, + 385.535, + 372.548 + ] + }, + "go": "go1.26.9", + "gomaxprocs": 9, + "iterations": 100, + "memory_cache": { + "check": [ + 2.604, + 4.339, + 6.21, + 2.258, + 3.652, + 5.211, + 1.888, + 2.034, + 6.651, + 2.111, + 2.27, + 1.815, + 2.977, + 2.263, + 2.48, + 2.013, + 5.629, + 2.178, + 2.171, + 2.034, + 2.487, + 1.663, + 6.685, + 2.55, + 2.208, + 2.126, + 2.062, + 2.017, + 2.921, + 1.746, + 3.219, + 2, + 3.273, + 2.17, + 2.175, + 2.037, + 8.034, + 1.856, + 2.573, + 3.069, + 2.368, + 3.17, + 2.336, + 5.619, + 2.462, + 6.79, + 2.45, + 7.385, + 2.228, + 7.427, + 2.199, + 1.845, + 2.849, + 2.462, + 9.35, + 1.928, + 1.928, + 3.256, + 10.233, + 1.941, + 2.607, + 4.617, + 1.974, + 2.253, + 2.87, + 2.133, + 2.158, + 9.266, + 2.29, + 2.171, + 7.231, + 2.205, + 2.409, + 2.606, + 9.903, + 2.256, + 2.678, + 2.547, + 2.046, + 1.717, + 2.804, + 1.703, + 7.724, + 2.613, + 2.134, + 2.181, + 8.415, + 2.36, + 2.386, + 1.652, + 10.544, + 2.279, + 3.24, + 2.316, + 7.463, + 2.339, + 2.615, + 7.662, + 2.43, + 1.745 + ], + "compile": [ + 391.836, + 9.12, + 8.867, + 9.371, + 8.471, + 9.064, + 8.618, + 8.166, + 8.841, + 8.255, + 8.171, + 10.02, + 10.634, + 8.359, + 8.861, + 8.59, + 10.235, + 8.504, + 9.39, + 8.899, + 8.639, + 8.687, + 9.426, + 8.704, + 9.544, + 8.744, + 8.943, + 9.71, + 21.355, + 11.16, + 10.5, + 9.816, + 11.773, + 9.015, + 11.609, + 8.384, + 10.293, + 8.857, + 9.974, + 10.557, + 8.42, + 8.682, + 9.455, + 10.643, + 9.51, + 9.898, + 8.723, + 9.238, + 8.452, + 10.585, + 8.645, + 9.593, + 9.09, + 9.718, + 9.144, + 8.54, + 9.984, + 9.839, + 8.464, + 8.864, + 8.693, + 8.954, + 8.795, + 8.862, + 8.843, + 8.144, + 8.806, + 8.906, + 8.549, + 9.187, + 9.392, + 9.059, + 8.479, + 9.564, + 9.561, + 8.611, + 9.345, + 8.382, + 9.853, + 8.215, + 8.735, + 9.17, + 9.392, + 9.521, + 9.838, + 8.541, + 9.231, + 9.021, + 8.394, + 9.006, + 9.025, + 8.608, + 9.732, + 9.378, + 9.519, + 8.939, + 8.848, + 9.382, + 8.862, + 8.922 + ], + "instantiate": [ + 0.089, + 0.335, + 0.346, + 0.074, + 0.317, + 0.072, + 0.342, + 0.831, + 0.35, + 0.172, + 0.697, + 0.075, + 0.068, + 0.07, + 0.07, + 0.427, + 0.067, + 0.067, + 0.08, + 0.392, + 0.703, + 0.071, + 0.188, + 0.08, + 0.071, + 0.391, + 0.126, + 0.166, + 0.088, + 0.08, + 0.382, + 0.084, + 0.14, + 0.454, + 0.184, + 0.344, + 0.072, + 0.064, + 0.561, + 0.071, + 0.065, + 0.487, + 0.442, + 0.09, + 0.498, + 0.079, + 0.067, + 0.476, + 0.083, + 0.283, + 0.07, + 0.347, + 0.064, + 0.065, + 0.059, + 0.383, + 0.57, + 0.072, + 0.072, + 0.081, + 0.71, + 0.07, + 0.076, + 3.688, + 0.374, + 0.291, + 0.203, + 0.081, + 0.169, + 0.333, + 0.387, + 0.207, + 0.201, + 0.1, + 0.133, + 0.07, + 0.443, + 0.07, + 0.198, + 0.379, + 0.372, + 0.071, + 0.071, + 0.073, + 0.402, + 0.385, + 0.189, + 0.13, + 0.564, + 0.076, + 0.063, + 0.07, + 0.534, + 0.075, + 0.188, + 0.071, + 0.614, + 0.231, + 0.065, + 2.549 + ], + "total": [ + 394.541, + 13.804, + 15.434, + 11.713, + 12.478, + 14.356, + 10.879, + 11.054, + 15.849, + 10.548, + 11.151, + 11.92, + 13.686, + 10.701, + 11.418, + 11.039, + 15.94, + 10.758, + 11.651, + 11.335, + 11.841, + 10.427, + 16.307, + 11.343, + 11.832, + 11.287, + 11.14, + 12.056, + 24.375, + 12.995, + 14.12, + 11.913, + 15.196, + 11.65, + 13.99, + 10.775, + 18.407, + 10.785, + 13.116, + 13.704, + 10.86, + 12.404, + 12.243, + 16.37, + 12.48, + 16.776, + 11.249, + 17.108, + 10.773, + 18.305, + 10.922, + 11.819, + 12.072, + 12.339, + 18.566, + 10.86, + 12.491, + 13.251, + 18.779, + 10.908, + 12.021, + 13.648, + 10.856, + 14.818, + 12.097, + 10.577, + 11.183, + 18.26, + 11.02, + 11.733, + 17.024, + 11.48, + 11.185, + 12.286, + 19.605, + 10.945, + 12.478, + 11.008, + 12.107, + 10.32, + 11.918, + 10.953, + 17.195, + 12.217, + 12.385, + 11.117, + 17.845, + 11.521, + 11.36, + 10.746, + 19.639, + 10.966, + 13.514, + 11.778, + 17.178, + 11.358, + 12.087, + 17.286, + 11.37, + 13.228 + ] + }, + "native": [ + 1.796, + 1.173, + 1.53, + 1.488, + 1.175, + 1.08, + 1.535, + 1.065, + 1.349, + 1.494, + 1.134, + 1.212, + 1.438, + 1.104, + 1.122, + 1.335, + 1.46, + 1.204, + 1.253, + 1.114, + 1.059, + 1.119, + 1.273, + 1.155, + 1.363, + 1.49, + 2.113, + 1.287, + 1.389, + 1.25, + 1.181, + 1.168, + 1.162, + 1.339, + 1.178, + 1.255, + 1.194, + 1.187, + 1.271, + 1.208, + 1.618, + 1.594, + 1.504, + 1.328, + 1.155, + 1.181, + 1.251, + 1.085, + 1.43, + 1.197, + 1.239, + 1.24, + 1.162, + 1.239, + 1.322, + 1.162, + 1.106, + 1.227, + 1.253, + 1.119, + 1.233, + 1.134, + 1.091, + 1.175, + 1.126, + 1.081, + 1.187, + 1.128, + 1.107, + 1.233, + 1.097, + 1.092, + 1.237, + 1.184, + 1.451, + 1.237, + 1.229, + 1.36, + 1.183, + 1.595, + 1.12, + 1.123, + 1.162, + 1.212, + 1.182, + 1.316, + 1.162, + 1.181, + 1.152, + 1.452, + 1.136, + 1.246, + 1.115, + 1.163, + 1.255, + 1.124, + 1.475, + 1.082, + 1.223, + 1.47 + ], + "native_path": "compiler/target/release/hyb", + "os": "linux", + "public_engine_check": [ + 2.71, + 2.713, + 3.015, + 2.021, + 2.361, + 1.789, + 3.384, + 2.275, + 3.118, + 1.577, + 2.512, + 1.766, + 6.091, + 1.944, + 3.052, + 2.013, + 3.128, + 2.295, + 2.639, + 1.605, + 2.143, + 1.996, + 2.68, + 2.211, + 4.865, + 1.855, + 2.187, + 2.114, + 2.598, + 1.689, + 1.974, + 1.579, + 2.881, + 1.794, + 2.033, + 1.756, + 2.104, + 2.282, + 3.324, + 1.812, + 2.362, + 1.856, + 5.693, + 1.734, + 2.252, + 1.753, + 4.644, + 1.83, + 2.539, + 1.789, + 2.467, + 2.906, + 2.188, + 1.899, + 5.624, + 2.003, + 6.092, + 1.607, + 2.589, + 2.037, + 2.587, + 1.572, + 2.328, + 2.046, + 7.176, + 1.722, + 1.935, + 1.752, + 2.16, + 1.774, + 1.875, + 1.798, + 7.189, + 1.635, + 2.07, + 1.755, + 4.897, + 1.942, + 4.417, + 1.696, + 6.871, + 2.086, + 5.201, + 1.717, + 2.151, + 1.783, + 2.713, + 1.729, + 3.975, + 1.685, + 2.313, + 1.642, + 2.179, + 3.58, + 5.756, + 1.987, + 2.331, + 1.874, + 4.608, + 1.539 + ], + "reused_compiled_module": { + "check": [ + 2.531, + 2.422, + 2.669, + 2.145, + 2.048, + 2.519, + 2.852, + 6.515, + 2.358, + 7.946, + 2.604, + 5.072, + 2.769, + 2.555, + 1.732, + 2.352, + 1.664, + 1.949, + 1.66, + 2.08, + 1.902, + 1.906, + 2, + 2.269, + 1.771, + 2.033, + 1.58, + 1.929, + 1.66, + 1.986, + 1.525, + 2.094, + 1.74, + 2.538, + 1.817, + 2.161, + 1.796, + 2.056, + 1.666, + 2.512, + 1.839, + 2.246, + 1.994, + 2.068, + 1.603, + 2.455, + 2.007, + 2.509, + 1.88, + 2.973, + 1.644, + 2.145, + 1.602, + 2.039, + 1.883, + 2.538, + 1.749, + 2.198, + 1.816, + 2.345, + 1.549, + 1.818, + 2.151, + 2.019, + 1.8, + 1.894, + 1.933, + 2.059, + 1.491, + 2.285, + 1.881, + 1.974, + 1.619, + 1.995, + 1.61, + 2.206, + 1.839, + 1.891, + 1.661, + 2.114, + 1.619, + 1.945, + 1.651, + 1.97, + 1.558, + 1.784, + 1.494, + 1.974, + 1.612, + 2.513, + 1.706, + 2.052, + 1.63, + 2.175, + 1.66, + 2.04, + 1.648, + 2.25, + 1.649, + 2.06 + ], + "instantiate": [ + 0.083, + 0.072, + 0.793, + 0.099, + 0.469, + 0.051, + 2.059, + 0.063, + 0.064, + 0.05, + 0.143, + 0.065, + 0.119, + 0.386, + 0.051, + 0.578, + 0.089, + 0.067, + 0.105, + 0.046, + 0.071, + 0.069, + 0.045, + 0.048, + 0.097, + 0.045, + 0.044, + 0.046, + 0.057, + 0.048, + 0.053, + 0.047, + 0.052, + 0.049, + 0.053, + 0.11, + 0.047, + 0.361, + 0.045, + 0.049, + 0.55, + 0.588, + 0.504, + 0.068, + 0.06, + 0.575, + 0.587, + 0.412, + 0.499, + 0.059, + 0.055, + 0.573, + 0.056, + 0.051, + 0.052, + 0.052, + 0.061, + 0.05, + 0.046, + 0.059, + 0.047, + 0.047, + 0.044, + 0.052, + 0.045, + 0.047, + 0.045, + 0.05, + 0.044, + 0.045, + 0.045, + 0.053, + 0.046, + 0.046, + 0.056, + 0.055, + 0.056, + 0.055, + 0.048, + 0.048, + 0.045, + 0.045, + 0.045, + 0.049, + 0.049, + 0.048, + 0.044, + 0.047, + 0.075, + 0.046, + 0.048, + 0.052, + 0.048, + 0.098, + 0.052, + 0.086, + 0.047, + 0.049, + 0.05, + 0.05 + ], + "total": [ + 2.621, + 2.499, + 3.472, + 2.25, + 2.522, + 2.575, + 4.952, + 6.586, + 2.426, + 8.002, + 2.759, + 5.144, + 2.9, + 2.948, + 1.787, + 2.938, + 1.759, + 2.023, + 1.769, + 2.13, + 1.977, + 1.981, + 2.049, + 2.322, + 1.871, + 2.083, + 1.627, + 1.98, + 1.729, + 2.045, + 1.589, + 2.148, + 1.797, + 2.598, + 1.875, + 2.278, + 1.847, + 2.422, + 1.716, + 2.568, + 2.402, + 2.843, + 2.504, + 2.146, + 1.667, + 3.042, + 2.602, + 2.93, + 2.39, + 3.042, + 1.705, + 2.728, + 1.663, + 2.097, + 1.939, + 2.597, + 1.814, + 2.253, + 1.868, + 2.409, + 1.6, + 1.87, + 2.199, + 2.077, + 1.863, + 1.945, + 1.983, + 2.115, + 1.54, + 2.335, + 1.93, + 2.075, + 1.669, + 2.046, + 1.67, + 2.268, + 1.9, + 1.953, + 1.713, + 2.168, + 1.668, + 1.994, + 1.699, + 2.024, + 1.611, + 1.837, + 1.547, + 2.027, + 1.692, + 2.564, + 1.757, + 2.109, + 1.681, + 2.279, + 1.718, + 2.133, + 1.698, + 2.304, + 1.702, + 2.115 + ] + }, + "reused_instance": [ + 2.703, + 1.16, + 1.253, + 1.095, + 1.027, + 1.015, + 1.007, + 1.038, + 1.035, + 1.029, + 0.994, + 1.102, + 1.069, + 0.995, + 1.055, + 1.196, + 1.169, + 1.072, + 1.035, + 1.131, + 1.284, + 1.107, + 1.082, + 1.134, + 1.084, + 1.038, + 1.005, + 1.141, + 1.068, + 1.021, + 1.33, + 1.119, + 1.01, + 1.274, + 1.088, + 1.045, + 0.997, + 1.37, + 1.135, + 1.036, + 1.004, + 1.099, + 1.023, + 1.015, + 0.999, + 1.079, + 0.991, + 1.149, + 1.096, + 1.097, + 1.162, + 1.023, + 1.198, + 1.084, + 1.067, + 1.206, + 1.223, + 1.051, + 1.114, + 1.051, + 1.014, + 1.074, + 1.398, + 1.077, + 1.069, + 1.38, + 1.074, + 1.076, + 1.252, + 1.087, + 1.252, + 1.107, + 1.176, + 1.291, + 1.107, + 1.032, + 1.138, + 1.304, + 1.073, + 1.521, + 1.268, + 1.063, + 1.277, + 1.092, + 1.362, + 1.134, + 1.321, + 1.31, + 1.079, + 1.207, + 1.153, + 1.295, + 1.195, + 1.41, + 1.225, + 1.194, + 1.152, + 1.36, + 1.16, + 1.068 + ], + "unit": "milliseconds", + "wasm_path": "compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm", + "tested_code_commit": "34c759f91e4ff76c8b6d587208cbf4bea6a51ca0", + "tested_code_tree": "d084a402e25d02ded364d7747fc7f776aa2c50af", + "prior_pr_head": "aa2d9bdd598d32cb6fd19a30b9fb500f9c96563f", + "date_utc": "2026-10-18", + "rust": "rustc 1.99.0 (b940084d7 2026-09-28)", + "node": "v24.19.0", + "cpu_model_host_label": "AMD EPYC 9V74 80-Core Processor", + "visible_cpus": 9, + "wasm_bytes": 1138172, + "wasm_sha256": "5e5c7feb5599e134cb5a21eaf68b4e7bec674e05967b4667a15db3c3d6c18b81", + "native_sha256": "256696d7de0fe8fe83e27cf04fab61af68f6c50290f70b17b0b9889d534d149c", + "measurement_scope": "One reject fixture, native JSON capture plus shared strict envelope validation; cancellation-enabled WASM. Each case is one sequential batch. No editor or LSP transport. Cached runtime cases remain in the same process." +} diff --git a/spikes/w012-inprocess/validation-2026-10-19/README.md b/spikes/w012-inprocess/validation-2026-10-19/README.md new file mode 100644 index 00000000..eb5badc5 --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-19/README.md @@ -0,0 +1,45 @@ +# W-012 archived CI measurements — 2026-10-19 + +This directory preserves the original JSON measurements from [W-012 validation run 39195170609](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609). The three JSON sample files are byte-for-byte copies extracted from that run's artifacts. Archiving them did not rerun the benchmark or produce new timing samples. These results describe the historical source and tested merge below; they do not validate later code changes. + +## Provenance + +| Item | Recorded value | +| --- | --- | +| Pull request | [#556](https://github.com/limitlessprojects/Go-Hybrid/pull/556) | +| Source HEAD | `6dafd4973114fe35c77133db89c19828a0ad07b6` | +| Merge actually checked out by CI | `b49e0606be4746d90cb5399ebdb04166024676b5` | +| Base at that run | `aa4a9d91b5edecd2bff37eba23fdd7befe60bd56` | +| Measurement job | [analyzer-parity, 117876286217](https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609/job/117876286217) | +| Raw measurement timestamps | 2026-10-19 03:34:19–03:34:20 UTC | +| Host fields emitted by the harness | Linux, amd64, Go 1.26.9; 4 visible CPUs; `GOMAXPROCS=4` | + +The artifact API identifies the source HEAD; the checkout log identifies the tested merge and its base. Those three commits are distinct. [manifest.json](manifest.json) records the original artifact URLs, sizes, expiration times, archive SHA-256 values, and SHA-256 values of the extracted files. Both downloaded ZIP digests were checked against GitHub's artifact metadata before extraction. The sample files contain synthetic filenames, hashes, measurements, environment fields, and limitation text; inspection found no credentials or private filesystem paths. + +The original artifacts expire on **2026-10-26**, before the maintainer's [2026-11-01 M1 review window](https://github.com/limitlessprojects/Go-Hybrid/issues/268#issuecomment-6229962219). Keeping these raw files in the repository preserves their reviewability beyond that artifact expiration. + +## Files and observed results + +| File | Recorded workload | Purpose | +| --- | --- | --- | +| [w012-edit-benchmark.json](w012-edit-benchmark.json) | 25 files, 100 edits, warmup 10; 100 records with three path timings each | Warm request comparison and separate cold initialization | +| [w012-cold-cache-first.json](w012-cold-cache-first.json) | 2 files, 2 edits, warmup 1 | First Go process using the optional compilation-cache directory | +| [w012-cold-cache-second.json](w012-cold-cache-second.json) | 2 files, 2 edits, warmup 1 | A second Go process reusing that directory | + +For the 100-edit run, the recorded summary fields are: + +| Request path | p50 (ms) | p95 (ms) | p99 (ms) | +| --- | ---: | ---: | ---: | +| Native subprocess plus decode | 1.150408 | 1.282607 | 1.369949 | +| Public `Engine.Check` | 2.096157 | 2.777000 | 3.084019 | +| Experimental sequential reused guest | 1.450742 | 1.581058 | 1.718235 | + +The public WASM path was slower than the native path on this host. Its separately recorded cold initialization was **358.947876 ms**. The two-process compilation-cache experiment recorded public `GetEngine` cold initialization of **395.981092 ms** in the first process and **12.217370 ms** in the second. These two cache runs have only two request samples each; they are not substitutes for the 100-edit comparison. All three files record matching WASM and native binary hashes. + +## Interpretation limits + +The timer starts after each file write. Each request checks only the edited file. The native path includes a new subprocess, reading the file, and decoding its envelope; the embedded path receives source bytes already in memory. This is an after-write request proxy, with no whole-workspace reanalysis, unsaved-buffer integration, gopls/LSP, or editor keystroke-to-diagnostic measurement. + +The reused guest is a sequential experiment, not the public instance strategy or a production pool. Its cancellation, concurrency, and lifetime safety remain unproven. The compilation-cache result concerns cold initialization and does not establish faster hot checks. One fixture on one hosted Linux machine cannot establish population-level latency, production memory behavior, a Docker footprint reduction, or an M1 award. + +The raw files retain their emitted `Limitations` strings unchanged. In the two-file cache runs, the generic string referring to “all 25 workspace files” is inherited harness wording; the authoritative `Files` field is **2**. The earlier [2026-10-18 snapshot](../validation-2026-10-18/README.md) remains a separate historical experiment. diff --git a/spikes/w012-inprocess/validation-2026-10-19/manifest.json b/spikes/w012-inprocess/validation-2026-10-19/manifest.json new file mode 100644 index 00000000..d5afdb2a --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-19/manifest.json @@ -0,0 +1,65 @@ +{ + "schema_version": 1, + "record_kind": "archived GitHub Actions measurements", + "repository": "limitlessprojects/Go-Hybrid", + "pull_request_url": "https://github.com/limitlessprojects/Go-Hybrid/pull/556", + "workflow_run_id": 39195170609, + "workflow_run_url": "https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609", + "job_id": 117876286217, + "job_url": "https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609/job/117876286217", + "source_head_sha": "6dafd4973114fe35c77133db89c19828a0ad07b6", + "tested_merge_sha": "b49e0606be4746d90cb5399ebdb04166024676b5", + "then_base_sha": "aa4a9d91b5edecd2bff37eba23fdd7befe60bd56", + "archived_at_utc": "2026-10-19T06:09:41Z", + "raw_json_preserved_byte_for_byte": true, + "archive_digests_verified_against": "GitHub Actions artifact API digest metadata", + "archive_extraction": "Only the exact expected JSON basenames were accepted; symlinks, directories, traversal names, and oversized entries were rejected.", + "scope": "Historical measurements for the source and merge SHAs above; no rebenchmark or later-head validation is implied.", + "artifacts": [ + { + "id": 12161826108, + "name": "w012-request-latency-raw", + "archive_sha256": "f8ed7df03472f308427a8e6b6c0ddb55f4a16169d7b4363b7e29b22c08a7c41a", + "archive_size_bytes": 3429, + "created_at_utc": "2026-10-19T03:34:22Z", + "original_expires_at_utc": "2026-10-26T03:34:21Z", + "url": "https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609/artifacts/12161826108", + "api_url": "https://api.github.com/repos/limitlessprojects/Go-Hybrid/actions/artifacts/12161826108", + "files": [ + { + "path": "w012-edit-benchmark.json", + "size_bytes": 20617, + "sha256": "03120eecaab9d5d336e4e9ab76f12d7eb6ea9e6bf63023c0cb1f2e18cae1f064", + "sample_count": 100, + "measurement_timestamp_utc": "2026-10-19T03:34:19Z" + } + ] + }, + { + "id": 12161412052, + "name": "w012-cross-process-cache-timings", + "archive_sha256": "998c6f8ee0a4e7f7b9af5b359c08e15cc0dd1754628540c9c4c8094f23e2da97", + "archive_size_bytes": 2420, + "created_at_utc": "2026-10-19T03:34:21Z", + "original_expires_at_utc": "2026-10-26T03:34:20Z", + "url": "https://github.com/limitlessprojects/Go-Hybrid/actions/runs/39195170609/artifacts/12161412052", + "api_url": "https://api.github.com/repos/limitlessprojects/Go-Hybrid/actions/artifacts/12161412052", + "files": [ + { + "path": "w012-cold-cache-first.json", + "size_bytes": 2205, + "sha256": "e23f92a11279b8b317462efa7da5c3b6e0d50724181bce7288d57d44349aee62", + "sample_count": 2, + "measurement_timestamp_utc": "2026-10-19T03:34:20Z" + }, + { + "path": "w012-cold-cache-second.json", + "size_bytes": 2212, + "sha256": "6d76dd27868b3fb825215981e9a3485366494ee61805e5b736e15da8700b514a", + "sample_count": 2, + "measurement_timestamp_utc": "2026-10-19T03:34:20Z" + } + ] + } + ] +} diff --git a/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-first.json b/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-first.json new file mode 100644 index 00000000..5c6a489b --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-first.json @@ -0,0 +1,71 @@ +{ + "Kind": "single-file-after-write request proxy, NOT gopls/LSP", + "TimestampUTC": "2026-10-19T03:34:20Z", + "OS": "linux", + "Arch": "amd64", + "Go": "go1.26.9", + "CPUs": 4, + "GOMAXPROCS": 4, + "Files": 2, + "Edits": 2, + "Warmup": 1, + "FixtureSHA256": "972c48b086eb3182a721d7a73f8fc9604d1d292ff84148fcf9f04198f29da8ed", + "WASMSHA256": "62c3e2eea177b4356b37a75d9c3441e3b5920a9bc4fa1018210fa7efb689b978", + "NativeSHA256": "a7f1f9db61754cbcb77509adb3883d771b8d6a37816f5e62f257a0d3b93370d7", + "WASMBytes": 1662985, + "NativeBytes": 3990272, + "ColdWASMInitMS": 395.981092, + "ColdReusedInitMS": 370.033233, + "Native": { + "Min": 1.23074, + "P50": 1.23074, + "P90": 1.333644, + "P95": 1.333644, + "P99": 1.333644, + "Mean": 1.282192 + }, + "InProcess": { + "Min": 2.113272, + "P50": 2.113272, + "P90": 2.419997, + "P95": 2.419997, + "P99": 2.419997, + "Mean": 2.2666345 + }, + "ReusedGuest": { + "Min": 1.422479, + "P50": 1.422479, + "P90": 1.548429, + "P95": 1.548429, + "P99": 1.548429, + "Mean": 1.485454 + }, + "NativeOverInProcessP50": 0.5823859872273895, + "NativeOverReusedP50": 0.8652078519261092, + "Samples": [ + { + "Edit": 1, + "File": "file-000.hy", + "First": "native", + "NativeMS": 1.23074, + "InProcessMS": 2.113272, + "ExperimentalReusedMS": 1.422479 + }, + { + "Edit": 2, + "File": "file-001.hy", + "First": "embedded", + "NativeMS": 1.333644, + "InProcessMS": 2.419997, + "ExperimentalReusedMS": 1.548429 + } + ], + "Limitations": [ + "Not stock gopls, not unsaved buffers, not editor keystroke-to-diagnostic latency.", + "Each request checks the edited file only, not all 25 workspace files.", + "Durations begin after the write; native includes subprocess, file read and envelope decode; embedded receives in-memory source bytes.", + "Cold WASM initialization is separate from warm Engine.Check timings.", + "The experimental reused guest is strictly sequential and not a production instance pool; cancellation, concurrency and lifetime safety remain unproven.", + "One fixture and one CI machine are insufficient for population-level performance or memory claims." + ] +} diff --git a/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-second.json b/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-second.json new file mode 100644 index 00000000..3343caaa --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-19/w012-cold-cache-second.json @@ -0,0 +1,71 @@ +{ + "Kind": "single-file-after-write request proxy, NOT gopls/LSP", + "TimestampUTC": "2026-10-19T03:34:20Z", + "OS": "linux", + "Arch": "amd64", + "Go": "go1.26.9", + "CPUs": 4, + "GOMAXPROCS": 4, + "Files": 2, + "Edits": 2, + "Warmup": 1, + "FixtureSHA256": "972c48b086eb3182a721d7a73f8fc9604d1d292ff84148fcf9f04198f29da8ed", + "WASMSHA256": "62c3e2eea177b4356b37a75d9c3441e3b5920a9bc4fa1018210fa7efb689b978", + "NativeSHA256": "a7f1f9db61754cbcb77509adb3883d771b8d6a37816f5e62f257a0d3b93370d7", + "WASMBytes": 1662985, + "NativeBytes": 3990272, + "ColdWASMInitMS": 12.21737, + "ColdReusedInitMS": 363.861369, + "Native": { + "Min": 1.147383, + "P50": 1.147383, + "P90": 1.464782, + "P95": 1.464782, + "P99": 1.464782, + "Mean": 1.3060825 + }, + "InProcess": { + "Min": 2.239082, + "P50": 2.239082, + "P90": 2.33654, + "P95": 2.33654, + "P99": 2.33654, + "Mean": 2.2878109999999996 + }, + "ReusedGuest": { + "Min": 1.342928, + "P50": 1.342928, + "P90": 1.425403, + "P95": 1.425403, + "P99": 1.425403, + "Mean": 1.3841655 + }, + "NativeOverInProcessP50": 0.5124345602349535, + "NativeOverReusedP50": 0.854389066279056, + "Samples": [ + { + "Edit": 1, + "File": "file-000.hy", + "First": "native", + "NativeMS": 1.147383, + "InProcessMS": 2.239082, + "ExperimentalReusedMS": 1.425403 + }, + { + "Edit": 2, + "File": "file-001.hy", + "First": "embedded", + "NativeMS": 1.464782, + "InProcessMS": 2.33654, + "ExperimentalReusedMS": 1.342928 + } + ], + "Limitations": [ + "Not stock gopls, not unsaved buffers, not editor keystroke-to-diagnostic latency.", + "Each request checks the edited file only, not all 25 workspace files.", + "Durations begin after the write; native includes subprocess, file read and envelope decode; embedded receives in-memory source bytes.", + "Cold WASM initialization is separate from warm Engine.Check timings.", + "The experimental reused guest is strictly sequential and not a production instance pool; cancellation, concurrency and lifetime safety remain unproven.", + "One fixture and one CI machine are insufficient for population-level performance or memory claims." + ] +} diff --git a/spikes/w012-inprocess/validation-2026-10-19/w012-edit-benchmark.json b/spikes/w012-inprocess/validation-2026-10-19/w012-edit-benchmark.json new file mode 100644 index 00000000..a06dbb9d --- /dev/null +++ b/spikes/w012-inprocess/validation-2026-10-19/w012-edit-benchmark.json @@ -0,0 +1,855 @@ +{ + "Kind": "single-file-after-write request proxy, NOT gopls/LSP", + "TimestampUTC": "2026-10-19T03:34:19Z", + "OS": "linux", + "Arch": "amd64", + "Go": "go1.26.9", + "CPUs": 4, + "GOMAXPROCS": 4, + "Files": 25, + "Edits": 100, + "Warmup": 10, + "FixtureSHA256": "972c48b086eb3182a721d7a73f8fc9604d1d292ff84148fcf9f04198f29da8ed", + "WASMSHA256": "62c3e2eea177b4356b37a75d9c3441e3b5920a9bc4fa1018210fa7efb689b978", + "NativeSHA256": "a7f1f9db61754cbcb77509adb3883d771b8d6a37816f5e62f257a0d3b93370d7", + "WASMBytes": 1662985, + "NativeBytes": 3990272, + "ColdWASMInitMS": 358.947876, + "ColdReusedInitMS": 364.607811, + "Native": { + "Min": 1.053992, + "P50": 1.150408, + "P90": 1.223904, + "P95": 1.282607, + "P99": 1.369949, + "Mean": 1.1595342600000003 + }, + "InProcess": { + "Min": 1.892369, + "P50": 2.096157, + "P90": 2.370467, + "P95": 2.777, + "P99": 3.084019, + "Mean": 2.1839310800000002 + }, + "ReusedGuest": { + "Min": 1.346975, + "P50": 1.450742, + "P90": 1.544022, + "P95": 1.581058, + "P99": 1.718235, + "Mean": 1.4600795700000004 + }, + "NativeOverInProcessP50": 0.5488176696688274, + "NativeOverReusedP50": 0.792979041070018, + "Samples": [ + { + "Edit": 1, + "File": "file-000.hy", + "First": "native", + "NativeMS": 1.214329, + "InProcessMS": 2.491504, + "ExperimentalReusedMS": 1.704104 + }, + { + "Edit": 2, + "File": "file-001.hy", + "First": "embedded", + "NativeMS": 1.329739, + "InProcessMS": 2.486396, + "ExperimentalReusedMS": 1.718235 + }, + { + "Edit": 3, + "File": "file-002.hy", + "First": "experimental-reused", + "NativeMS": 1.203107, + "InProcessMS": 3.780891, + "ExperimentalReusedMS": 1.77394 + }, + { + "Edit": 4, + "File": "file-003.hy", + "First": "native", + "NativeMS": 1.232792, + "InProcessMS": 2.114675, + "ExperimentalReusedMS": 1.584153 + }, + { + "Edit": 5, + "File": "file-004.hy", + "First": "embedded", + "NativeMS": 1.369949, + "InProcessMS": 3.084019, + "ExperimentalReusedMS": 1.455368 + }, + { + "Edit": 6, + "File": "file-005.hy", + "First": "experimental-reused", + "NativeMS": 1.223904, + "InProcessMS": 2.735082, + "ExperimentalReusedMS": 1.426956 + }, + { + "Edit": 7, + "File": "file-006.hy", + "First": "native", + "NativeMS": 1.180663, + "InProcessMS": 2.031979, + "ExperimentalReusedMS": 1.506686 + }, + { + "Edit": 8, + "File": "file-007.hy", + "First": "embedded", + "NativeMS": 1.316879, + "InProcessMS": 1.984868, + "ExperimentalReusedMS": 1.452504 + }, + { + "Edit": 9, + "File": "file-008.hy", + "First": "experimental-reused", + "NativeMS": 1.130824, + "InProcessMS": 2.91945, + "ExperimentalReusedMS": 1.462179 + }, + { + "Edit": 10, + "File": "file-009.hy", + "First": "native", + "NativeMS": 1.128369, + "InProcessMS": 1.996606, + "ExperimentalReusedMS": 1.491333 + }, + { + "Edit": 11, + "File": "file-010.hy", + "First": "embedded", + "NativeMS": 1.19193, + "InProcessMS": 2.05873, + "ExperimentalReusedMS": 1.391792 + }, + { + "Edit": 12, + "File": "file-011.hy", + "First": "experimental-reused", + "NativeMS": 1.136061, + "InProcessMS": 3.042461, + "ExperimentalReusedMS": 1.398363 + }, + { + "Edit": 13, + "File": "file-012.hy", + "First": "native", + "NativeMS": 1.149657, + "InProcessMS": 2.023677, + "ExperimentalReusedMS": 1.556831 + }, + { + "Edit": 14, + "File": "file-013.hy", + "First": "embedded", + "NativeMS": 1.147803, + "InProcessMS": 2.042466, + "ExperimentalReusedMS": 1.493536 + }, + { + "Edit": 15, + "File": "file-014.hy", + "First": "experimental-reused", + "NativeMS": 1.123707, + "InProcessMS": 2.260194, + "ExperimentalReusedMS": 1.467056 + }, + { + "Edit": 16, + "File": "file-015.hy", + "First": "native", + "NativeMS": 1.134303, + "InProcessMS": 2.062296, + "ExperimentalReusedMS": 1.481598 + }, + { + "Edit": 17, + "File": "file-016.hy", + "First": "embedded", + "NativeMS": 1.203128, + "InProcessMS": 1.985118, + "ExperimentalReusedMS": 1.444762 + }, + { + "Edit": 18, + "File": "file-017.hy", + "First": "experimental-reused", + "NativeMS": 1.153833, + "InProcessMS": 2.163302, + "ExperimentalReusedMS": 1.46309 + }, + { + "Edit": 19, + "File": "file-018.hy", + "First": "native", + "NativeMS": 1.093261, + "InProcessMS": 2.019541, + "ExperimentalReusedMS": 1.405353 + }, + { + "Edit": 20, + "File": "file-019.hy", + "First": "embedded", + "NativeMS": 1.173268, + "InProcessMS": 1.983036, + "ExperimentalReusedMS": 1.358442 + }, + { + "Edit": 21, + "File": "file-020.hy", + "First": "experimental-reused", + "NativeMS": 1.185996, + "InProcessMS": 2.270019, + "ExperimentalReusedMS": 1.468157 + }, + { + "Edit": 22, + "File": "file-021.hy", + "First": "native", + "NativeMS": 1.14789, + "InProcessMS": 2.063747, + "ExperimentalReusedMS": 1.497502 + }, + { + "Edit": 23, + "File": "file-022.hy", + "First": "embedded", + "NativeMS": 1.190018, + "InProcessMS": 2.129958, + "ExperimentalReusedMS": 1.536991 + }, + { + "Edit": 24, + "File": "file-023.hy", + "First": "experimental-reused", + "NativeMS": 1.111759, + "InProcessMS": 2.423571, + "ExperimentalReusedMS": 1.485464 + }, + { + "Edit": 25, + "File": "file-024.hy", + "First": "native", + "NativeMS": 1.126141, + "InProcessMS": 2.100593, + "ExperimentalReusedMS": 1.504924 + }, + { + "Edit": 26, + "File": "file-000.hy", + "First": "embedded", + "NativeMS": 1.230464, + "InProcessMS": 2.017688, + "ExperimentalReusedMS": 1.445523 + }, + { + "Edit": 27, + "File": "file-001.hy", + "First": "experimental-reused", + "NativeMS": 1.154529, + "InProcessMS": 2.145435, + "ExperimentalReusedMS": 1.474057 + }, + { + "Edit": 28, + "File": "file-002.hy", + "First": "native", + "NativeMS": 1.106732, + "InProcessMS": 2.097348, + "ExperimentalReusedMS": 1.426725 + }, + { + "Edit": 29, + "File": "file-003.hy", + "First": "embedded", + "NativeMS": 1.179726, + "InProcessMS": 2.029325, + "ExperimentalReusedMS": 1.419444 + }, + { + "Edit": 30, + "File": "file-004.hy", + "First": "experimental-reused", + "NativeMS": 1.173472, + "InProcessMS": 2.34321, + "ExperimentalReusedMS": 1.375327 + }, + { + "Edit": 31, + "File": "file-005.hy", + "First": "native", + "NativeMS": 1.157008, + "InProcessMS": 2.062526, + "ExperimentalReusedMS": 1.544093 + }, + { + "Edit": 32, + "File": "file-006.hy", + "First": "embedded", + "NativeMS": 1.240665, + "InProcessMS": 2.062676, + "ExperimentalReusedMS": 1.418623 + }, + { + "Edit": 33, + "File": "file-007.hy", + "First": "experimental-reused", + "NativeMS": 1.481127, + "InProcessMS": 2.777, + "ExperimentalReusedMS": 1.445013 + }, + { + "Edit": 34, + "File": "file-008.hy", + "First": "native", + "NativeMS": 1.285818, + "InProcessMS": 2.218181, + "ExperimentalReusedMS": 1.629782 + }, + { + "Edit": 35, + "File": "file-009.hy", + "First": "embedded", + "NativeMS": 1.282607, + "InProcessMS": 2.096157, + "ExperimentalReusedMS": 1.4627 + }, + { + "Edit": 36, + "File": "file-010.hy", + "First": "experimental-reused", + "NativeMS": 1.192451, + "InProcessMS": 2.30132, + "ExperimentalReusedMS": 1.487226 + }, + { + "Edit": 37, + "File": "file-011.hy", + "First": "native", + "NativeMS": 1.204129, + "InProcessMS": 2.054684, + "ExperimentalReusedMS": 1.475759 + }, + { + "Edit": 38, + "File": "file-012.hy", + "First": "embedded", + "NativeMS": 1.251982, + "InProcessMS": 2.04594, + "ExperimentalReusedMS": 1.434076 + }, + { + "Edit": 39, + "File": "file-013.hy", + "First": "experimental-reused", + "NativeMS": 1.17842, + "InProcessMS": 2.361073, + "ExperimentalReusedMS": 1.409649 + }, + { + "Edit": 40, + "File": "file-014.hy", + "First": "native", + "NativeMS": 1.163368, + "InProcessMS": 2.060753, + "ExperimentalReusedMS": 1.54207 + }, + { + "Edit": 41, + "File": "file-015.hy", + "First": "embedded", + "NativeMS": 1.194488, + "InProcessMS": 2.037508, + "ExperimentalReusedMS": 1.481808 + }, + { + "Edit": 42, + "File": "file-016.hy", + "First": "experimental-reused", + "NativeMS": 1.194214, + "InProcessMS": 2.91245, + "ExperimentalReusedMS": 1.456941 + }, + { + "Edit": 43, + "File": "file-017.hy", + "First": "native", + "NativeMS": 1.17819, + "InProcessMS": 2.168736, + "ExperimentalReusedMS": 1.541359 + }, + { + "Edit": 44, + "File": "file-018.hy", + "First": "embedded", + "NativeMS": 1.190983, + "InProcessMS": 2.186202, + "ExperimentalReusedMS": 1.494507 + }, + { + "Edit": 45, + "File": "file-019.hy", + "First": "experimental-reused", + "NativeMS": 1.140698, + "InProcessMS": 2.345974, + "ExperimentalReusedMS": 1.39705 + }, + { + "Edit": 46, + "File": "file-020.hy", + "First": "native", + "NativeMS": 1.16586, + "InProcessMS": 2.185391, + "ExperimentalReusedMS": 1.473666 + }, + { + "Edit": 47, + "File": "file-021.hy", + "First": "embedded", + "NativeMS": 1.159843, + "InProcessMS": 2.032591, + "ExperimentalReusedMS": 1.394507 + }, + { + "Edit": 48, + "File": "file-022.hy", + "First": "experimental-reused", + "NativeMS": 1.117888, + "InProcessMS": 2.14836, + "ExperimentalReusedMS": 1.538805 + }, + { + "Edit": 49, + "File": "file-023.hy", + "First": "native", + "NativeMS": 1.150408, + "InProcessMS": 2.069186, + "ExperimentalReusedMS": 1.453335 + }, + { + "Edit": 50, + "File": "file-024.hy", + "First": "embedded", + "NativeMS": 1.122641, + "InProcessMS": 2.050428, + "ExperimentalReusedMS": 1.44364 + }, + { + "Edit": 51, + "File": "file-000.hy", + "First": "experimental-reused", + "NativeMS": 1.154243, + "InProcessMS": 2.233233, + "ExperimentalReusedMS": 1.449339 + }, + { + "Edit": 52, + "File": "file-001.hy", + "First": "native", + "NativeMS": 1.100733, + "InProcessMS": 2.051138, + "ExperimentalReusedMS": 1.470872 + }, + { + "Edit": 53, + "File": "file-002.hy", + "First": "embedded", + "NativeMS": 1.087883, + "InProcessMS": 2.038249, + "ExperimentalReusedMS": 1.486726 + }, + { + "Edit": 54, + "File": "file-003.hy", + "First": "experimental-reused", + "NativeMS": 1.163783, + "InProcessMS": 2.186677, + "ExperimentalReusedMS": 1.418633 + }, + { + "Edit": 55, + "File": "file-004.hy", + "First": "native", + "NativeMS": 1.137749, + "InProcessMS": 2.041704, + "ExperimentalReusedMS": 1.480637 + }, + { + "Edit": 56, + "File": "file-005.hy", + "First": "embedded", + "NativeMS": 1.147364, + "InProcessMS": 2.05228, + "ExperimentalReusedMS": 1.436691 + }, + { + "Edit": 57, + "File": "file-006.hy", + "First": "experimental-reused", + "NativeMS": 1.139212, + "InProcessMS": 2.215081, + "ExperimentalReusedMS": 1.38356 + }, + { + "Edit": 58, + "File": "file-007.hy", + "First": "native", + "NativeMS": 1.192445, + "InProcessMS": 2.051178, + "ExperimentalReusedMS": 1.378292 + }, + { + "Edit": 59, + "File": "file-008.hy", + "First": "embedded", + "NativeMS": 1.128044, + "InProcessMS": 2.103177, + "ExperimentalReusedMS": 1.354196 + }, + { + "Edit": 60, + "File": "file-009.hy", + "First": "experimental-reused", + "NativeMS": 1.157669, + "InProcessMS": 2.300341, + "ExperimentalReusedMS": 1.412223 + }, + { + "Edit": 61, + "File": "file-010.hy", + "First": "native", + "NativeMS": 1.160597, + "InProcessMS": 2.12506, + "ExperimentalReusedMS": 1.390159 + }, + { + "Edit": 62, + "File": "file-011.hy", + "First": "embedded", + "NativeMS": 1.118175, + "InProcessMS": 2.124119, + "ExperimentalReusedMS": 1.483041 + }, + { + "Edit": 63, + "File": "file-012.hy", + "First": "experimental-reused", + "NativeMS": 1.119641, + "InProcessMS": 2.341762, + "ExperimentalReusedMS": 1.508028 + }, + { + "Edit": 64, + "File": "file-013.hy", + "First": "native", + "NativeMS": 1.145611, + "InProcessMS": 2.119542, + "ExperimentalReusedMS": 1.48914 + }, + { + "Edit": 65, + "File": "file-014.hy", + "First": "embedded", + "NativeMS": 1.158079, + "InProcessMS": 2.057317, + "ExperimentalReusedMS": 1.504302 + }, + { + "Edit": 66, + "File": "file-015.hy", + "First": "experimental-reused", + "NativeMS": 1.162516, + "InProcessMS": 2.299453, + "ExperimentalReusedMS": 1.544022 + }, + { + "Edit": 67, + "File": "file-016.hy", + "First": "native", + "NativeMS": 1.108534, + "InProcessMS": 2.123077, + "ExperimentalReusedMS": 1.505404 + }, + { + "Edit": 68, + "File": "file-017.hy", + "First": "embedded", + "NativeMS": 1.109396, + "InProcessMS": 2.023496, + "ExperimentalReusedMS": 1.566216 + }, + { + "Edit": 69, + "File": "file-018.hy", + "First": "experimental-reused", + "NativeMS": 1.185396, + "InProcessMS": 2.258942, + "ExperimentalReusedMS": 1.487356 + }, + { + "Edit": 70, + "File": "file-019.hy", + "First": "native", + "NativeMS": 1.21171, + "InProcessMS": 2.056917, + "ExperimentalReusedMS": 1.581058 + }, + { + "Edit": 71, + "File": "file-020.hy", + "First": "embedded", + "NativeMS": 1.154363, + "InProcessMS": 2.039081, + "ExperimentalReusedMS": 1.395809 + }, + { + "Edit": 72, + "File": "file-021.hy", + "First": "experimental-reused", + "NativeMS": 1.15228, + "InProcessMS": 2.370467, + "ExperimentalReusedMS": 1.447327 + }, + { + "Edit": 73, + "File": "file-022.hy", + "First": "native", + "NativeMS": 1.130237, + "InProcessMS": 2.130879, + "ExperimentalReusedMS": 1.406224 + }, + { + "Edit": 74, + "File": "file-023.hy", + "First": "embedded", + "NativeMS": 1.128976, + "InProcessMS": 2.084189, + "ExperimentalReusedMS": 1.427196 + }, + { + "Edit": 75, + "File": "file-024.hy", + "First": "experimental-reused", + "NativeMS": 1.149738, + "InProcessMS": 2.299805, + "ExperimentalReusedMS": 1.366204 + }, + { + "Edit": 76, + "File": "file-000.hy", + "First": "native", + "NativeMS": 1.215395, + "InProcessMS": 2.101504, + "ExperimentalReusedMS": 1.371973 + }, + { + "Edit": 77, + "File": "file-001.hy", + "First": "embedded", + "NativeMS": 1.154464, + "InProcessMS": 1.993782, + "ExperimentalReusedMS": 1.354446 + }, + { + "Edit": 78, + "File": "file-002.hy", + "First": "experimental-reused", + "NativeMS": 1.10585, + "InProcessMS": 2.291257, + "ExperimentalReusedMS": 1.346975 + }, + { + "Edit": 79, + "File": "file-003.hy", + "First": "native", + "NativeMS": 1.098739, + "InProcessMS": 2.048665, + "ExperimentalReusedMS": 1.422469 + }, + { + "Edit": 80, + "File": "file-004.hy", + "First": "embedded", + "NativeMS": 1.134644, + "InProcessMS": 2.00586, + "ExperimentalReusedMS": 1.459104 + }, + { + "Edit": 81, + "File": "file-005.hy", + "First": "experimental-reused", + "NativeMS": 1.120153, + "InProcessMS": 2.182847, + "ExperimentalReusedMS": 1.425493 + }, + { + "Edit": 82, + "File": "file-006.hy", + "First": "native", + "NativeMS": 1.14556, + "InProcessMS": 2.010797, + "ExperimentalReusedMS": 1.447807 + }, + { + "Edit": 83, + "File": "file-007.hy", + "First": "embedded", + "NativeMS": 1.075589, + "InProcessMS": 2.059742, + "ExperimentalReusedMS": 1.373125 + }, + { + "Edit": 84, + "File": "file-008.hy", + "First": "experimental-reused", + "NativeMS": 1.082546, + "InProcessMS": 2.058905, + "ExperimentalReusedMS": 1.386744 + }, + { + "Edit": 85, + "File": "file-009.hy", + "First": "native", + "NativeMS": 1.106692, + "InProcessMS": 1.994863, + "ExperimentalReusedMS": 1.434126 + }, + { + "Edit": 86, + "File": "file-010.hy", + "First": "embedded", + "NativeMS": 1.070667, + "InProcessMS": 1.965509, + "ExperimentalReusedMS": 1.43023 + }, + { + "Edit": 87, + "File": "file-011.hy", + "First": "experimental-reused", + "NativeMS": 1.053992, + "InProcessMS": 2.120988, + "ExperimentalReusedMS": 1.459575 + }, + { + "Edit": 88, + "File": "file-012.hy", + "First": "native", + "NativeMS": 1.074894, + "InProcessMS": 1.943576, + "ExperimentalReusedMS": 1.375168 + }, + { + "Edit": 89, + "File": "file-013.hy", + "First": "embedded", + "NativeMS": 1.123929, + "InProcessMS": 1.9724, + "ExperimentalReusedMS": 1.359914 + }, + { + "Edit": 90, + "File": "file-014.hy", + "First": "experimental-reused", + "NativeMS": 1.131299, + "InProcessMS": 2.342789, + "ExperimentalReusedMS": 1.450742 + }, + { + "Edit": 91, + "File": "file-015.hy", + "First": "native", + "NativeMS": 1.099671, + "InProcessMS": 1.964748, + "ExperimentalReusedMS": 1.394557 + }, + { + "Edit": 92, + "File": "file-016.hy", + "First": "embedded", + "NativeMS": 1.092381, + "InProcessMS": 1.949024, + "ExperimentalReusedMS": 1.401747 + }, + { + "Edit": 93, + "File": "file-017.hy", + "First": "experimental-reused", + "NativeMS": 1.090096, + "InProcessMS": 2.240439, + "ExperimentalReusedMS": 1.42984 + }, + { + "Edit": 94, + "File": "file-018.hy", + "First": "native", + "NativeMS": 1.093091, + "InProcessMS": 2.044428, + "ExperimentalReusedMS": 1.434818 + }, + { + "Edit": 95, + "File": "file-019.hy", + "First": "embedded", + "NativeMS": 1.201144, + "InProcessMS": 2.02569, + "ExperimentalReusedMS": 1.452865 + }, + { + "Edit": 96, + "File": "file-020.hy", + "First": "experimental-reused", + "NativeMS": 1.180918, + "InProcessMS": 2.28, + "ExperimentalReusedMS": 1.571504 + }, + { + "Edit": 97, + "File": "file-021.hy", + "First": "native", + "NativeMS": 1.155139, + "InProcessMS": 1.956396, + "ExperimentalReusedMS": 1.452004 + }, + { + "Edit": 98, + "File": "file-022.hy", + "First": "embedded", + "NativeMS": 1.09156, + "InProcessMS": 1.892369, + "ExperimentalReusedMS": 1.373525 + }, + { + "Edit": 99, + "File": "file-023.hy", + "First": "experimental-reused", + "NativeMS": 1.073221, + "InProcessMS": 2.191349, + "ExperimentalReusedMS": 1.419424 + }, + { + "Edit": 100, + "File": "file-024.hy", + "First": "native", + "NativeMS": 1.080397, + "InProcessMS": 1.963696, + "ExperimentalReusedMS": 1.412123 + } + ], + "Limitations": [ + "Not stock gopls, not unsaved buffers, not editor keystroke-to-diagnostic latency.", + "Each request checks the edited file only, not all 25 workspace files.", + "Durations begin after the write; native includes subprocess, file read and envelope decode; embedded receives in-memory source bytes.", + "Cold WASM initialization is separate from warm Engine.Check timings.", + "The experimental reused guest is strictly sequential and not a production instance pool; cancellation, concurrency and lifetime safety remain unproven.", + "One fixture and one CI machine are insufficient for population-level performance or memory claims." + ] +} diff --git a/test_wasm_parity.js b/test_wasm_parity.js new file mode 100644 index 00000000..8b1a3afa --- /dev/null +++ b/test_wasm_parity.js @@ -0,0 +1,131 @@ +const fs = require('fs'); +const path = require('path'); +const {spawnSync} = require('child_process'); +const assert = require('assert'); + +async function main() { + const wasmPath = process.env.HYBRID_WASM_PATH || path.join(__dirname, 'compiler/target/wasm32-unknown-unknown/release/hy_wasm.wasm'); + const nativePath = process.env.HYBRID_NATIVE_PATH || path.join(__dirname, 'compiler/target/release/hyb'); + const wasmBytes = fs.readFileSync(wasmPath); + + console.log(`Loaded WASM: ${wasmBytes.length} bytes (${(wasmBytes.length / 1024).toFixed(1)} KB)`); + + const tInit0 = performance.now(); + const wasmModule = await WebAssembly.instantiate(wasmBytes, {}); + const tInit1 = performance.now(); + console.log(`WASM instantiation time: ${(tInit1 - tInit0).toFixed(2)} ms`); + + const { hy_wasm_alloc, hy_wasm_free, hy_wasm_check, memory } = wasmModule.instance.exports; + + function checkSource(srcText) { + const encoder = new TextEncoder(); + const encoded = encoder.encode(srcText); + + const srcPtr = hy_wasm_alloc(encoded.length); + const memU8 = new Uint8Array(memory.buffer); + memU8.set(encoded, srcPtr); + + const packed = BigInt(hy_wasm_check(srcPtr, encoded.length)); + hy_wasm_free(srcPtr, encoded.length); + + const outPtr = Number(packed >> 32n); + const outLen = Number(packed & 0xFFFFFFFFn); + + const outSlice = new Uint8Array(memory.buffer, outPtr, outLen); + const decoder = new TextDecoder(); + // Decode before returning ownership of the output buffer to the guest. + let jsonStr; + try { + jsonStr = decoder.decode(outSlice); + } finally { + hy_wasm_free(outPtr, outLen); + } + return JSON.parse(jsonStr); + } + + // Test a sample accept file + const sampleAccept = fs.readFileSync(path.join(__dirname, 'tests/checker-corpus/16-move-send-clean.hy'), 'utf8'); + const resAccept = checkSource(sampleAccept); + console.log('Sample Accept verdict:', resAccept.summary.verdict, 'errors:', resAccept.summary.errors); + + // Test a sample reject file + const sampleReject = fs.readFileSync(path.join(__dirname, 'tests/checker-corpus/01-shared-map-across-tasks.hy'), 'utf8'); + const resReject = checkSource(sampleReject); + console.log('Sample Reject verdict:', resReject.summary.verdict, 'errors:', resReject.summary.errors, 'rule:', resReject.diagnostics[0]?.tier0_rule); + + // Test across all corpus files and measure time + const corpusDir = path.join(__dirname, 'tests/checker-corpus'); + const files = fs.readdirSync(corpusDir).filter(f => f.endsWith('.hy')).sort(); + + console.log(`\nVerifying all ${files.length} corpus files against WASM...`); + let passed = 0; + let nativeMatches = 0; + const times = []; + + for (const f of files) { + const fullPath = path.join(corpusDir, f); + const src = fs.readFileSync(fullPath, 'utf8'); + const m = src.match(/^\/\/\s*EXPECTED-VERDICT:\s*(\S+)\b\s*(\S*)/m); + const expectedVerdict = m ? m[1] : null; + const expectedRule = (m && m[2]) ? m[2] : null; + + const t0 = performance.now(); + const res = checkSource(src); + const t1 = performance.now(); + times.push(t1 - t0); + + // Compare complete parsed envelopes after normalizing only the input + // filename (the WASM ABI currently calls its input "input.hy"). + const native = spawnSync(nativePath, ['check', fullPath, '--json'], {encoding:'utf8'}); + if (native.error || ![0,1].includes(native.status)) throw native.error || new Error(`native CLI failed: ${native.status}: ${native.stderr}`); + const nativeOutput = JSON.parse(native.stdout); + function normalize(value) { + if (typeof value === 'string') return value.split(fullPath).join('input.hy'); + if (Array.isArray(value)) return value.map(normalize); + if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(k => [k, normalize(value[k])])); + return value; + } + assert.deepStrictEqual(normalize(res), normalize(nativeOutput), `native/WASM diagnostic mismatch: ${f}`); + nativeMatches++; + + const verdict = res.summary ? res.summary.verdict : res.verdict; + const errors = res.summary ? res.summary.errors : res.errors; + + let match = false; + if (expectedVerdict === 'ACCEPT') { + match = verdict === 'accept' && errors === 0; + } else if (expectedVerdict === 'ACCEPT-MANAGED') { + match = verdict === 'accept' && errors === 0; + } else if (expectedVerdict === 'REJECT') { + const firstErr = res.diagnostics.find(d => d.severity === 'error'); + const ruleOk = expectedRule ? (firstErr && firstErr.tier0_rule === expectedRule) : true; + match = verdict === 'reject' && ruleOk; + } + + if (match) { + passed++; + } else { + console.error(`FAIL ${f}: expected ${expectedVerdict} (${expectedRule}), got ${res.verdict}`); + } + } + + times.sort((a, b) => a - b); + const mean = times.reduce((a, b) => a + b, 0) / times.length; + const p50 = times[Math.floor(times.length / 2)]; + const p90 = times[Math.floor(times.length * 0.9)]; + const p99 = times[times.length - 1]; + + console.log(`Corpus fixture agreement: ${passed}/${files.length}`); + console.log(`Native/WASM normalized envelope parity: ${nativeMatches}/${files.length}`); + if (files.length === 0 || passed !== files.length || nativeMatches !== files.length) process.exitCode = 1; + console.log(`WASM Check Latency per file:`); + console.log(` mean: ${mean.toFixed(3)} ms`); + console.log(` p50: ${p50.toFixed(3)} ms`); + console.log(` p90: ${p90.toFixed(3)} ms`); + console.log(` p99: ${p99.toFixed(3)} ms`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +});