From 8d7466faa5199fe667f367bd8eb1806c04d7e2ac Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 01:55:54 +0000 Subject: [PATCH 01/15] chore: bring deploy/reblame-c100 into master deploy/reblame-c100 holds the code that built the -C100 corpus. Its last commit, b558112, is tagged corpus-c100-2026-10; its 33 commits stay reachable through that tag. Master and deploy split at 72f26d0 on 2026-09-01, and 34 files conflicted. This commit is the merge of the two lines, squashed, because the repository allows no merge commits. Resolution rules: 1. Token bytes, tokenizer identity, exit status and exclusion, blame output and the dataset: deploy wins. srcML stays at the v1.1.0 tag (#101), not the develop pin of #93. tokenize.pl, tokenizeSrcMl.pl, CregitLanguages.pm, CregitSrcMl.pm and the Rust main.rs keep deploy's bytes, because the identity hashes them or the binaries built from them. blobExec/src is deploy's (#97, #98, #100, #101, --retokenize, the 209-blob TSV denylist). shellcheck stays out of devenv `packages`: a new package changes the shell env in srcml2token's RUNPATH, so the identity too. 2. A fix made on both lines is kept once: the -C100 blame fix, the trailer and provenance fixes, the workdir and memo guards, the sharded re-fold. Tests of both sides are kept where they differ. 3. Master's changes that leave those outputs alone are kept: Cachix in CI, the tests/ layout, devenv test, blame --porcelain and largest-first dispatch, the runner helper functions, and the comment, usage and README cuts. Decisions: - Deploy: no crash-rate gate (861c153). A crashed or timed-out blob is excluded and the run exits 0 (#100). - Deploy: .C and .H tokenize as C (b2c589c dropped). - Deploy: --ensure-artifacts rebuilds only a stale Rust tokenizer. - Deploy: --mask-widened stays. - Master: the stall window has a floor of one child's life (--blob-timeout + 90 s); a lower --stall-timeout is refused (634be5f, 1e22e9d). - Master: the runner exits with blobExec's resumable status 4, 5 or 6 after it writes the marker; other failures exit 1. - Dropped master refactors written against code that #100 and #101 changed: ChildRunner, Saturating, BlobDenylistEntries, TimeoutRetrySpec. Token bytes and the tokenizer identity equal the tag's on 974 files (fixtures, 706 C/C++/Java files, 252 Rust files), on the default path and in the worker. Blame output equals the tag's on jq, cregit and a move and rename fixture. --- .github/workflows/ci.yml | 38 +- README.md | 98 ++- blameRepo/blameRepoFiles.pl | 7 +- blobExec/shard_build.sh | 4 +- .../cregit/blobexec/blob-denylist.tsv | 498 +++++++++++++ .../scala/cregit/blobexec/BlobDenylist.scala | 105 ++- .../cregit/blobexec/BlobDenylistEntries.scala | 27 - .../main/scala/cregit/blobexec/BlobExec.scala | 330 +++++++-- .../scala/cregit/blobexec/ChildRunner.scala | 114 --- .../src/main/scala/cregit/blobexec/Main.scala | 496 ++++++++++--- .../main/scala/cregit/blobexec/Mapping.scala | 661 +++++++++++++++++- .../scala/cregit/blobexec/Saturating.scala | 11 - .../cregit/blobexec/TokenizerIdentity.scala | 98 +++ .../scala/cregit/blobexec/TokenizerMemo.scala | 91 +++ .../cregit/blobexec/TokenizerWorkerPool.scala | 142 ++++ .../main/scala/cregit/blobexec/Walker.scala | 435 ++++++------ .../cregit/blobexec/BlobDenylistSpec.scala | 279 +++++++- .../scala/cregit/blobexec/BlobExecSpec.scala | 96 ++- .../cregit/blobexec/FailedBlobSpec.scala | 161 +++++ .../cregit/blobexec/MainOptionsSpec.scala | 162 ++--- .../scala/cregit/blobexec/MappingSpec.scala | 302 +++++++- .../cregit/blobexec/RetokenizeSpec.scala | 482 +++++++++++++ .../cregit/blobexec/TimeoutRetrySpec.scala | 179 ----- .../blobexec/TokenizerIdentitySpec.scala | 69 ++ .../cregit/blobexec/TokenizerMemoSpec.scala | 151 ++++ .../blobexec/TokenizerWorkerPoolSpec.scala | 136 ++++ .../blobexec/WalkerIntegrationSpec.scala | 316 ++++++++- devenv.nix | 10 +- generate_dataset/DATASET.md | 59 +- generate_dataset/generate_dataset.py | 233 +++++- nix/srcml.nix | 27 +- run_pipeline_process.sh | 339 ++++++++- tests/ensure_artifacts.sh | 297 ++++++++ tests/fixtures/Tiny.java | 4 + tests/fixtures/Utf8.java | 3 + tests/fixtures/empty.c | 0 tests/fixtures/latin1.c | 1 + tests/fixtures/tiny.c | 1 + tests/fixtures/tiny.cpp | 2 + tests/fixtures/tiny.rs | 3 + tests/fixtures/upper.H | 6 + tests/fixtures/utf8.c | 2 + tests/pipeline_workdir_guard.sh | 75 ++ tests/reblame_passthrough.sh | 165 +++++ tests/retokenize_passthrough.sh | 247 +++++++ tests/t/languages.t | 14 +- tests/t/tokenizeSrcMl.t | 21 +- tests/t/tokenizerIdentity.t | 218 ++++++ tests/test_generate_dataset.py | 498 ++++++++++++- tests/test_token_worker.sh | 19 + tests/token_worker_driver.py | 184 +++++ tests/tokenize_gate.sh | 3 +- tokenize/CregitLanguages.pm | 108 ++- tokenize/CregitSrcMl.pm | 225 ++++++ tokenize/rustTokenizer/src/main.rs | 49 +- tokenize/srcMLtoken/Makefile | 36 +- tokenize/srcMLtoken/README.md | 26 +- tokenize/srcMLtoken/srcml2token.cpp | 156 +++-- tokenize/tokenize.pl | 12 +- tokenize/tokenizeSrcMl.pl | 124 +--- tokenize/tokenizerIdentity.pl | 173 +++++ tokenizeByBlobId/WORKER_PROTOCOL.md | 43 ++ tokenizeByBlobId/tokenBySha.pl | 8 +- tokenizeByBlobId/tokenWorker.pl | 160 +++++ 64 files changed, 7761 insertions(+), 1278 deletions(-) create mode 100644 blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv delete mode 100644 blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala delete mode 100644 blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala delete mode 100644 blobExec/src/main/scala/cregit/blobexec/Saturating.scala create mode 100644 blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala create mode 100644 blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala create mode 100644 blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala create mode 100644 blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala create mode 100644 blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala delete mode 100644 blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala create mode 100644 blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala create mode 100644 blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala create mode 100644 blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala create mode 100755 tests/ensure_artifacts.sh create mode 100644 tests/fixtures/Tiny.java create mode 100644 tests/fixtures/Utf8.java create mode 100644 tests/fixtures/empty.c create mode 100644 tests/fixtures/latin1.c create mode 100644 tests/fixtures/tiny.c create mode 100644 tests/fixtures/tiny.cpp create mode 100644 tests/fixtures/tiny.rs create mode 100644 tests/fixtures/upper.H create mode 100644 tests/fixtures/utf8.c create mode 100755 tests/reblame_passthrough.sh create mode 100755 tests/retokenize_passthrough.sh create mode 100644 tests/t/tokenizerIdentity.t create mode 100755 tests/test_token_worker.sh create mode 100755 tests/token_worker_driver.py create mode 100644 tokenize/CregitSrcMl.pm mode change 100644 => 100755 tokenize/tokenize.pl create mode 100755 tokenize/tokenizerIdentity.pl create mode 100644 tokenizeByBlobId/WORKER_PROTOCOL.md mode change 100644 => 100755 tokenizeByBlobId/tokenBySha.pl create mode 100755 tokenizeByBlobId/tokenWorker.pl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 161affb5f..94d48cc1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,14 +76,48 @@ jobs: - name: Build & test Rust tokenizer (rust_tokenizer) run: devenv shell -- bash -c 'cd tokenize/rustTokenizer && make && make test' + - name: "Tokenizer worker: same output as tokenBySha.pl" + run: devenv shell -- bash tests/test_token_worker.sh + - name: Perl tests (prove) run: devenv shell -- bash -c 'prove tests/t' - name: Pipeline tests (shell) - run: devenv shell -- bash -c 'tests/tokenize_gate.sh && tests/pipeline_workdir_guard.sh' + run: | + devenv shell -- bash -c 'set -e + for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + bash "tests/$t.sh" + done' - name: Dataset tests (pytest) - run: devenv shell -- bash -c 'pytest -q' + run: devenv shell -- pytest -q --junitxml=junit-generate-dataset.xml + + # `devenv shell -- pytest` does not flush its summary line, so read the counts + # from the JUnit XML. Standard library only, so it needs no devenv shell. + - name: Re-check the pytest JUnit counts + if: always() + shell: bash + run: | + f=junit-generate-dataset.xml + [ -f "$f" ] || { echo "no $f: pytest did not get far enough to write one"; exit 1; } + python3 - "$f" <<'PY' + import sys, xml.etree.ElementTree as ET + root = ET.parse(sys.argv[1]).getroot() + s = root.find("testsuite") if root.tag == "testsuites" else root + a = s.attrib + print(f"tests={a['tests']} failures={a['failures']} " + f"errors={a['errors']} skipped={a['skipped']}") + if int(a["failures"]) or int(a["errors"]): + sys.exit("failures or errors in the suite") + PY + + - name: Upload the pytest JUnit report + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-generate-dataset + path: junit-generate-dataset.xml + if-no-files-found: warn - name: Build & test legacy onejar modules (sbt test one-jar, Scala 2.10 / JDK 8) run: | diff --git a/README.md b/README.md index d3a6c95fe..1bdbbfb32 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ HTML views land in the sibling directory `../cregit-files/html`. See ## Preliminaries - Code is written in Scala, C++, Rust and Perl. -- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the pinned `srcml` parser from source for those platforms (`nix/srcml.nix`). The 1.1.0 release binary segfaults under `--position`, which the pipeline always passes, and the fix is upstream but unreleased, so we pin a revision rather than a release. `srcml --version` still prints `1.1.0`. +- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the pinned `srcml` + 1.1.0 parser from its release tag (`nix/srcml.nix`). ## Prerequisites @@ -87,7 +88,7 @@ This builds, in dependency order: | artifact | module | toolchain | | ------------------------------------------------------------- | ------------------- | ------------------------- | -| `tokenize/srcMLtoken/srcml2token` | C++ transcoder | gcc + xerces-c | +| `tokenize/srcMLtoken/srcml2token` | srcML tokenizer | gcc + xerces-c + libsrcml | | `tokenize/rustTokenizer` binary | Rust tokenizer | cargo | | `blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar` | tokenization driver | sbt, JDK 21 | | `{slickGitLog,persons,remapCommits}/target/scala-2.10/*-one-jar.jar` | history / persons / remap tools | sbt 0.13, JDK 8 | @@ -111,9 +112,12 @@ cd ../tokenize/srcMLtoken && make && make test cd ../rustTokenizer && make && make test cd ../.. +bash tests/test_token_worker.sh prove tests/t pytest -q -tests/tokenize_gate.sh && tests/pipeline_workdir_guard.sh +for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + bash "tests/$t.sh" +done for module in slickGitLog persons remapCommits; do (cd "$module" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar) @@ -162,6 +166,20 @@ Flags (see `./run_pipeline_process.sh --help` for the full list): | `--memo-dir` | where to memoize tokenized blobs; outside `--work` it survives the full-run wipe | `/memo` | | `--mode` / `--shards` | tokenizer walk mode / shard count for `sharded` | `pipeline` / `4` | | `--jobs` | concurrent blame/HTML processes | `CREGIT_JOBS` or up to `4` CPUs | +| `--reblame` | re-blame every file in step 7, replacing existing `.blame` output | off — a resume skips files already blamed | + +**`git blame` runs with `-C100` copy detection** (`blameRepo/formatBlame.pl`), so a +token moved between files keeps its original author. Upstream shipped that commented +out. Enabling it changed the author of up to 25% of a project's tokens, so blame output +produced before 2026-09-22 is not comparable with output produced after it. + +**Pass `--reblame` whenever the blame itself changed**, not the file list. +`blameRepoFiles.pl` skips any file whose `.blame` output already exists. That skip makes +a resume cheap, and it makes a re-blame a silent no-op: step 7 reports every file as +already done and exits 0, then step 10 rebuilds from the old blame. One pilot changed +**0 of 15,036,195 tokens** that way, with identical row counts and `rc=0`. Read step 7's +summary to confirm a real re-blame — `Already done [0]` is the proof. Do **not** pass the +flag to resume an interrupted run. A full run starts by **deleting the work directory** — to keep several target repositories side by side, give each its own `--work`. To resume a failed run @@ -174,19 +192,79 @@ returns without invoking `srcml` at all. Put it out of reach with `--memo-dir`, names neither repository nor extension). The runner refuses a step-1 wipe that would delete a memo of 10,000 entries or more; `--force-clean` overrides it. -Four blobs in one corpus project are on a **blob denylist** -(`blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala`): srcML 1.1.0 does -not terminate on them (upstream srcML/srcML#2361, which the revision we now pin closes, -so the entries should be inert). They are never handed to -the tokenizer, are dropped from the rewritten trees rather than kept as raw -source, are counted as `blobsDenylisted` and named with a reason and a citation — -and, unlike a tokenizer timeout, they do **not** change the exit status. Adding an +209 blobs are on a **blob denylist** +(`blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv`): 12 on which +srcML 1.1.0 does not terminate (8 Java, srcML/srcML#2361; 4 C), and 197 C and C++ +blobs on which it crashes under `--position`. The file header cites each defect. +Denylisted blobs never reach the tokenizer, are dropped from the rewritten trees, +are counted as `blobsDenylisted`, and do **not** change the exit status. Adding an entry means editing that file and rebuilding the jar. +A blob whose tokenizer times out or reports a parser crash is excluded the same +way without an entry: it is named on an `EXCLUDED failed blob` line, tried once +per run, and the run exits 0. An entry still saves the `--blob-timeout` (600 s by +default) that a known hang costs on each run. + Example run (cregit run on itself): ![Example cregit run](cregit.gif) p.s.: long pauses are trimmed. +### When a tokenizer is corrected: `--retokenize` + +A corrected tokenizer does **not** by itself produce corrected tokens on a +resume. blobExec decides whether to reuse a cached tokenization from the +recorded `command` and `mask`. `command` is the constant path +`tokenizeByBlobId/tokenBySha.pl`, and `mask` says *which* files to tokenize, +never *how* — so rebuilding a tokenizer moves neither value, every cached row for +that language stays a cache hit, and the run reproduces the old tokenizer's +output and exits 0. Measured, for the Rust fix in `729643e`: 741,869 `.rs` +(blob, path) pairs across 45 projects. + +Two mechanisms close that: + +1. **Every run reports a tokenizer identity** — one opaque digest per file + extension, covering that extension's whole parser toolchain + (`tokenize/tokenizerIdentity.pl`). blobExec records it in the blob map's + `meta` table and compares it on every later run. A mismatch, on an extension + that actually has cached rows, **refuses the run** (exit 3) and names the flag + to fix it. Nothing is invalidated automatically. +2. **`--retokenize EXTS` is the opt-in past that refusal**, and it is selective: + re-tokenizing is 88% of total pipeline time, so a `.rs`-only defect costs + `.rs` entries only. + +To re-tokenize only the `.rs` entries of one project, after rebuilding the +tokenizer: + +```sh +# 1. make sure the binary is actually current (this also proves the identity moved) +./run_pipeline_process.sh --ensure-artifacts + +# 2. resume that project at step 2 — never step 1, which deletes the work +./run_pipeline_process.sh \ + --repo-url --repo-name \ + --work ../cregit-files- \ + --retokenize rs \ + 2 +``` + +`--retokenize` needs `FROM_STEP=2` exactly: step 1 deletes the work directory +(so there would be nothing cached to invalidate) and step 3 or later skips the +invalidation entirely. It is not available with `--mode sharded`. + +What it does, in one transaction: drops the `blob_map` rows for those extensions, +purges **their entries in the memo** (which is keyed on `sha1` of the file's +content with no tokenizer in the key, so dropping only the `blob_map` row would +let the memo answer with the same stale tokens), and drops `tree_map`, +`commit_map` and `ref_map` because a tree names its blobs. Every other +extension's tokenizations survive. + +It cannot quietly do nothing. blobExec refuses, **before changing anything**, if +no cached row carries a named extension, if the memo held none of the affected +blobs (which means `--memo-dir` is not this project's), if another extension's +tokenizer also changed and was not named, or if a retained `new_blob` id is +missing from the cregit repository. An ineffective `--retokenize` exits 7, never +0. + ### Outputs Everything lands in the work directory (default: `../cregit-files`, a sibling diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index eb86b9891..41a5ad891 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -207,7 +207,7 @@ =head1 SYNOPSIS blameRepoFiles.pl [options] Options: - --override overwrite existing files + --overwrite re-blame files whose .blame output already exists --help brief help message --man full documentation --jobs=N maximum concurrent blame processes (default 1) @@ -228,9 +228,10 @@ =head1 OPTIONS Prints the manual page and exits. -=item B<--override> +=item B<--overwrite> - By default, if an output file exists it is skipped. This changes that behaviour. + By default, if an output file exists it is skipped. This changes that + behaviour and re-blames the file. =back diff --git a/blobExec/shard_build.sh b/blobExec/shard_build.sh index 035b5dcb9..a1393e818 100755 --- a/blobExec/shard_build.sh +++ b/blobExec/shard_build.sh @@ -80,9 +80,9 @@ if [ -z "$THREADS" ]; then fi # Default tokenizer dispatcher (srcML for .c/.h, rustTokenizer for .rs, …); the -# --srcml/--ctags paths are forwarded only to the srcML parser. +# --srcml2token/--ctags paths are forwarded only to the srcML parser. if [ -z "$TOK_CMD" ]; then - TOK_CMD="$CREGIT/tokenize/tokenize.pl --srcml2token=$CREGIT/tokenize/srcMLtoken/srcml2token --srcml=$(command -v srcml) --ctags=$(command -v ctags)" + TOK_CMD="$CREGIT/tokenize/tokenize.pl --srcml2token=$CREGIT/tokenize/srcMLtoken/srcml2token --ctags=$(command -v ctags)" fi export BFG_TOKENIZE_CMD="$TOK_CMD" JAVA="$(command -v java)" diff --git a/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv b/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv new file mode 100644 index 000000000..64e65c6ad --- /dev/null +++ b/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv @@ -0,0 +1,498 @@ +# blobExec blob denylist — blobs excluded from the rewrite by content id. +# +# A denylisted blob is never handed to the tokenizer. It is dropped from every +# rewritten tree, exactly like an oversized blob (Walker.readBlob), so the file is +# ABSENT from the tokenized repository rather than present as raw source: it +# produces no blame and no dataset row. It is counted separately from +# blobsOversized and blobsTimedOut, logged once with its sha, path and reason, and +# it does NOT affect the exit status. That last point is the difference between +# this list and a timeout: an entry here is a diagnosed, deterministic, cited +# exclusion, while a timeout is a hang nobody has explained yet and must keep +# blocking publication. +# +# This is a DATA file, not a constant in the code, so a paper can cite the list +# itself. It is loaded from the classpath (BlobDenylist.ResourcePath) and shipped +# inside the assembly jar, so adding an entry means editing this file and running +# `sbt assembly`. Nothing at run time can extend or override it: an undocumented +# exclusion is the one thing that would make the dataset's omissions unauditable. +# +# FORMAT: three TAB-separated fields. Blank lines and lines starting with # are +# ignored. Leading and trailing whitespace on a field is stripped. +# +# +# +# The key is the GIT BLOB id, which is what the walker has in hand +# (Walker.BlobMissTask.origId) and what blobExec's own timeout line prints. It is +# not the memo's key: tokenizeByBlobId/tokenBySha.pl hashes the file contents +# alone, with no git object header. +# +# The key is deliberately NOT the path. The entries below are the historical +# versions of 36 source files, one of them at two paths after an OpenJDK +# repository reorganisation and one of them a single blob vendored into TWO +# DIFFERENT PROJECTS (zlib's `deflate.c`, byte-identical in ossec__ossec-hids and +# azerothcore__azerothcore-wotlk — one entry covers both). A path-keyed list would +# have to be updated every time a project moves a file, would need a duplicate +# entry per vendored copy, and would not distinguish the versions of a file that +# fail from the versions that parse — which is a 4/4 split on one file in class 2 +# and, in class 3, splits as narrow as 3 of 423. Content is the stable identity. +# The paths are recorded for a reader rather than matched on. +# +# =========================================================================== +# THREE DISTINCT DEFECTS, and the third is NOT LIKE THE OTHER TWO. +# +# classes 1 and 2 are srcML NON-TERMINATION: srcml spins forever, and the +# pipeline only ever sees a timeout. +# class 3 is a srcML CRASH: srcml DIES ON A SIGNAL (SIGSEGV or SIGABRT) in +# milliseconds. It is not a hang, it must not be described as one, and it +# reaches the pipeline through a different code path (exit 33 from +# tokenizeSrcMl.pl, `blobsParserCrashed` and exit 6 from blobExec). +# +# Only class 1 has an upstream issue. Do not cite #2361 for class 2 or class 3. +# =========================================================================== +# +# --------------------------------------------------------------------------- +# CLASS 1 — the JAVA parser, on type-annotated array types. +# Upstream: srcML/srcML#2361, open, no patch. +# 8 blobs, TWO files, both in tencent__tencentkona-21 (OpenJDK-derived, +# 50,663 .java files at HEAD, 147,265 blobs recorded). Both are OpenJDK +# langtools regression tests whose stated purpose is to stress exactly the +# grammar that breaks the parser: not production source, and carrying no +# contributor-behaviour signal. That is what makes excluding them cheap. +# --------------------------------------------------------------------------- +# +# TestNewCastArray.java — @bug 8005681 "Repeated annotations on new,array,cast" +# test/langtools/tools/javac/annotations/typeAnnotations/classfile/ +# 2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65 14,503 B +# langtools/test/tools/javac/annotations/typeAnnotations/classfile/ +# 303c1a109f1fdc4b9d84e15111c56931202844ac 14,506 B +# 504aaf9109fc01d6d62d99ac529ded99a8dfa8ca 14,451 B +# c91924d1e87cb82d372a73b24579ccc34609efaa 14,450 B +# +# CheckErrorsForSource7.java — @run main CheckErrorsForSource7 +# langtools/test/tools/javac/annotations/typeAnnotations/failures/ +# 56c8adf588e697a3e42b600b2e5ec54710592c61 8,388 B trigger at line 178 +# 5e9c9138ed9c43c20591d1814cb72ad014d1a9e5 8,750 B trigger at line 181 +# 6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7 8,645 B trigger at line 179 +# test/langtools/tools/javac/annotations/typeAnnotations/failures/ +# afbd81e5789ba060147b0de3be4f01759963299d 8,617 B trigger at line 178 +# (5e9c9138… lives at both paths; all four versions of the file hang) +# +# TestNewCastArray's trigger, minimised in srcml-nontermination-analysis.md, is a +# declaration whose declared type is a multi-dimensional array carrying a type +# annotation on its first dimension WHEN the initializer is an array-creation +# expression that also carries one: +# +# String @A [] [] s = new String @A [2] [2]; +# +# CheckErrorsForSource7 is a DIFFERENT FILE with a SECOND, DISTINCT sufficient +# construct in the same #2361 family, minimised here on 2026-09-20. It needs no +# initializer at all: +# +# Object [] @TA [] arr @TA[]; // the real line +# class C { int @A [] a @A []; } // minimal reproducer, 31 B +# +# BOTH an annotated dimension in the declared type AND an annotated trailing +# C-style array dimension on the declarator are required; either one alone parses +# in under 90 ms, and it makes no difference whether the declaration is a field or +# a local, or whether an initializer is present. The neighbouring line 179, +# ` @TA int @TA[] ret(Object obj) @TA[]`, was measured and is NOT a trigger: on +# its own it parses in 8 ms. Offer this reproducer upstream on #2361 alongside the +# first one — it is a second sufficient condition the issue does not record. +# +# --------------------------------------------------------------------------- +# CLASS 2 — the C parser, on C++ inside a `.h`. +# NO UPSTREAM ISSUE EXISTS. This is NOT #2361: different language, different +# construct, separately minimised. The citation is the analysis document. +# 4 blobs, ONE file, in facebook__sapling. Unlike class 1 this is ordinary +# production source, so the exclusion does cost real signal — 4 blobs of a +# 1.9-2.6 kB header, bounded below. +# --------------------------------------------------------------------------- +# +# eden/fs/utils/StatTimes.h (historical only; NOT present at HEAD) +# 5460d43f1884e3be230e02215151b9b963844a48 1,861 B +# 720914b27e170b73b38ef662fb04239273dd0a34 2,580 B +# 99c191e9a208547f5c9b1224c290f106a640de59 2,443 B +# dbb885e0bf046755c4fcfbddfc2454f90c550c62 2,442 B +# +# CregitLanguages.pm maps the `.h` extension to language C, so tokenBySha.pl runs +# `srcml -l C` over a header whose contents are C++. The C parser then does not +# terminate. Minimised on 2026-09-20 from 1,861 bytes to 41: +# +# namespace n{struct T&f(const struct S);} +# +# Necessary and sufficient, each condition measured. Remove any one and the input +# parses in under 12 ms: +# +# - an enclosing NAMED `namespace X {` or `class X {` block. An anonymous +# `namespace {` does NOT trigger it. +# - a return type that is a REFERENCE to an ELABORATED struct type, `struct T&`. +# `struct T*` parses; `T&` without the `struct` keyword parses. +# - a parameter of CV-QUALIFIED ELABORATED struct type, `const struct S` +# (`volatile` does it too). The parameter need not be a reference and need not +# be named. `const S` without `struct`, and `struct S` without the qualifier, +# both parse. +# +# THE THREE BACKSLASH-CONTINUED `#elif defined(…) || \` CONDITIONALS AT LINES 24, +# 37 AND 50 ARE NOT THE TRIGGER. They are the file's most conspicuous feature and +# the obvious suspect, and they are innocent: extracted on their own they parse in +# 8 ms, and — the decisive evidence — this file has EIGHT historical versions, ALL +# EIGHT carry three or four such continued `#elif` conditionals, and only FOUR +# hang. The split is exactly the reference return type: the four that hang all +# declare `inline const struct timespec& stAtime(const struct stat& st)`, and the +# four that parse in milliseconds all return `struct timespec` BY VALUE. A natural +# experiment in the corpus's own history, 4/4, agreeing with the minimisation. +# +# The same 1,861 bytes parse as C++ in 9 ms and produce 14,125 bytes of srcML, so +# a `.hpp` name would never have hit this. That narrowness — C++ declared in a +# `.h`, named namespace, reference to an elaborated struct, cv-qualified +# elaborated struct parameter — is why 188 C and C++ projects produced ONE +# instance of this class rather than thousands. +# +# Blast radius, measured rather than assumed: a bounded probe of ALL 29,100 +# mask-selected C/C++/Java blobs in facebook__sapling's whole history found these +# 4 and no others. 4 of 29,100 is 0.014%. +# +# --------------------------------------------------------------------------- +# WHY THIS IS NON-TERMINATION AND NOT SLOWNESS, for every CLASS 1 AND CLASS 2 +# entry. Class 3 is not a hang at all; its evidence is further down. +# --------------------------------------------------------------------------- +# Zero bytes of output, one core at 100%, and resident memory FLAT, at every +# budget tried. Flat memory with a saturated core rules out backtracking that +# would eventually finish and rules out memory exhaustion. Measured 2026-09-20 +# against the pinned binary, reading the counters from the srcml CHILD and not +# from the `timeout` supervisor, which blocks in wait4 at 0% CPU and would look +# idle: +# +# StatTimes.h as C 5 s / 15 s / 60 s / 300 s 0 B out, 100%, RSS 9.2-9.4 MB +# the other 3 .h as C 15 s / 60 s 0 B out, 100%, RSS 9.3-9.4 MB +# CheckErrors… as Java 15 s / 60 s 0 B out, 100%, RSS 10.1 MB +# all 6 newly listed here also killed at 600 s in the +# pipeline, status 124 from +# `timeout` — not 137, not OOM +# +# srcML v1.1.0 (2025-08-14) is the latest release, so there is nothing to upgrade +# to. A working patch for class 1 was built and validated at 12,016/12,016 +# byte-identical blobs and deliberately NOT adopted, on provenance grounds. Both +# write-ups, and the minimisation behind class 2, live in +# cregit-token-pipeline/.superpowers/sdd/peppy-pondering-music/ — +# srcml-nontermination-analysis.md, srcml-upstream-fix-adjudication.md and +# task-8c-finish-report.md. +# +# --------------------------------------------------------------------------- +# THIS LIST IS NOT THE COMPLETE SET OF NON-TERMINATING BLOBS IN THE CORPUS +# --------------------------------------------------------------------------- +# A FURTHER non-termination class exists and is deliberately NOT listed here +# (it is not class 3 below, which is a crash, not a hang), because excluding it +# is a decision about the dataset's content rather than about a stress test: the +# C++ parser also fails to terminate on `src/hotspot/share/runtime/ +# continuationFreezeThaw.cpp` in tencent__tencentkona-21 — ~105 kB of ORDINARY +# PRODUCTION SOURCE in the JVM runtime, 0 bytes out at 15 s and 120 s with the +# same flat-memory, one-core signature. Denylisting production source removes real +# contributor-behaviour signal, so it is escalated in task-8c-finish-report.md +# rather than decided in this file. Until that decision is taken, +# tencent__tencentkona-21 cannot complete step 2, and the timeout — which DOES +# block publication — is the correct thing to be stopping it. +# +# +# --------------------------------------------------------------------------- +# CLASS 3 — the C and C++ parsers, CRASHING ON A SIGNAL under `--position`. +# NO UPSTREAM ISSUE EXISTS. This is NOT #2361 and NOT class 2: those are +# non-termination, this is a SEGFAULT OR ABORT in milliseconds. 197 blobs, +# 36 files, 18 projects (19 corpus projects are affected — see the shared +# `deflate.c` note above). Cited to silent-empty-sweep.md. +# Ordinary production source, not stress tests, so this exclusion does cost +# real signal — bounded and enumerated below. +# --------------------------------------------------------------------------- +# +# WHY THIS CLASS HAD TO BE ADDED AT ALL. Before the tokenizer was made to fail +# closed, this defect was INVISIBLE. The shell pipeline in tokenizeSrcMl.pl, +# +# srcml -l C --position FILE | srcml2token +# +# returned PIPESTATUS 139,0: srcml died on SIGSEGV, and srcml2token exited 0 +# anyway because it exits 0 even on truncated XML. The caller therefore wrote a +# 0-byte tokenization and REPORTED SUCCESS, and the file silently vanished from +# the dataset with no counter recording it. 36 files in 19 of 186 published +# projects were lost that way. tokenizeSrcMl.pl now reads both PIPESTATUS stages +# and exits 33, and blobExec counts `blobsParserCrashed` and exits 6, so the loss +# is loud. These entries are what lets those 19 projects finish: a denylisted +# blob never reaches the tokenizer at all (Walker.readBlob checks the list before +# opening the blob), so it cannot trip the new gate, and the omission is counted +# as `blobsDenylisted` and named with this reason instead of disappearing. +# +# HOW THE 197 WERE FOUND, and why it is not 36. The sweep measured HEAD only, but +# the pipeline walks WHOLE HISTORIES, so a HEAD-only list would not let a single +# affected project complete. Every historical version of all 36 paths was +# enumerated — `git log --all --full-history -m --raw --no-abbrev`, which is the +# invocation that catches blobs introduced by merge commits and blobs hidden by +# history simplification; a plain `git log --raw` misses both, and did miss one +# project's HEAD blob — giving 904 distinct blobs. Every one of the 904 was then +# run through the tokenizer's real chain: +# +# 197 crash (153 SIGSEGV, 44 SIGABRT) -> listed here +# 707 parse cleanly, exit 0, non-empty -> NOT listed +# 0 time out -> class 3 is disjoint from 1 and 2 +# +# Every blob here was verified individually; none is listed on the strength of +# the sweep alone. The 707 that parse are the control: this is a content-triggered +# crash, not a property of the file name, and the splits are narrow — 3 of 423 +# revisions of apache's `server/util.c`, 51 of 178 of one googleapis test, but +# 24 of 24 of oceanbase's `ob_ctype_uca.cc` and 16 of 16 of kamailio's +# `duktape.c`. +# +# `--position` IS THE TRIGGER FOR MOST OF THESE, BUT NOT FOR ALL, AND THE SWEEP +# OVERSTATES IT. Measured per blob, with and without the flag: +# +# 132 --position is the trigger: exit 0 without it, SIGSEGV with it +# 49 NOT the trigger: srcML dies on a signal EITHER WAY; the flag only +# changes WHICH signal (typically SIGSEGV without, SIGABRT with) +# 16 dropping the flag would not help: srcML exits 1 with a parse +# diagnostic without it, and still yields no tokenization +# +# silent-empty-sweep.md concludes "`--position` is the trigger ... Dropping +# `--position` makes these same files parse cleanly." That is true for 132 of the +# 197, and FALSE for 65. It is false even for 7 of the 33 distinct blobs the sweep +# itself measured, including ALL FOUR of its SIGABRT cases: the sweep's control +# was `ext/liblzma/check/crc32_small.c`, which happens to be in the +# position-only group. Recorded here rather than quietly inherited, because the +# difference matters: dropping `--position` is not a fix, and for 65 of these +# blobs it would not even be a workaround. It cannot be dropped in any case — +# the token format carries `line:col` positions. +# +# So each entry's reason states which of the three it is, measured, rather than +# repeating a single claim across all 197. +# +# NOT SLOWNESS AND NOT A HANG, for every class 3 entry: srcml is dead within +# milliseconds with a core dumped, 0 bytes on stdout, wait status 139 (SIGSEGV) +# or 134 (SIGABRT) reported through bash PIPESTATUS. No budget is involved, which +# is exactly why the blob denylist and not `--blob-timeout` is the right +# instrument, and why these do not and must not block publication. +# +# Language: 119 C, 78 C++. Java is untouched — 273,410 Java files in the corpus +# go through the same tokenizeSrcMl.pl with the same `--position` and produce not +# one crash, so this is srcML's C/C++ position tracking specifically. Rust never +# reaches srcml at all (%LANG_PARSER_REL routes it to rustTokenizer). Size is +# uninformative: 1,023 B to 3,900,518 B, failures at both extremes. +# +# WHAT IS STILL NOT COVERED, said plainly. These 197 make the 36 KNOWN PATHS safe +# across their whole history. They do NOT prove the 19 projects will complete. A +# file whose HEAD version parses but whose older versions crash is invisible both +# to the sweep (HEAD-only) and to this probe (these 36 paths only), and would +# still stop its project at exit 6. Establishing otherwise needs a full-history +# probe of every mask-selected blob in those 19 projects, which was not run: +# that is hundreds of thousands of blobs. Two further gaps are inherited from the +# sweep: `torvalds__linux` has no Parquet and was never measured, and +# `tencent__tencentkona-21` lives only in a backup with no Parquet to join. +# +# sha citation reason +2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65 srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar +303c1a109f1fdc4b9d84e15111c56931202844ac srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar +504aaf9109fc01d6d62d99ac529ded99a8dfa8ca srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar +c91924d1e87cb82d372a73b24579ccc34609efaa srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar +56c8adf588e697a3e42b600b2e5ec54710592c61 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 178, `Object [] @TA [] arr @TA[];` — an annotated array dimension in the declared type together with an annotated trailing C-style dimension on the declarator, needing no initializer; minimal reproducer `class C { int @A [] a @A []; }` (0 bytes out, 100% of one core, RSS flat at 10,108 kB, at 15s and 60s here and at 600s twice in the pipeline); line 179 is not a trigger; OpenJDK langtools regression test CheckErrorsForSource7, 8,388 B +5e9c9138ed9c43c20591d1814cb72ad014d1a9e5 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 181, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); a second historical version of CheckErrorsForSource7.java, 8,750 B, recorded at both of the file's two paths +6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 179, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); a third historical version of CheckErrorsForSource7.java, 8,645 B +afbd81e5789ba060147b0de3be4f01759963299d srcML/srcML#2361 srcML 1.1.0 does not terminate on line 178, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); the fourth historical version of CheckErrorsForSource7.java, 8,617 B, after the OpenJDK path move +5460d43f1884e3be230e02215151b9b963844a48 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: srcML 1.1.0's C parser does not terminate on this C++ header, which CregitLanguages.pm maps to language C; minimal reproducer `namespace n{struct T&f(const struct S);}` — a named namespace or class block containing a function returning a reference to an elaborated struct type and taking a cv-qualified elaborated struct parameter (0 bytes out, 100% of one core, RSS flat at ~9,300 kB, at 5s/15s/60s/300s here and at 600s twice in the pipeline); the continued `#elif` conditionals are NOT the trigger and the same bytes parse as C++ in 9 ms; eden/fs/utils/StatTimes.h, 1,861 B +720914b27e170b73b38ef662fb04239273dd0a34 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); a second historical version of eden/fs/utils/StatTimes.h, 2,580 B +99c191e9a208547f5c9b1224c290f106a640de59 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); a third historical version of eden/fs/utils/StatTimes.h, 2,443 B +dbb885e0bf046755c4fcfbddfc2454f90c550c62 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); the fourth historical version of eden/fs/utils/StatTimes.h, 2,442 B +# +# --- CLASS 3 (srcML signal death under --position); 197 blobs, 36 files, 18 projects --- +046a9a1a8b09089625dc77d04185415a710cd1aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,549 B +0cdf543499d0ee5e962b37fdd574131f8b03f711 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 117,124 B +d1d06fc15b4e6abbd1dffb7c11e6d5c7bdff736e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,431 B +1de38760dd5228b923c133c4b40002e1bde6b9aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,745 B +1e35a9a56726e5884dd3bca8ea8426308aa0061c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,903 B +414e0887fe445325673b38c8fef8d26164e69e02 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,731 B +94ffeaaf43b92a2dc665a4e2b691a3a7122a064f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 34,194 B +a090ac7b09bbc414319da6000cfe027e853b25ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,165 B +e35e38af931a76ff0e2a9b98d8ce9cab567e03c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. aws__amazon-redshift-odbc-driver src/pgclient/src/backend/libpq/zstd-in.c, 2,057,541 B +28530d12c2ed7de827ca2e14ce099262a34492de silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 6,158 B +39a6ec8d4c8a45ae6558bbe78d9b562c33ca47ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 9,471 B +67e558666399f02c73e8606b72af5f7c9d3964c1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,226 B +69b1d0edde928e85f7afa4c8285e46eba84051dc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,249 B +715d5b85a6bf60ce63a87f9ae984dcd5f98363c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,221 B +abb9cde9969ea21171a0b1278dc3f5c6cbe7af4e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,242 B +bd5bf7a19faa968b9ae40a58b967a0f74dcc4fbd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 12,978 B +be5b2ada16d13bf6438eb7a6d3395645d7935681 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,226 B +c4a7154ba732cc19a00f8cf7d7f203c29d8b0249 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,249 B +e2b23bbeb7ee563c332baaf3e0847ab645a5d2b8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,256 B +3b15148d68dc75dbe727df52fe6cd9242481512f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 26,726 B +456629abf1392f382a62c569fe99080ec8f07404 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 27,429 B +a17001dd91b1247f83213a80442800dbb2614604 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 26,789 B +1e2976b83bc76371b044cbc0e680e081a1416c7f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 16,586 B +1f59b32426b3ece9b10cb947de09c52f2d0ed065 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 18,702 B +50f2a29a7237e264ea437e299147c1af3e7210c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 17,339 B +7bb26742930a9e6d494ca3693ebd3bcdb64ebf69 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 16,614 B +f0787fd85cd49490497fe8743d8eec107e0b2353 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. elfmz__far2l plugins/multiarc/src/formats/7z/C/Sha512.c, 19,801 B +933fcad8c46bd263441bbac4dab084c53090004b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. elfmz__far2l plugins/multiarc/src/formats/rar/unrar/blake2s_sse.cpp, 4,461 B +05ed4a412d7acff9629abd4824ca98ac22724875 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 152,327 B +0c24a13cb2c2ea0543d3f4b4ebe0fa3741285d30 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,263 B +0ec1d427e27bbc5c0ad42ced54e5afa8a19ae15a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,050 B +0f3bfe9b3b225213ce7880fb5119cf786296df86 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,673 B +0f707c0d21a74a74654ff03bc14bf616e23e0110 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,529 B +2c031624ce9da4a9f75d9dc29eb5076a002901e3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 164,630 B +2f193091254956a10f9feb4712a155a0818fa7f2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,394 B +2fb3a04df9fcd8ad3786b8494b5066bd95308eb7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,616 B +33e10e272e8de9b3d0f31e3fa6a30b8d25d8a62a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,905 B +33e33b44fa67050820759b63d1dd419a9ef45dbf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 154,285 B +383f345c57e8a718ffc6093beee12fb228b24607 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 161,635 B +3e5313015bd651ae57a1155f236d748cfce434e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,029 B +404e2daeba97c1d3cb4e6550b064ec338cbe0f6f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,497 B +4177bb55017d51c2e2e3ac997e60fcc84e1bddd6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,049 B +4509d2f318f19398da26a0f0506d8751dd26b24e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,499 B +4b30e2af77bcb5ff028f3bd947602cef36ec3ab2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,335 B +51e90452f68b0f627092148f52a87657701026a4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 161,634 B +52aa3a0bf7091d618239c73168f07820e853861e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,410 B +5e365fa0b1bc5b4f6d96a5447cfc11181b81a730 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 162,192 B +602546e19350bc001678de2e8be8a8bb70c7b18d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,337 B +6171c31762db866fbb545797ce21f269c11640dd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,544 B +65bbeae52abae4975141f4ed2a04563e361c8cec silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,036 B +6f02daee0dcb4f3107441c0009b888000c1342de silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,741 B +7e1db5cda9298e20c837e2824060559aef1aa868 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,429 B +87fd46e27098697fcabecd0a86a82c60d2bc1390 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 171,316 B +89c41a15747b2a9ce0f120c403f89a751d17c3e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,922 B +8cc2d1e47bfac43d027ef03ad7c763e07ea107e6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,863 B +916d906d99baea144dd1134c10c1c01d32fc1d15 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 175,544 B +96390ceef0abcbe821337850f0584d81de34adb4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,471 B +9a03addfdacdb46f0310952164ca451d5a223427 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,550 B +9a1ee408870ae2d572b242d006e6bd3e042489c8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,034 B +a0a74b95221d915d95678f2716ebe599f21cd11b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,035 B +ab86df42865e5aebbc3f60b547a365b3bb8b0575 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,732 B +ae43f731c3c433a608f5dc4e83c2bdfaeae9190d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,334 B +b5dc5f6630e608e49e7208d18cd7a371845a202c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,954 B +ba0134cc103445972e162e8e2b0e6b1ea57115b5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,055 B +bd3cf5ecdd953ee1d0b2523977f6045914ab243c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,826 B +c32127a08e590715f0342f223378b40a8d5882ce silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,389 B +cf65f8536ac904ae955c6ec9bc62c28e3f1cbf07 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,220 B +d0655644a8d0dd386ad46413efa0dd27ea8147b9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,474 B +d60d3dd4e2ae1b271f1f2ca66da946f4d15951f5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 153,857 B +def89df2cdd9275cbe7e5929b0521b90bae51139 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,540 B +e02931f0940695f5da27ac080af14cf9b63e0bc9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 175,420 B +e112238123a7a8f78c5ed5579dbe71fbc6bd6a19 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 165,385 B +ea6a60c02519c14cdfcd7de0feb00d3df8f25dee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,830 B +eadd203c8afa9fba2680c0f61a26305f7a086789 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,740 B +ef3d052b726a202edc7cf09a4a53fbd5c00ad118 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,125 B +f0b497b6014ca90691dc415830fa0a5491ad9554 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,642 B +f4f93443ddf9549005694ac1cf9d6d60f3248ea7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,221 B +f5b021c8e98afac62fe5a9dd99d2e17430d7980e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,397 B +fabb972a818449b3df1cec8df75f783e1dec263b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,033 B +1c85746e4c4bc6488f8726a1efa8a15dccdddc04 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. java-native-access__jna native/libffi/testsuite/libffi.call/many2.c, 1,070 B +98eac6013a4381071b6785fff399fe1b52cce310 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. java-native-access__jna native/libffi/testsuite/libffi.call/many2.c, 1,069 B +03283dabd1c32502905c92398e06b6f91fdacb60 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,410 B +228f8bd0b451a34040e5f759158e7968d63214e4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,393,936 B +366883c533a0b821c236c0714f3ff7ad79fce474 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,670,090 B +4042c1d6bc6610f4dd8ac85488807cd84275a6ec silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,500 B +573e70c2b27e1fab1d4fbe4c2ac2edca14df7afa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,429 B +81363cec7afdbcd0a35b9265c17c47cb73aeef87 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,574,270 B +8de338781a9ea4640d30bcf6b0fd04da0eccf1e8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,398,868 B +97bd7dcf18b7bfdd3861a03d1827554d8fb5f310 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,505 B +9e0580406539837a64a1f25868f869ff0002b2ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,670,082 B +b5c33d555fa2b71760e771481810242bdf09096a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,415 B +ba4572cb83bd59a19a2745bfee680fe2bcb2ff4c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,465,940 B +ba55550ed25b41aa9e66749389f649586426c697 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,514,074 B +c3781f766fba4ec06491897a9965428ee58a6898 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,574,278 B +c6f1f3b49ab279df3fdf4909229babdb8cac318a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,518 B +cd2f7919024d8e863aae68ca0d7f7427e2412890 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,675,329 B +e836ef4bc55254f70a0c535bc8a6c1d0b3b9a58d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,675,310 B +53dacfa43d8932e31c2ba1cf5dac9a09bef3237e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kicad__kicad-source-mirror thirdparty/eigen/Eigen/src/Core/arch/MSA/Complex.h, 17,541 B +3575d3b6aa7d0ea111cd5cb80212fc670d8a10cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. nationalsecurityagency__ghidra Ghidra/Features/Decompiler/src/decompile/zlib/deflate.c, 81,787 B +30597abfa1e1320dfb5e91e7445b2b534c6672d1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,448 B +309853a1f6910e72061c215686b43aca0441f783 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,091 B +3f231fd0435ba0f4be9c9c8c3985bb7fb0bfa0cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,314 B +4ed2f82097b0a6436b610ea14f1a1fe10e5f8b41 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 299,493 B +6258b7a7135bd2beab1a8d239ea309a8c8acd21d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,058 B +6930d8f7a42292a35e76f442b1c960343e5204b4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,424 B +6c04e1c1b62f35a51bb1ddc07db3b7f7dbaad014 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,269 B +79dbbc97aac295ee5a1c44f3a2a7a7007891ece2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,813 B +8256f5a96a8f1db7629e2c5651c599f62a7db53e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,736 B +907f04f2f4b0edf4d8ea37ff0cdfde4feccc20fc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,824 B +90c4a9adacb5423ef6f2f306378377117deaf4c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,118 B +9ce560f932dc17daeda1047a3bce933e451d707b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,277 B +aa013c1884f847ae3c6b78fb25d54bd8a01644bb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 117,839 B +af218291846c944e3fd4dd29d811c1593c57f9ef silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,253 B +c9a41217fbc425ff0c867613473ff9c177809ecc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,857 B +d3283082d7871dbce2e305a98ec165c3c9fe6ebd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,835 B +d57032de2134b34e0e99191f9f96bc85bbb548c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,262 B +db2e30f1f0994a55231784858640bf9038c466aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,733 B +dede365cbe9be3cf8410be0cd4f11def94182faf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 118,428 B +df1da8d1e931da4c081960e3839239b4a66dfbe7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,821 B +e3ec9472d5501c7fcd13f7528c0ca5bce8ebdcd3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,170 B +ebfa62a39ad6d1418075c18310dc4585ab139fc3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,286 B +ef36c1b86571b8faea2e8cb068a6748ae0eef9ca silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,094 B +f5e4d64b0c3dab24c0b776114934740e3445f6e2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,194 B +2ee2dd4d33960766e04e4ccf58bca5a9e8afe0ca silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,884 B +333d4687230a78f8f6b9fd50e1a4af58510a1e4d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,709 B +3a8727bd6b473ddf301a61fb2b369b888e2b5826 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,043 B +4b77786aa5b7e6090f50be39c43da7dce0c585e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,745 B +4cd23b5415822f5c9b8b762b6ed6366fa329dca1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,665 B +6084d12c5ff6ffc0b662649848ec7f3ea19ecc30 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,848 B +60be21e572963097c0a29a279228f51d8d922724 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,658 B +69303749878b3535bf61b40d425a3de380a74872 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,782 B +72903c04d785a966a580d4a5fb43522d194c3f32 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,519 B +76bb8392b107bf6ee1a7af6623217316f8c1bdd5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,059 B +8f569ae5f2e701e770e5612c7740a606a278b068 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,876 B +9dc3d5ec0d9cb5a0a993e830c30361538f760a1f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,756 B +a48a2ada09935e4e9b477342f4eceb478d160027 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,849 B +c7ef4dcd816f0c564a90d045d7a3be0140094866 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,030 B +cf3211cb9740d1fd851ebd6e2d23ab3b0f9bc344 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,677 B +dc68ca6fc776617bf5a4400c775f42022bc483df silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,364 B +e7bee4e9aa5e2f4b019ec68929c0b95b13d57d9e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,540 B +ea3aa7f42b60a23aeceea1a39f5ca7215e2603a7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,169 B +ee6c056ef7a96062cf43e4728a6f4343f3f2bc63 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,676 B +f76fe3ed5fa1279aeefa6bd090d74a86608ce87d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,168 B +012ea8148e8d21cf515a9ff47d4af719d049b276 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. ossec__ossec-hids src/external/zlib-1.3.1/deflate.c, 81,731 B +32ba0a14f4cbe94f638959a8ff1ed5e19bf8814c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/a-unrar/unrar.cpp, 705,097 B +ffc37ddd5fd2895ceb6ce95744250b51ee4ba408 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/a-zlib/zlib.c, 819,904 B +675e80674311343726ca271fc2ab79ebca3a2d81 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/chmdec/chm.c, 72,897 B +cd8c0847a22e1f1dcf4e0ec44e3e3dd859318371 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/libjpeg-turbo/src/jdhuff.c, 26,826 B +4a62830c807a5a2aff73f474891c38ed913d39c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/liblzma/check/crc32_small.c, 1,263 B +ee4ea26f67d0df5e9a39be5d60234c1393c89b3c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/liblzma/check/crc64_small.c, 1,023 B +791d224754ee54e930a43141b09a7bf459658a4c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/webview2/WebView2EnvironmentOptions.h, 19,500 B +6d2176c258b34fa814fef75f6934e110973dfa78 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. tencent__tendbcluster-tdbctl cmd-line-utils/libedit/history.c, 25,445 B +6917923a35b45c9f556ba06baa84262066f5625d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. tencent__tendbcluster-tendb storage/rocksdb/rocksdb/utilities/document/json_document.cc, 16,883 B +c229eda7d995851318f82aa3fa462cf227f09884 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_feather/flexspi_nor_config.c, 1,106 B +3bab1761d7c85b9b9f5074db9f2220f9f48e94d3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_feather/mmfeather_sdram_ini_dcd.c, 7,281 B +cac7fa33ccf1bfd40439ffb5ee0ffb5d3d081468 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_swiftio/mmswiftio_sdram_ini_dcd.c, 7,281 B +7a59acd6c54fe2497a442d7fb3505ccb1a4e2f2f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/nxp/vmu_rt1170/flexspi_nor_config.c, 4,437 B +85ad476cf84c051d510190a73b5274fdb0496ed8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/nxp/vmu_rt1170/flexspi_nor_config.c, 4,415 B +020bf6abd57b2676c207174943f5ed1d06130ecf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/pjrc/teensy4/flexspi_nor_config.c, 1,146 B +27aa72b69a124dae8bdcadfa8d6a497ea59efecb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,509 B +29484ffe569ec3c4ec13acb88577b8b7cf252463 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,554 B +2c56e15018af94c8aadd91ea77d08cc356eaa21d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,557 B +37cf5d5b1985a7510ad9c952897ef253231e80f7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,560 B +6b5078677fba7ef7de770623564e68e9e38cee0a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,549 B +8498d2ea64b268b431d3c38cde0ea6ebf27c34d2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,538 B +b804a92bc31de459b9781febb3fb87c9af54c6fb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,501 B +ccc226a02dccbbee5306569fa945cd4b5996a2e2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,512 B +e6ba7d66d7f12f57e87f158decc659276cb5f4cc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,490 B +1142febb016545cd524a49f8019e1a5086d7abee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,670 B +447a342e7012f9cbad0c5681d8d2ce13adb1e6a4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,750 B +4d706c3a95a53e6cbdecd770238c42cf5b12a8c9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,834 B +62ac281ce0e097b987ffad41b02a5d71325ec15c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 16,589 B +7779e10e68dcfeedcf6f55d7f91b75086f98fffe silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,068 B +80b69eea271bef4d8f39fe681b23f307284444f0 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,387 B +89f1e216984a0ffcae4b0f6762d11baedf5df6b0 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,129 B +8ae02efd5abb170fc942396588fa5e403aecaf48 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,774 B +8ca2c0aaeb1e5a0ef5de2592426befdfaf11434e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,606 B +95ee516350cae32d784ad46c1a156fd2dc2e07c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,784 B +a90ca6913f268161ab85d919d8cf25b24d508d43 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,028 B +a9c2022be5fc36bc4aaf1de3a9f7696ab3301eff silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,386 B +b7acae43082196238a408a9cf5a2aed169a79e06 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,785 B +be480dff5a7c1aead25e0e6e8ee4b8b93129cf38 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,149 B +c4e180617725397a291f948c566b799155361d55 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 8,502 B +c600708556fc3f92056b87d73ab7e94168eb89f5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,412 B +d607f13c33fa52261dba8d81315fbb39af719c87 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 16,601 B +dd7c5ffc1f8662932b8f799576554f730923d393 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,323 B +dfce9c5d7046f70a9f8f0af22ecba28ee9a9c4f6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,147 B +e405850bf7f659dc01dc53dcba9571b21227bb7d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,040 B +eba3aba4c5b8ed7528eb97f5c2a9aed3f7488ce4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 14,884 B +f605b07bdd5490314575bca73fe9d17d61a3acfa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,092 B +fb88a5b70235c9e9cd87239b98590ccdf46e704c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,103 B +ffdf47ff4d20bbf5c96196b27fe346f0d86a8336 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,808 B +32b67a3b19775cc4a84a8818811ec9feb84dba85 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,315 B +46b6463387b685600af0e28dddf14022d1e4990c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,961 B +895f6e75239f112a3496156b5b0b4e278a8248cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,312 B +a837359201821dff2b906194381b228dd1a4a926 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,980 B +aeda953c0d26024896b54451fa0aa339f6e30233 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,978 B diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala index c9032c00c..222ec72d5 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala @@ -1,29 +1,110 @@ package cregit.blobexec -/** Blobs never handed to the tokenizer, keyed by git blob id: srcML 1.1.0 does not - * terminate on them. Excluded like an oversized blob, and deliberately not gating - * the exit status. The entries live in [[BlobDenylistEntries]]. */ +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ +import scala.util.Using + +/** Blobs that must never be handed to the tokenizer, keyed by git blob id. + * + * The corpus holds four blobs on which srcML 1.1.0 does not terminate: it + * produces 0 bytes, pegs one core and never finishes, at every budget from 5 s to + * 600 s. The per-blob timeout (see [[BlobExec]]) excludes such a blob too, but + * only after it has spent the whole budget on it in every run. This hang is + * diagnosed, with a minimal reproducer and an open upstream defect, so the list + * excludes it at once and records the reason. + * + * So a denylisted blob takes the exclusion path instead, the same one an + * oversized blob takes: excluded in milliseconds, dropped from every rewritten + * tree, counted on its own, logged with its sha, path and reason, and NOT gating + * the exit status. The timeout stays exactly as it was. + * + * The entries live in a data file rather than in this code + * ([[BlobDenylist.ResourcePath]]), because the list is a documented property of + * the published dataset and a paper has to be able to cite it. + */ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.Entry]) { - def entryFor(sha: String): Option[BlobDenylist.Entry] = bySha.get(sha.toLowerCase) + /** The entry for `sha`, or None. Case-insensitive, because a git id is hex and + * both cases name the same object. */ + def entryFor(sha: String): Option[BlobDenylist.Entry] = + if (bySha.isEmpty) None else bySha.get(sha.toLowerCase) def size: Int = bySha.size def isEmpty: Boolean = bySha.isEmpty + def shas: Set[String] = bySha.keySet + def entries: Vector[BlobDenylist.Entry] = bySha.values.toVector.sortBy(_.sha) } object BlobDenylist { - /** Why one blob is excluded. Both fields are required: an exclusion nobody can - * trace back to a defect is indistinguishable from data loss. */ - final case class Entry(citation: String, reason: String) + /** One excluded blob: the git blob id, an upstream citation, and one line of + * reason. All three are required — an exclusion with no citation and no reason + * is indistinguishable from data loss. */ + final case class Entry(sha: String, citation: String, reason: String) { + /** The half of the log line that comes from the data file. */ + def describe: String = s"$reason [$citation]" + } - /** Where a reader finds the list, quoted in the excluded-blob report. */ - val EntriesSource = "BlobDenylistEntries.scala" + /** Classpath location of the shipped list. A resource, so it travels inside the + * assembly jar and cannot go missing between the build and the run. */ + val ResourcePath = "/cregit/blobexec/blob-denylist.tsv" val empty: BlobDenylist = new BlobDenylist(Map.empty) - def apply(bySha: Map[String, Entry]): BlobDenylist = - new BlobDenylist(bySha.map { case (sha, e) => sha.toLowerCase -> e }) + private val ShaPattern = "^[0-9a-f]{40}$".r + + /** Parse the TSV. Malformed input throws: a denylist that silently drops a line + * would silently re-introduce a blob that costs a 600 s timeout in every run, + * which is precisely the cost this list removes. + * + * `where` names the source in any error message, because the same parser reads + * the shipped resource and a test fixture. + */ + def parse(lines: IterableOnce[String], where: String): BlobDenylist = { + val acc = scala.collection.mutable.LinkedHashMap.empty[String, Entry] + lines.iterator.zipWithIndex.foreach { case (raw, i) => + val line = raw.trim + if (line.nonEmpty && !line.startsWith("#")) { + // Trailing tabs are tolerated; an empty field in the middle is not. A + // missing citation or a missing reason has to be an error, or the list + // stops being something a paper can cite. + val fields = line.split("\t", -1).map(_.trim).reverse.dropWhile(_.isEmpty).reverse + if (fields.length != 3 || fields.exists(_.isEmpty)) + throw new IllegalArgumentException( + s"$where:${i + 1}: expected 3 non-empty tab-separated fields " + + s"(sha, citation, reason), got ${fields.length}: [$line]") + val Array(sha, citation, reason) = fields + val key = sha.toLowerCase + if (ShaPattern.findFirstIn(key).isEmpty) + throw new IllegalArgumentException( + s"$where:${i + 1}: [$sha] is not a 40-character hex git blob sha. The key is the " + + "git blob id, which is what blobExec logs, not the memo's content hash.") + if (acc.contains(key)) + throw new IllegalArgumentException( + s"$where:${i + 1}: [$key] is listed twice. Two reasons for one blob means one of " + + "them is wrong, and the list is what the paper cites.") + acc.update(key, Entry(key, citation, reason)) + } + } + new BlobDenylist(acc.toMap) + } + + def fromFile(path: Path): BlobDenylist = + parse(Files.readAllLines(path, UTF_8).asScala, path.toString) + + def fromResource(resource: String): BlobDenylist = { + val stream = Option(getClass.getResourceAsStream(resource)).getOrElse( + throw new IllegalStateException( + s"blobExec resource [$resource] is missing from the classpath. The blob denylist " + + "ships inside the jar; a jar built without it would hand four known " + + "non-terminating blobs to srcml again.")) + Using.resource(scala.io.Source.fromInputStream(stream, UTF_8.name))(src => + parse(src.getLines().toVector, resource)) + } - val shipped: BlobDenylist = apply(BlobDenylistEntries.NonTerminating) + /** The shipped list. Parsed once, on first use, and a parse failure is fatal by + * design: running with a list nobody could read is worse than not running. */ + lazy val shipped: BlobDenylist = fromResource(ResourcePath) } diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala deleted file mode 100644 index 6fb83a03f..000000000 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala +++ /dev/null @@ -1,27 +0,0 @@ -package cregit.blobexec - -/** The denylisted blobs themselves, keyed by git blob id. Data only: [[BlobDenylist]] - * holds the lookup, and Walker drops these paths from every rewritten tree. - * - * Keyed on content, not path, so one file's history across a move is several - * entries sharing a reason. Each defect class is named once below, which is why the - * shas are grouped rather than each carrying its own prose. */ -private[blobexec] object BlobDenylistEntries { - - import BlobDenylist.Entry - - /** srcML 1.1.0 loops forever on `String @A [] [] s = new String @A [2] [2];`. - * Open upstream against the latest release. */ - private val JavaAnnotatedArrays = Entry( - "srcML/srcML#2361", - "srcML 1.1.0 does not terminate on this file's annotated multi-dimensional " + - "array declarations; OpenJDK langtools regression test @bug 8005681" - ) - - val NonTerminating: Map[String, Entry] = Map( - "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65" -> JavaAnnotatedArrays, - "303c1a109f1fdc4b9d84e15111c56931202844ac" -> JavaAnnotatedArrays, - "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca" -> JavaAnnotatedArrays, - "c91924d1e87cb82d372a73b24579ccc34609efaa" -> JavaAnnotatedArrays - ) -} diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala index b71e7f2e5..143181617 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala @@ -3,38 +3,111 @@ package cregit.blobexec import org.eclipse.jgit.lib.Constants.OBJ_BLOB import org.eclipse.jgit.lib.{ObjectId, ObjectInserter} +import java.io.{InputStream, OutputStream} +import java.nio.charset.StandardCharsets import java.util.{Arrays => JavaArrays} +import scala.io.Source +import scala.sys.process.{Process, ProcessIO} /** - * Run the per-blob command and map its result onto a git object. + * Run an external per-blob command and return the resulting `ObjectId`. * - * The command reads the original blob on stdin and writes the replacement on - * stdout. `BFG_FILENAME` is the basename rather than the path because cregit's - * `tokenBySha.pl` keys its on-disk memo by it. + * Pure helper carved out of the old `BlobExecModifier`. Contract: * - * - env `BFG_BLOB` / `BFG_FILENAME` / `BFG_PATH` - * - exit != 0 → `Skip` (or `Abort` if `abortOnError`) - * - killed → `Skip`, never `Abort`, never `Replace` - * - stdout == stdin → `Skip` - * - otherwise → `Replace(newBlob)` + * - env `BFG_BLOB` = orig blob 40-hex + * - env `BFG_FILENAME` = blob basename (preserved for backward + * compatibility with cregit's `tokenBySha.pl`, + * which keys its on-disk memo by basename) + * - env `BFG_PATH` = blob's full repo-root relative path + * (e.g. `src/lib/foo.c`). Enables tokenizers to + * make path-aware decisions. + * - stdin = original blob bytes + * - stdout = replacement blob bytes + * - exit != 0 → `Skip` (or `Abort` if `abortOnError`) + * - timeout → child killed, `Skip` (never `Abort`, never `Replace`) + * - stdout == stdin → `Skip` (no inserter activity) + * - otherwise → `Replace(newBlob)` */ object BlobExec { /** Per-blob wall-clock budget for the external command, in seconds. */ val DefaultTimeoutSeconds: Int = 600 + /** Synthetic exit code reported by [[invoke]] when the child was killed for + * exceeding its budget. A real child can never produce it: on Unix a waited + * status is 0..255 (128+signal when killed), so negative values are free. */ + val TimeoutExitCode: Int = -1 + + /** Exit status by which the tokenizer reports "srcML died on a signal, or this + * tokenization is otherwise unusable" — the defect that used to arrive here as + * a zero exit with empty stdout and become a silent 0-byte blob. + * + * Emitted by `tokenize/tokenizeSrcMl.pl` (`$PARSER_CRASH_EXIT` there) and + * propagated unchanged by `tokenize.pl` and `tokenizeByBlobId/tokenBySha.pl`. + * The two constants must stay in step; 33 is below 128 so it cannot collide + * with a shell's 128+signal encoding, and is clear of GNU `timeout`'s 124/137 + * and of blobExec's own 2/3/4 exit statuses. */ val ParserCrashExitCode: Int = 33 + /** "the waiter never observed an exit status" — distinct from + * [[TimeoutExitCode]] so an interrupted wait can never be mistaken for a + * completed one, nor a completed one for a timeout. */ + private val NotExitedSentinel: Int = Int.MinValue + + /** Seconds GNU `timeout` waits between its SIGTERM and its SIGKILL (`-k`). */ + private val KillGraceSeconds: Int = 5 + + /** Slack on the JVM-side backstop latch beyond the child's own budget plus + * the kill grace. The latch should only ever fire if `timeout` itself is + * missing or wedged. */ + private val BackstopSlackSeconds: Int = 25 + + /** Longest one [[invoke]] can take: the budget, the kill grace and the backstop. */ + private[blobexec] def maxChildLifetimeSeconds(timeoutSeconds: Int): Int = + math.min(math.max(1, timeoutSeconds).toLong + KillGraceSeconds + BackstopSlackSeconds, Int.MaxValue.toLong).toInt + + /** GNU `timeout`'s own statuses: 124 = the budget expired, 137 = 128+SIGKILL, + * i.e. the command ignored SIGTERM and needed the `-k` follow-up. Both mean + * "we killed it", and both must reach the Skip branch rather than the + * `exitCode != 0` branch, which `--abort-on-error` turns into a run abort. */ + private val TimeoutStatuses: Set[Int] = Set(124, 137) + + /** GNU `timeout`, if it is on PATH. It is the only cheap way to kill the + * whole child *process group*: `Process.destroy()` signals the direct child's + * pid alone, and the tokenizer chain is `tokenBySha.pl` -> `sh` -> `srcml`, + * so the grandchildren survive, inherit the JVM's stderr pipe, and keep the + * reader thread blocked — which is how one wedged blob wedged a whole run. + * Resolved once; the warning is therefore printed at most once per process. */ + private lazy val gnuTimeout: Option[String] = { + val found = sys.env + .getOrElse("PATH", "") + .split(java.io.File.pathSeparatorChar) + .iterator + .filter(_.nonEmpty) + .map(dir => java.nio.file.Paths.get(dir, "timeout")) + .find(java.nio.file.Files.isExecutable) + .map(_.toString) + if (found.isEmpty) { + System.err.println( + "blobExec: warning: GNU `timeout` was not found on PATH. Per-blob budgets " + + "still apply, but a killed child's grandchildren can survive and hold the " + + "pipe open. Install coreutils to get process-group kills." + ) + } + found + } + sealed trait Outcome object Outcome { - case object Skip extends Outcome - final case class Replace(newBlob: ObjectId) extends Outcome + case object Skip extends Outcome + final case class Replace(newBlob: ObjectId) extends Outcome final case class Abort(stderr: String, exitCode: Int) extends Outcome } /** - * Run `command` against `bytes`. Pure aside from the child process and the - * jgit inserter, which must be confined to the calling thread. + * Run `command` against `bytes`. Pure aside from the JVM process and the + * (single-threaded-per-call) jgit inserter. Thread-safe so long as + * `inserter` is confined to the calling thread. */ def run( bytes: Array[Byte], @@ -46,54 +119,198 @@ object BlobExec { inserter: ObjectInserter, timeoutSeconds: Int = DefaultTimeoutSeconds, onTimeout: () => Unit = () => (), - onParserCrash: () => Unit = () => () + onParserCrash: () => Unit = () => (), + workerPool: Option[TokenizerWorkerPool] = None ): Outcome = { - val env = Seq("BFG_BLOB" -> origSha, "BFG_FILENAME" -> filename, "BFG_PATH" -> fullPath) - - new ChildRunner(timeoutSeconds).run(command, bytes, env) match { - case ChildRunner.Outcome.Killed(why) => - System.err.println( - s"Warning: command [$command] on blob $origSha at path [$fullPath] gave no usable " + - s"result ($why): blob left untokenized" - ) - onTimeout() - Outcome.Skip - - case ChildRunner.Outcome.Exited(ParserCrashExitCode, _, stderr) => - reportParserCrash(command, origSha, fullPath, s"reported a parser crash (exit $ParserCrashExitCode)", stderr) - onParserCrash() - Outcome.Skip - - case ChildRunner.Outcome.Exited(0, stdout, stderr) if stdout.isEmpty && bytes.nonEmpty => - reportParserCrash(command, origSha, fullPath, - s"reported a parser crash: exited 0 with no output for a ${bytes.length}-byte blob", stderr) - onParserCrash() - Outcome.Skip - - case ChildRunner.Outcome.Exited(status, _, stderr) if status != 0 => - logError(command, origSha, fullPath, status, stderr) - if (abortOnError) Outcome.Abort(stderr, status) else Outcome.Skip - - case ChildRunner.Outcome.Exited(_, stdout, _) if JavaArrays.equals(bytes, stdout) => - Outcome.Skip - - case ChildRunner.Outcome.Exited(_, stdout, _) => - Outcome.Replace(inserter.insert(OBJ_BLOB, stdout)) + val (exitCode, stdout, stderr) = workerPool match { + case Some(pool) => pool.invoke(bytes, filename, timeoutSeconds) + case None => invoke(bytes, origSha, filename, fullPath, command, timeoutSeconds) + } + + if (exitCode == TimeoutExitCode) { + // A timed-out child skips exactly one blob. Deliberately *not* routed + // through `abortOnError`: one wedged tokenizer must never take down a + // run that has already folded thousands of commits, and its (discarded, + // possibly truncated) stdout must never be mistaken for a tokenization. + // `onTimeout` is how the caller counts it: a Skip that leaves raw source + // where tokens belong must not be invisible to the run's statistics. + System.err.println( + s"Warning: command [$command] timed out after ${timeoutSeconds}s on blob $origSha " + + s"at path [$fullPath]: child killed, blob excluded" + ) + onTimeout() + Outcome.Skip + } else if (exitCode == ParserCrashExitCode) { + // srcML died on a signal (SIGSEGV on 32 of the 36 known corpus files, + // SIGABRT on the other 4), or the token stream came back empty, which a + // healthy srcML parse never is. Treated exactly like a timeout in flow, and + // deliberately so: Skip rather than Replace, so the (empty or truncated) + // stdout can never become a blob; and NOT routed through `abortOnError`, so + // one crashing blob cannot take down a run that has folded thousands of + // commits. Counted separately from a timeout via `onParserCrash` because it + // is a different defect with a different fix — more time never helps a + // segfault — and because the operator needs to know which one happened. + System.err.println( + s"Warning: command [$command] reported a parser crash (exit $exitCode) on blob $origSha " + + s"at path [$fullPath]: srcML died or produced no tokens, so this blob is excluded " + + "rather than written as an empty tokenization" + ) + if (stderr.nonEmpty) { + System.err.println(s"--- stderr from $command on $origSha ($fullPath) ---") + System.err.print(stderr) + if (!stderr.endsWith("\n")) System.err.println() + System.err.println("--- end stderr ---") + } + onParserCrash() + Outcome.Skip + } else if (exitCode != 0) { + logError(command, origSha, fullPath, exitCode, stderr) + if (abortOnError) Outcome.Abort(stderr, exitCode) else Outcome.Skip + } else if (stdout.isEmpty && bytes.nonEmpty) { + // Defence in depth, and the other half of "zero output is never success". + // The tokenizer now detects its own empty stream and exits + // ParserCrashExitCode, but this is the backstop for every way that could be + // bypassed — a wrapper that swallows the status, a different tokenizer, a + // future parser. Consuming real source and emitting nothing is a failure + // whatever the exit code says, and inserting it is precisely how a 1,263-byte + // C file became a 0-byte blob in a published dataset. + // + // The two cases the brief asks to be told apart are told apart here by + // `bytes.nonEmpty`: a genuinely empty input producing empty output is + // untouched and falls through to the `equals` branch below as a plain Skip. + System.err.println( + s"Warning: command [$command] exited 0 but produced no output for the ${bytes.length}-byte " + + s"blob $origSha at path [$fullPath]. Refusing to write an empty tokenization: counting " + + "this as a parser crash and excluding the blob." + ) + onParserCrash() + Outcome.Skip + } else if (JavaArrays.equals(bytes, stdout)) { + Outcome.Skip + } else { + Outcome.Replace(inserter.insert(OBJ_BLOB, stdout)) } } - private def reportParserCrash( - command: String, + /** Visible for testing. Runs the process and returns (exit, stdout, stderr). + * + * The child is bounded by `timeoutSeconds`; on expiry its whole process + * group is killed (via GNU `timeout`) and [[TimeoutExitCode]] is returned + * with empty stdout/stderr. */ + private[blobexec] def invoke( + bytes: Array[Byte], origSha: String, + filename: String, fullPath: String, - what: String, - stderr: String - ): Unit = { - System.err.println( - s"Warning: command [$command] $what on blob $origSha at path [$fullPath]: " + - "blob left untokenized rather than written as an empty tokenization" + command: String, + timeoutSeconds: Int = DefaultTimeoutSeconds + ): (Int, Array[Byte], String) = { + val stdoutBuilder = new java.io.ByteArrayOutputStream(math.max(bytes.length, 1024)) + val stderrBuilder = new StringBuilder + + val readStdout: InputStream => Unit = in => { + try transfer(in, stdoutBuilder) finally in.close() + } + val writeStdin: OutputStream => Unit = out => { + try { out.write(bytes); out.flush() } finally out.close() + } + val readStderr: InputStream => Unit = err => { + val src = Source.fromInputStream(err, StandardCharsets.UTF_8.name) + try stderrBuilder.append(src.mkString) finally src.close() + } + + // daemonizeThreads = true: the stdout/stderr readers must not keep the JVM + // alive. `exitValue()` still joins them on the healthy path, so this only + // matters at JVM exit — and on the timeout path they are deliberately + // abandoned while blocked in `FileInputStream.readBytes`. Left non-daemon, + // two such threads kept the process running after the walk had finished, + // reintroducing the very stall this timeout exists to remove. + val io = new ProcessIO(writeStdin, readStdout, readStderr, daemonizeThreads = true) + + val secs = math.max(1, timeoutSeconds) + // Prefer the kernel over a pid-by-pid kill: GNU `timeout` runs the command + // in its own process group and signals the *group* on expiry, so `sh`, + // `tokenize.pl`, `srcml` and `srcml2token` all die together instead of + // orphaning themselves onto the JVM's pipes. + val argv = gnuTimeout match { + case Some(bin) => Seq(bin, "-k", KillGraceSeconds.toString, secs.toString, command) + case None => Seq(command) + } + val groupKilled = gnuTimeout.isDefined + val pb = Process( + argv, + None, + "BFG_BLOB" -> origSha, + "BFG_FILENAME" -> filename, + "BFG_PATH" -> fullPath ) - printStderr(command, origSha, fullPath, stderr) + val proc = pb.run(io) + + // A stuck srcml/ctags used to park the caller forever: the child stops + // producing output but holds its stdout open, the reader thread blocks in + // pipe_read, `exitValue()` (which joins the io threads) never returns, and + // the whole project goes silent. Bound the child instead of trusting it to + // exit. `exitValue()` is moved onto a daemon thread so the timeout can be + // observed even when that join is the thing that is wedged. + val finished = new java.util.concurrent.CountDownLatch(1) + val exitHolder = new java.util.concurrent.atomic.AtomicInteger(NotExitedSentinel) + val waiter = new Thread( + () => { + try exitHolder.set(proc.exitValue()) + catch { case _: InterruptedException => Thread.currentThread().interrupt() } + finally finished.countDown() + }, + s"blobexec-wait-$origSha" + ) + waiter.setDaemon(true) + waiter.start() + + // `timeout` is the primary kill, so the latch is only a backstop: give it + // the child's own budget, the -k grace, and slack. It fires only when + // `timeout` is absent or itself wedged. + val latchBudget = maxChildLifetimeSeconds(secs).toLong + + // The io threads are abandoned rather than joined on every timeout path: if + // a surviving grandchild still holds a pipe, joining them is exactly the + // hang we are escaping. That makes `stdoutBuilder`/`stderrBuilder` live, + // racy state, so neither is read there — a timed-out blob's output is + // discarded by the caller anyway. + if (!finished.await(latchBudget, java.util.concurrent.TimeUnit.SECONDS)) { + System.err.println( + s"blobExec: no exit from [$command] ${latchBudget}s after start on blob $origSha " + + s"($fullPath); destroying the direct child" + + (if (groupKilled) "" else " (no GNU timeout: grandchildren may survive)") + ) + proc.destroy() + (TimeoutExitCode, Array.emptyByteArray, "") + } else + exitHolder.get() match { + case NotExitedSentinel => + // The waiter was interrupted (e.g. pool.shutdownNow) before any status + // was observed. The blob may well have succeeded, but we cannot claim + // that, so it is reported as a timeout: Skip, never Replace. + System.err.println( + s"blobExec: wait for blob $origSha ($fullPath) was interrupted before the child " + + "reported a status; treating it as a timeout and discarding its output" + ) + (TimeoutExitCode, Array.emptyByteArray, "") + case code if groupKilled && TimeoutStatuses.contains(code) => + System.err.println( + s"blobExec: timeout after ${secs}s on blob $origSha ($fullPath); " + + s"child process group killed by `timeout` (status $code)" + ) + (TimeoutExitCode, Array.emptyByteArray, "") + case code => + (code, stdoutBuilder.toByteArray, stderrBuilder.toString) + } + } + + private def transfer(in: InputStream, out: java.io.OutputStream): Unit = { + val buf = new Array[Byte](8192) + Iterator + .continually(in.read(buf)) + .takeWhile(_ != -1) + .foreach(n => out.write(buf, 0, n)) } private def logError( @@ -106,14 +323,11 @@ object BlobExec { System.err.println( s"Warning: error executing command [$command] on blob $origSha at path [$fullPath]: exit code $exitCode" ) - printStderr(command, origSha, fullPath, stderr) - } - - private def printStderr(command: String, origSha: String, fullPath: String, stderr: String): Unit = if (stderr.nonEmpty) { System.err.println(s"--- stderr from $command on $origSha ($fullPath) ---") System.err.print(stderr) if (!stderr.endsWith("\n")) System.err.println() System.err.println("--- end stderr ---") } + } } diff --git a/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala b/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala deleted file mode 100644 index ec96ff8be..000000000 --- a/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala +++ /dev/null @@ -1,114 +0,0 @@ -package cregit.blobexec - -import java.io.{InputStream, OutputStream} -import java.nio.charset.StandardCharsets.UTF_8 -import java.util.concurrent.{CountDownLatch, TimeUnit} -import scala.jdk.CollectionConverters._ - -/** One run of an external command under a wall-clock budget. Owns the child's - * environment, its pipes, its budget and its kill. */ -final class ChildRunner(budgetSeconds: Int, drainGraceSeconds: Int = ChildRunner.DrainGraceSeconds) { - - import ChildRunner._ - - private val budget: Int = math.max(1, budgetSeconds) - - def run(command: String, stdin: Array[Byte], env: Seq[(String, String)]): Outcome = { - val builder = new ProcessBuilder(command) - env.foreach { case (name, value) => builder.environment.put(name, value) } - val child = builder.start() - - val stdout = new java.io.ByteArrayOutputStream(math.max(stdin.length, StdoutBufferBytes)) - val stderr = new StringBuilder - val readers = new CountDownLatch(2) - - pump("stdin", () => writeThenClose(child.getOutputStream, stdin)) - pump("stdout", () => try copyBytes(child.getInputStream, stdout) finally readers.countDown()) - pump("stderr", () => try copyText(child.getErrorStream, stderr) finally readers.countDown()) - - val exited = - try child.waitFor(budget.toLong, TimeUnit.SECONDS) - catch { - case _: InterruptedException => - Thread.currentThread().interrupt() - false - } - - if (!exited) kill(child, s"no exit within ${budget}s") - else if (!readers.await(drainGraceSeconds.toLong, TimeUnit.SECONDS)) - // A surviving grandchild still holds a pipe, so the output is incomplete - // and the buffers are not safe to read. - kill(child, s"output still open ${drainGraceSeconds}s after exit") - else Outcome.Exited(child.exitValue, stdout.toByteArray, stderr.toString) - } - - private def kill(child: Process, why: String): Outcome = { - val killed = killTree(child) - Outcome.Killed(if (killed > 0) s"$why; killed $killed process(es)" else why) - } -} - -object ChildRunner { - - sealed trait Outcome - object Outcome { - final case class Exited(status: Int, stdout: Array[Byte], stderr: String) extends Outcome - - /** No result that can be trusted. Never carries output. */ - final case class Killed(why: String) extends Outcome - } - - /** Seconds an open pipe is waited for after the child itself has exited. */ - private[blobexec] val DrainGraceSeconds: Int = 5 - - /** Seconds a forced kill needs to take effect. */ - private val KillSettleSeconds: Int = 5 - - private val StdoutBufferBytes: Int = 1024 - - private[blobexec] def maxLifetimeSeconds(budgetSeconds: Int): Int = - Saturating.sum(math.max(1, budgetSeconds), DrainGraceSeconds + KillSettleSeconds) - - /** Kill the child and every process beneath it, and report how many were - * signalled. Signalling the direct child alone leaves grandchildren holding - * this JVM's pipes. */ - private[blobexec] def killTree(child: Process): Int = { - val handle = child.toHandle - // Snapshot before killing anything: a dead parent's children are reparented - // to init and leave the tree, but a handle taken now stays valid. - val tree = handle.descendants().iterator().asScala.toVector - val parent = if (handle.destroyForcibly()) 1 else 0 - val known = tree.count(_.destroyForcibly()) - val late = tree.flatMap(_.descendants().iterator().asScala).count(_.destroyForcibly()) - parent + known + late - } - - private def pump(name: String, body: () => Unit): Unit = { - val t = new Thread(() => body(), s"blobexec-$name") - t.setDaemon(true) // an abandoned reader must never keep the JVM alive - t.start() - } - - private def writeThenClose(out: OutputStream, bytes: Array[Byte]): Unit = - // A command that ignores its input breaks this pipe; that is not a failure. - try { out.write(bytes); out.flush() } - catch { case _: java.io.IOException => () } - finally closeQuietly(out) - - private def copyBytes(in: InputStream, out: java.io.ByteArrayOutputStream): Unit = - try { - val buf = new Array[Byte](8192) - Iterator.continually(in.read(buf)).takeWhile(_ != -1).foreach(n => out.write(buf, 0, n)) - } catch { case _: java.io.IOException => () } finally closeQuietly(in) - - private def copyText(in: InputStream, out: StringBuilder): Unit = - try { - val reader = new java.io.InputStreamReader(in, UTF_8) - val buf = new Array[Char](4096) - Iterator.continually(reader.read(buf)).takeWhile(_ != -1).foreach(n => out.appendAll(buf, 0, n)) - } catch { case _: java.io.IOException => () } finally closeQuietly(in) - - private def closeQuietly(c: java.io.Closeable): Unit = - try c.close() - catch { case _: java.io.IOException => () } -} diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index c608e0358..efcc33307 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -29,82 +29,200 @@ import java.nio.file.{Files, Paths} */ object Main { - /** Memo `meta` key holding the cumulative number of blobs whose command was - * killed for exceeding its budget. Durable because the skip is durable. */ - private[blobexec] val BlobsTimedOutMetaKey = "blobs_timed_out" - - /** Exit status when any blob in this memo has ever timed out: the walk itself - * succeeded, but the output is incomplete and must not be validated. */ - private[blobexec] val TimedOutExitStatus = 4 - - private[blobexec] val ParserCrashedExitStatus = 6 - - /** Upper bound on either timeout flag. A week is already far past any real - * budget, and bounding the input here is what keeps every window derived from - * it inside `Int`. */ - private[blobexec] val MaxTimeoutSeconds: Int = 7 * 86400 - - /** Seconds for `--blob-timeout=` / `--stall-timeout=`: a positive whole number - * within [[MaxTimeoutSeconds]], else None. Zero is rejected rather than read - * as "no limit". */ - - private[blobexec] val ToleratedCrashRate = 0.01 - - private[blobexec] def toleratedCrashes(blobsTokenized: Long): Long = - math.max(1L, (blobsTokenized * ToleratedCrashRate).toLong) - - private[blobexec] def exitStatus(stats: WalkStats): Int = { - val tokenized = stats.blobsRunThroughCommand.toLong + stats.blobsCacheHit.toLong - if (stats.aborted) 2 - else if (stats.blobsTimedOut > 0) TimedOutExitStatus - else if (stats.blobsParserCrashed > toleratedCrashes(tokenized)) ParserCrashedExitStatus - else 0 - } + /** Exit status when `--retokenize` was asked for and would have invalidated + * nothing. + * + * Its own status, and not 3, because it is a different kind of answer. 3 means + * "this memo does not match this run, so the run did not start". This means + * "the run could have started, and the invalidation you asked for was a no-op". + * An operator scripting a corpus-wide re-tokenization needs to tell those + * apart: the first is a refusal to proceed, the second is a request that did + * nothing, and treating the second as success is how a poisoned cache gets + * published. 7 is the next free status after 1, 2, 3, 4, 5, 6; + * `retokenizeStatusIsUnique` in MainOptionsSpec pins that. */ + private[blobexec] val RetokenizeIneffectiveExitStatus = 7 + + /** The process exit status for a finished walk: 2 on an abort, else 0. + * Excluded blobs (oversized, denylisted, or whose tokenizer failed) are + * named on EXCLUDED lines and do not change it. */ + private[blobexec] def exitStatus(stats: WalkStats): Int = + if (stats.aborted) 2 else 0 /** Value parser for the `--blob-timeout=` / `--stall-timeout=` seconds: a * positive whole number, else None (which the caller reports and exits 1 on). * Zero and negatives are rejected rather than read as "no limit" — an * unbounded blob is the defect this whole change exists to remove. */ private[blobexec] def parsePositiveSeconds(spec: String): Option[Int] = - spec.toIntOption.filter(s => s > 0 && s <= MaxTimeoutSeconds) + spec.toIntOption.filter(_ > 0) + + /** Multiple of `--blob-timeout` used when the stall window has to be widened + * for it, matching the ratio of the two defaults (600 and 1800). */ + private[blobexec] val StallTimeoutMultiple = 3 + + /** An explicit window below the floor is refused; a defaulted one is widened to + * [[widenedStall]]. Overriding a number the operator typed is worse than stopping. */ + private[blobexec] def resolveStallTimeout( + blobTimeoutSeconds: Int, + stallTimeoutSeconds: Int, + stallExplicit: Boolean + ): Either[String, Int] = { + val floor = Walker.stallFloorFor(blobTimeoutSeconds) + if (stallTimeoutSeconds >= floor) Right(stallTimeoutSeconds) + else if (stallExplicit) + Left( + s"--stall-timeout=$stallTimeoutSeconds must be at least ${floor}s to go with " + + s"--blob-timeout=$blobTimeoutSeconds. A blob completing or being killed is the only " + + "progress a pure-blob commit makes, and killing one takes the budget plus the kill " + + "grace, so a smaller stall window kills runs whose blobs are merely slow. Try " + + s"--stall-timeout=${widenedStall(blobTimeoutSeconds)} with --blob-timeout=$blobTimeoutSeconds." + ) + else Right(widenedStall(blobTimeoutSeconds)) + } + + private[blobexec] def widenedStall(blobTimeoutSeconds: Int): Int = + math.max( + math.min(math.max(1, blobTimeoutSeconds).toLong * StallTimeoutMultiple, Int.MaxValue.toLong).toInt, + Walker.stallFloorFor(blobTimeoutSeconds) + ) // `raw` (not `s`): the mask example below contains a regex backslash, which a // processed-escape interpolator rejects. `$$` therefore renders a literal `$`. private val Usage = - raw"""Usage: blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] [--blob-timeout=] [--stall-timeout=] + raw"""Usage: blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] [--mask-widened] [--tokenizer-identity==,...] [--retokenize=,...] [--memo-dir=] [--tokenizer-worker=] [--blob-timeout=] [--stall-timeout=] | | --abort-on-error exit immediately (status 2) on the first non-zero | exit from , instead of skipping that blob + | --mask-widened resume against a blob map recorded under a DIFFERENT + | , keeping the tokenizations already in + | blob_map. Without it a mask change is refused (status + | 3), which is the right default and stays the default. + | Valid only because the mask decides WHICH blobs are + | tokenized, never HOW: the language comes from the + | file's extension, per file, so the same (blob, path) + | yields the same tokens under any mask that selects it. + | + | It is not taken on trust. Two checks run first, and + | NOTHING is written until both pass: + | 1. every path of a tokenized row (orig <> new) must + | still match the new mask. One regex test per row, + | against the data, so a narrowing mislabelled as a + | widening is caught and named. Regexes are never + | compared to each other. + | 2. a spread sample of the retained new_blob ids must + | resolve in . Those ids exist only there, + | so the map is reusable only alongside it. + | + | Then: tree_map, commit_map and ref_map are emptied + | (trees gain entries, commits name trees, refs name + | commits), and so are blob_map's IDENTITY rows + | (orig == new). The identity rows are the subtle part: + | they record "this path was not selected, its bytes pass + | through", and under a wider mask some of those paths + | ARE selected. Keeping them would serve RAW SOURCE as a + | cache hit for precisely the files the widening exists + | to tokenize. + | + | This is a step-2 resume that KEEPS the work directory, + | never a fresh run — a fresh run deletes dst.git, and + | check 2 then refuses. + | --tokenizer-identity==[,=...] + | which tokenizer produced the tokens for each file + | extension, as an opaque value per extension (a digest + | of the parser toolchain; tokenize/tokenizerIdentity.pl + | computes it). Recorded in the blob map's meta table on + | first sight and COMPARED on every later run. + | + | This closes a hole that could not. + | is the constant path tokenizeByBlobId/tokenBySha.pl, so + | when a tokenizer behind it is corrected, nothing the + | reuse decision looks at changes: every cached row for + | that language stays a cache hit and the run reproduces + | the OLD tokenizer's output with no error anywhere. That + | happened — a rustTokenizer binary 16 days older than + | its source kept emitting a `line:col` prefix, and + | 741,869 .rs entries in 45 projects carried it. + | + | A mismatch REFUSES the run (status 3) and names the + | flag to fix it. It never invalidates anything by + | itself, which is why comparing is safe as a default: + | the 186 published projects are not touched. An + | extension with nothing recorded is not a mismatch — + | that is what every blob map built before this flag + | looks like — so it is simply recorded. + | --retokenize=[,...] + | the opt-in past that refusal: invalidate the cached + | tokenizations of these extensions and nothing else. + | Requires --tokenizer-identity and --memo-dir, and a + | step-2 resume (the ids live in ). + | + | SELECTIVE on purpose. Re-tokenizing is 88% of total + | pipeline time, so a defect in one language's tokenizer + | must cost one language's entries. Deliberately NOT + | --drop-memo, which destroys 2.6 M memoized + | tokenizations wholesale, and deliberately not + | --mask-widened, which answers a different question. + | + | BOTH cache layers go, together, and neither can be + | done without the other: + | - blob_map's tokenized rows on those extensions, and + | - their entries in the memo under --memo-dir. The + | memo is keyed on sha1(contents) ALONE + | (tokenBySha.pl:76) with no tokenizer in the key, so + | dropping only the blob_map row just moves the stale + | answer one layer down. + | tree_map, commit_map and ref_map go too: a tree names + | its blobs, and a retained tree_map row short-circuits + | the re-walk of the subtree holding the file. + | + | IT CANNOT QUIETLY DO NOTHING. It refuses, before + | changing anything, when: another extension's tokenizer + | also changed and was not named; no tokenized row + | carries any named extension (status ${RetokenizeIneffectiveExitStatus}); the memo held + | none of the affected blobs and --memo-dir is not + | $$BFG_MEMO_DIR, the memo tokenBySha.pl reads (status ${RetokenizeIneffectiveExitStatus}); + | or a RETAINED new_blob id does not resolve in . + | --memo-dir= the memo directory ($$BFG_MEMO_DIR) whose entries + | --retokenize must purge. Only read with --retokenize; + | passing it alone is an error rather than a no-op. + | --tokenizer-worker= + | send each blob to one of a pool of persistent + | tokenizer processes (tokenizeByBlobId/tokenWorker.pl) + | instead of running once per blob. Its + | output is the same. Needs --pipeline or --pipeline-trees. | --blob-timeout= - | budget for one invocation (default ${BlobExec.DefaultTimeoutSeconds}). - | A child that exceeds it is killed with everything - | beneath it; that blob is left untokenized, the walk - | records nothing for its commit, and the run exits - | ${TimedOutExitStatus} so an incomplete project cannot be published. - | Re-running retries exactly those blobs. + | wall-clock budget for one invocation + | (default ${BlobExec.DefaultTimeoutSeconds}). A child that exceeds it is + | killed (whole process group) and that blob is + | excluded as failed (see below); the run continues. | --stall-timeout= - | watchdog window (default ${Walker.DefaultStallTimeoutSeconds}, derived from - | --blob-timeout). If nothing completes anywhere in this - | window the run exits ${Walker.StalledExitStatus}. Neither flag normally - | needs setting: the window follows the budget, and a - | window that one blob's lifetime could trip is refused. + | watchdog window (default ${Walker.DefaultStallTimeoutSeconds}); must clear one + | blob's whole lifetime (--blob-timeout plus the kill + | path), since a pure-blob commit's only progress is a + | blob finishing or being killed. If it is not, an + | explicit value is refused and a defaulted one is raised + | to ${StallTimeoutMultiple}x --blob-timeout or that floor. If no blob, tree, + | commit or blob copy completes anywhere in this window, + | the run is stuck in a way the per-blob kill did not + | cover: it is reported with the work in flight and the + | process is killed with status ${Walker.StalledExitStatus}. The memo is + | durable, so re-running resumes. | - | Blobs on the shipped denylist (${BlobDenylist.EntriesSource}) are never - | handed to : they are dropped from the rewritten trees, counted as - | blobsDenylisted, named with their reason and citation on an EXCLUDED line, - | and they do NOT change the exit status. srcML 1.1.0 does not terminate on - | the listed blobs, the defect is diagnosed and cited upstream, and a - | diagnosed exclusion must not block publication the way an unexplained - | timeout does. + | Blobs on the shipped denylist (${BlobDenylist.ResourcePath} inside this + | jar) are never handed to : they are dropped from the rewritten + | trees, counted as blobsDenylisted, named with their reason and citation on + | an EXCLUDED line, and they do NOT change the exit status. srcML 1.1.0 does + | not terminate on the listed blobs, and the defect is diagnosed and cited + | upstream, so the list saves the --blob-timeout each run would spend. + | + | A blob whose tokenizer times out or reports a parser crash is excluded + | the same way: dropped from the rewritten trees, counted as blobsTimedOut + | or blobsParserCrashed, named on an 'EXCLUDED failed blob' line, and tried + | only once per run. It does NOT change the exit status. | | Exit status: 0 = clean, 1 = usage, 2 = aborted on a command error, - | 3 = memo meta mismatch, ${TimedOutExitStatus} = completed - | but some blob timed out (output incomplete, re-run to - | retry), ${Walker.StalledExitStatus} = killed by the stall watchdog, - | ${ParserCrashedExitStatus} = tokenizer crashes past ${(ToleratedCrashRate * 100).toInt}% of the blobs - | tokenized. Below that the blob is excluded and the run - | still publishes: a crash is deterministic, so one - | pathological input must not cost a whole project. + | 3 = memo meta mismatch (including a changed tokenizer with + | no --retokenize), ${Walker.StalledExitStatus} = killed by the stall watchdog, + | ${RetokenizeIneffectiveExitStatus} = --retokenize would have invalidated nothing. | --pipeline use the look-ahead pipelined walker (producer runs | ahead so the blob-command pool stays saturated); | output is identical to the default serial walker @@ -145,10 +263,40 @@ object Main { var blobTimeoutSeconds = BlobExec.DefaultTimeoutSeconds var stallTimeoutSeconds = Walker.DefaultStallTimeoutSeconds var stallExplicit = false + var maskWidened = false + var tokenizerIdentity = TokenizerIdentity.empty + var retokenizeExtensions: Set[String] = Set.empty + var memoDir: Option[java.nio.file.Path] = None + var tokenizerWorker: Option[java.nio.file.Path] = None flags.foreach { case "--abort-on-error" => abortOnError = true case "--pipeline" => pipeline = true case "--pipeline-trees" => pipelineTrees = true + case "--mask-widened" => maskWidened = true + case t if t.startsWith("--tokenizer-identity=") => + TokenizerIdentity.parse(t.stripPrefix("--tokenizer-identity=")) match { + case Right(id) => tokenizerIdentity = id + case Left(why) => + System.err.println(s"Error: --tokenizer-identity: $why") + sys.exit(1) + } + case t if t.startsWith("--retokenize=") => + TokenizerIdentity.parseExtensions(t.stripPrefix("--retokenize=")) match { + case Right(exts) => retokenizeExtensions = exts + case Left(why) => + System.err.println(s"Error: --retokenize: $why") + sys.exit(1) + } + case t if t.startsWith("--memo-dir=") => + val p = Paths.get(t.stripPrefix("--memo-dir=")) + if (!Files.isDirectory(p)) { + System.err.println( + s"Error: --memo-dir [$p] is not a directory. It must be the SAME memo this project's " + + "tokenizations were written into, or the invalidation would leave the real memo's " + + "stale entries in place.") + sys.exit(1) + } + memoDir = Some(p) case s if s.startsWith("--shard=") => val spec = s.stripPrefix("--shard=") spec.split("/", -1) match { @@ -171,6 +319,8 @@ object Main { sys.exit(1) } warmPath = Some(p) + case t if t.startsWith("--tokenizer-worker=") => + tokenizerWorker = Some(Paths.get(t.stripPrefix("--tokenizer-worker="))) case t if t.startsWith("--blob-timeout=") => val spec = t.stripPrefix("--blob-timeout=") parsePositiveSeconds(spec) match { @@ -193,12 +343,12 @@ object Main { sys.exit(1) } - Walker.resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) match { + resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) match { case Right(secs) => if (secs != stallTimeoutSeconds) { System.err.println( s"blobExec: raising the stall window from ${stallTimeoutSeconds}s to ${secs}s, because " + - s"--blob-timeout=${blobTimeoutSeconds}s needs a watchdog window larger than itself. " + + s"--blob-timeout=${blobTimeoutSeconds}s needs a window that clears one blob's whole lifetime. " + "Pass --stall-timeout explicitly to choose your own." ) stallTimeoutSeconds = secs @@ -214,12 +364,84 @@ object Main { sys.exit(1) } + // A shard writes its own fresh dst.git and blobmap.db every run, so there is + // never a recorded mask for --mask-widened to widen. Refusing says so instead + // of letting the flag be a silent no-op; --warm is the sharded equivalent. + if (shard.isDefined && maskWidened) { + System.err.println("Error: --mask-widened has nothing to do under --shard: each shard builds a " + + "fresh dst.git and blobmap.db, so no mask is recorded to widen. Reuse a prior run's " + + "tokenizations with --warm= instead.") + System.err.println(Usage) + sys.exit(1) + } + + // Every way --retokenize could end up doing nothing, refused before the walk. + // A flag whose whole purpose is to force work must never be able to run and + // force none. + if (retokenizeExtensions.nonEmpty) { + if (tokenizerIdentity.isEmpty) { + System.err.println( + "Error: --retokenize needs --tokenizer-identity. Invalidating the entries without " + + "recording which tokenizer replaces them leaves nothing for the next run to detect a " + + "change against, so the next tokenizer defect would be just as silent as this one.") + System.err.println(Usage) + sys.exit(1) + } + if (memoDir.isEmpty) { + System.err.println( + "Error: --retokenize needs --memo-dir. There are two caches, not one: dropping a blob_map " + + "row makes the walker re-run , and is tokenizeByBlobId/tokenBySha.pl, " + + "which answers from $BFG_MEMO_DIR keyed on sha1(contents) with no tokenizer in the key. " + + "Invalidating one layer without the other invalidates nothing at all.") + System.err.println(Usage) + sys.exit(1) + } + val unknown = retokenizeExtensions -- tokenizerIdentity.extensions + if (unknown.nonEmpty) { + System.err.println( + s"Error: --retokenize names extension(s) --tokenizer-identity does not cover: " + + s"${unknown.toVector.sorted.mkString(", ")}. Known: " + + s"${tokenizerIdentity.extensions.toVector.sorted.mkString(", ")}.") + sys.exit(1) + } + if (maskWidened) { + System.err.println( + "Error: --mask-widened and --retokenize cannot be combined. Each verifies its own " + + "precondition against the rows, and together each would verify against a state the " + + "other is about to change. Widen first, then resume with --retokenize.") + sys.exit(1) + } + if (shard.isDefined) { + System.err.println( + "Error: --retokenize has nothing to do under --shard: each shard builds a fresh dst.git " + + "and blobmap.db, so there are no cached tokenizations to invalidate. Retokenize the " + + "merged result, or drop the shards' --warm=.") + sys.exit(1) + } + } else if (memoDir.isDefined) { + System.err.println( + "Error: --memo-dir has no effect without --retokenize. Nothing else in blobExec reads the " + + "memo — tokenizeByBlobId/tokenBySha.pl takes it from $BFG_MEMO_DIR in the environment.") + sys.exit(1) + } + if (shard.isDefined && (pipeline || pipelineTrees)) { System.err.println("Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") System.err.println(Usage) sys.exit(1) } + tokenizerWorker.foreach { path => + if (!Files.isRegularFile(path) || !Files.isExecutable(path)) { + System.err.println(s"Error: --tokenizer-worker [$path] is not an executable file") + sys.exit(1) + } + if (!(pipeline || pipelineTrees)) { + System.err.println("Error: --tokenizer-worker needs --pipeline or --pipeline-trees") + sys.exit(1) + } + } + if (positional.length != 5) { System.err.println(Usage) sys.exit(1) @@ -248,7 +470,16 @@ object Main { val dbParent = dbPath.getParent if (dbParent != null && !Files.isDirectory(dbParent)) Files.createDirectories(dbParent) - val denylist = BlobDenylist.shipped + // Loaded here rather than on first use: a jar built without the resource, or a + // malformed line in it, must stop the run now and say so, not silently hand a + // known non-terminating blob to srcml an hour into the walk. + val denylist = + try BlobDenylist.shipped + catch { + case e: Exception => + System.err.println(s"Error: cannot read the blob denylist: ${e.getMessage}") + sys.exit(1) + } val incremental = Files.isDirectory(dstPath) val shardStr = shard.map { case (k, n) => s"$k/$n" }.getOrElse("none") @@ -258,35 +489,111 @@ object Main { s"abortOnError=$abortOnError pipeline=$pipeline pipelineTrees=$pipelineTrees " + s"shard=$shardStr warm=$warmStr blobTimeout=${blobTimeoutSeconds}s " + s"stallTimeout=${stallTimeoutSeconds}s incremental=$incremental " + - s"denylistEntries=${denylist.size}" + s"maskWidened=$maskWidened denylistEntries=${denylist.size} " + + s"tokenizerIdentity=${if (tokenizerIdentity.isEmpty) "none" else tokenizerIdentity.render} " + + s"retokenize=${if (retokenizeExtensions.isEmpty) "none" else retokenizeExtensions.toVector.sorted.mkString(",")} " + + s"memoDir=${memoDir.map(_.toString).getOrElse("none")} " + + s"tokenizerWorker=${tokenizerWorker.map(_.toString).getOrElse("none")}" ) val src: FileRepository = openSrc(srcPath) val dst: FileRepository = openOrInitDst(dstPath) - val mapping = try Mapping.open(dbPath, command, mask, warmPath) catch { + + // The reachability probe the widening needs. It reads dst, which is why the + // whole decision lives here and not inside Mapping.open's signature alone: + // `blob_map` is only meaningful next to the repository its new_blob ids are in. + val widening = + if (!maskWidened) None + else Some(Mapping.MaskWidening( + newBlobResolves = id => { + val reader = dst.newObjectReader() + try reader.has(org.eclipse.jgit.lib.ObjectId.fromString(id)) + catch { case _: IllegalArgumentException => false } + finally reader.close() + }, + report = msg => println(s"blobExec: $msg") + )) + + // The invalidation, wired to the two things only this layer can supply: the + // dst reachability probe, and a memo purge that reads the original blobs out + // of src to recompute their content sha1 — the memo's only key + // (tokenizeByBlobId/tokenBySha.pl:76). + val retokenize = + if (retokenizeExtensions.isEmpty) None + else Some(Mapping.Retokenize( + extensions = retokenizeExtensions, + newBlobResolves = id => { + val reader = dst.newObjectReader() + try reader.has(org.eclipse.jgit.lib.ObjectId.fromString(id)) + catch { case _: IllegalArgumentException => false } + finally reader.close() + }, + purgeMemo = blobs => TokenizerMemo.purge(memoDir.get, blobs, sha => { + val reader = src.newObjectReader() + try Some(reader.open(org.eclipse.jgit.lib.ObjectId.fromString(sha), + org.eclipse.jgit.lib.Constants.OBJ_BLOB).getBytes) + catch { case _: Exception => None } + finally reader.close() + }), + report = msg => println(s"blobExec: $msg"), + memoDirIsTokenizerMemo = + memoDir.exists(d => TokenizerMemo.isTokenizerMemo(d, sys.env.get("BFG_MEMO_DIR"))) + )) + + val mapping = try Mapping.open(dbPath, command, mask, warmPath, widening, + tokenizerIdentity, retokenize) catch { case m: Mapping.MetaMismatchException => System.err.println(s"Error: ${m.getMessage}") + if (!maskWidened && m.getMessage.contains("mask")) + System.err.println( + "Hint: if the new mask is a strict superset of the recorded one, --mask-widened reuses " + + "the tokenizations already in blob_map instead of redoing them. It verifies that " + + "against the rows themselves and refuses if it is not true. It requires the work " + + "directory to be intact (resume at step 2), because blob_map's ids live in dst." + ) + src.close(); dst.close() + sys.exit(3) + case n: Mapping.MaskNarrowedException => + System.err.println(s"Error: ${n.getMessage}") + src.close(); dst.close() + sys.exit(3) + case d: Mapping.DanglingNewBlobException => + System.err.println(s"Error: ${d.getMessage}") + src.close(); dst.close() + sys.exit(3) + case t: Mapping.TokenizerChangedException => + System.err.println(s"Error: ${t.getMessage}") src.close(); dst.close() sys.exit(3) + case n: Mapping.NothingInvalidatedException => + // Deliberately NOT 0. The walk could have run; the invalidation asked for + // could not. Exiting 0 here would let a corpus script publish a project + // whose poisoned entries were never touched. + System.err.println(s"Error: ${n.getMessage}") + src.close(); dst.close() + sys.exit(RetokenizeIneffectiveExitStatus) + case i: IllegalArgumentException => + // Mapping.open's own preconditions on the flag combinations, re-checked + // there so a library caller cannot bypass what main() validates. + System.err.println(s"Error: ${i.getMessage}") + src.close(); dst.close() + sys.exit(1) } - // The cumulative figure is forensic only: gating on it would make one past - // timeout permanent. - val (stats, timedOutEver) = try { + val stats = try { val parallelism = math.max(1, Runtime.getRuntime.availableProcessors) + val workerPool = tokenizerWorker.map(path => new TokenizerWorkerPool(path.toString, parallelism)) val walker = new Walker( src, dst, mapping, mask.r, command, abortOnError, parallelism, pipeline, pipelineTrees, shard, destinationMayContainObjects = incremental, blobTimeoutSeconds = blobTimeoutSeconds, stallTimeoutSeconds = stallTimeoutSeconds, + workerPool = workerPool, denylist = denylist ) - val s = walker.run() - val prior = mapping.getMeta(BlobsTimedOutMetaKey).flatMap(_.toLongOption).getOrElse(0L) - val total = prior + s.blobsTimedOut - if (s.blobsTimedOut > 0) mapping.setMeta(BlobsTimedOutMetaKey, total.toString) - (s, total) + try walker.run() + finally workerPool.foreach(_.close()) } finally { mapping.close() dst.close() @@ -307,7 +614,6 @@ object Main { s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + s"refsProjected=${stats.refsProjected} " + s"blobsTimedOut=${stats.blobsTimedOut} " + - s"blobsTimedOutEver=$timedOutEver " + s"blobsOversized=${stats.blobsOversized} " + s"blobsDenylisted=${stats.blobsDenylisted} " + s"blobsParserCrashed=${stats.blobsParserCrashed} " + @@ -315,10 +621,9 @@ object Main { ) if (stats.blobsDenylisted > 0) { - // Reported, never fatal. System.err.println( s"blobExec: ${stats.blobsDenylisted} blob(s) were excluded by the blob denylist " + - s"(${BlobDenylist.EntriesSource}, ${denylist.size} entr" + + s"(${BlobDenylist.ResourcePath} in this jar, ${denylist.size} entr" + s"${if (denylist.size == 1) "y" else "ies"}). Each one is named with its sha, path, " + "reason and upstream citation on an 'EXCLUDED denylisted blob' line above; those " + "lines and that file are the record of what this project's dataset does not contain. " + @@ -329,37 +634,34 @@ object Main { } if (stats.blobsOversized > 0) { + // Reported, never fatal. An oversized blob is deterministic and fully + // explained by the EXCLUDED lines above, so gating publication on it would + // only mean this project could never publish while telling us nothing new. System.err.println( - s"blobExec: ${stats.blobsOversized} blob(s) were excluded as oversized. Each is named " + - "with its sha, path and size on an 'EXCLUDED oversized blob' line above. Reported, " + - "never fatal." - ) - } - - if (stats.blobsTimedOut > 0) { - System.err.println( - s"blobExec: INCOMPLETE, DO NOT PUBLISH: ${stats.blobsTimedOut} blob(s) timed out this " + - s"run ($timedOutEver ever for this memo). Resume at step 2 to retry exactly those " + - s"blobs, or raise --blob-timeout (currently ${blobTimeoutSeconds}s)." + s"blobExec: ${stats.blobsOversized} blob(s) were excluded as oversized (>= " + + s"${Walker.MaxBlobBytes} bytes, JGit's stream-file threshold). Each one is named with " + + "its sha, path and size on an 'EXCLUDED oversized blob' line above; those lines are the " + + "record of what this project's dataset does not contain. The files are absent from the " + + "tokenized repository, not present as raw source, so they produce no blame and no " + + "dataset row. This is not a failure and does not affect the exit status." ) } - if (stats.blobsParserCrashed > 0) { - val tokenized = stats.blobsRunThroughCommand.toLong + stats.blobsCacheHit.toLong - val gated = stats.blobsParserCrashed > toleratedCrashes(tokenized) + if (stats.blobsTimedOut + stats.blobsParserCrashed > 0) { System.err.println( - s"blobExec: ${stats.blobsParserCrashed} of $tokenized blob(s) crashed their tokenizer " + - s"and were excluded; each is named above. " + - (if (gated) s"That is past the ${ToleratedCrashRate * 100}% this run tolerates: " + - "DO NOT PUBLISH. Deterministic, so re-running will not clear it." - else "Within tolerance, so the run still publishes.") + s"blobExec: ${stats.blobsTimedOut + stats.blobsParserCrashed} blob(s) were excluded because " + + s"their tokenizer failed (${stats.blobsTimedOut} timed out after ${blobTimeoutSeconds}s, " + + s"${stats.blobsParserCrashed} reported a parser crash). Each one is named on an " + + "'EXCLUDED failed blob' line above; those lines are the record of what this project's " + + "dataset does not contain. This does not affect the exit status." ) } - val status = exitStatus(stats) - if (status != 0) System.err.println(s"blobExec: exiting $status") - // Exit explicitly: abandoned daemon readers must not decide JVM exit. - sys.exit(status) + // Always exit explicitly. A timed-out blob abandons its (daemon) reader + // threads while they are blocked on a pipe, and on the pre-fix build the + // JVM outlived the finished walk on exactly those threads — a silent stall + // behind a done-line that read `aborted=false`. + sys.exit(exitStatus(stats)) } private def openSrc(path: java.nio.file.Path): FileRepository = { diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala index 11814b18d..3c09c4911 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala @@ -2,6 +2,7 @@ package cregit.blobexec import java.nio.file.Path import java.sql.{Connection, DriverManager, PreparedStatement} +import scala.util.matching.Regex /** SQLite-backed persistent mapping between original and rewritten object ids. * One JDBC connection confined to the walker thread (only the external command @@ -104,6 +105,83 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends def setMeta(key: String, value: String): Unit = Mapping.execute(insMeta, key, value) + /** The tokenizer identity recorded for `extension`, if any. */ + def storedTokenizerId(extension: String): Option[String] = + getMeta(Mapping.TokenizerIdKeyPrefix + extension) + + def setTokenizerId(extension: String, value: String): Unit = + setMeta(Mapping.TokenizerIdKeyPrefix + extension, value) + + /** Every extension whose recorded tokenizer identity differs from `identity`. + * + * An extension with nothing recorded is NOT a change: it is a map written + * before identities were recorded at all, which describes every blob map on + * this machine today. Treating that as a change would refuse all 188 + * projects at once and say nothing true. */ + def tokenizerIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = + identity.byExtension.toVector.sorted.flatMap { case (ext, requested) => + storedTokenizerId(ext) match { + case Some(stored) if stored != requested => + Some(Mapping.IdentityChange(ext, stored, requested)) + case _ => None + } + } + + /** The subset of [[tokenizerIdentityChanges]] that can actually poison this run: + * the extensions whose tokenizer moved AND which have cached tokenizations to + * reuse. + * + * The filter is not an optimisation, it is what keeps the mechanism usable. The + * pipeline reports one identity per extension it can parse, for every project, + * because the identity is a property of the checkout and not of the project. A + * Java-only project therefore carries a `c` identity it has no rows for — and + * refusing that project because the C toolchain changed would be a refusal with + * no remedy: `--retokenize=c` would then fail too, correctly, with "no tokenized + * row carries that extension". A deadlock where the only way forward is to + * delete the work. + * + * With no rows there is nothing to serve from cache, so the new identity is + * simply recorded: whatever gets tokenized from now on is tokenized by the + * tokenizer that is recorded. */ + def cachePoisoningIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = + tokenizerIdentityChanges(identity) + .filter(c => countTokenizedRowsForExtensions(Set(c.extension)) > 0L) + + /** How many TOKENIZED rows sit on a path with one of these extensions. Zero is + * what makes a `--retokenize` request a no-op, and a no-op has to fail. */ + def countTokenizedRowsForExtensions(extensions: Set[String]): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery( + s"SELECT COUNT(*) FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + + /** The distinct ORIGINAL blob ids of the tokenized rows for these extensions. + * Distinct because the memo is keyed on content, so the same blob reached + * under two paths is one memo entry, and hashing it twice is wasted reads. + * + * Materialised rather than streamed: the caller hands the whole vector to the + * memo purge, which must complete before anything is deleted from the map. + * The corpus figure is 741,869 rows for a `.rs` invalidation, which is a + * vector of 40-character strings — tens of megabytes, not gigabytes. */ + def tokenizedOrigBlobsForExtensions(extensions: Set[String]): Vector[String] = { + val st = conn.createStatement() + try { + val rs = st.executeQuery( + s"SELECT DISTINCT orig_blob FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + try Iterator + .continually(if (rs.next()) Some(rs.getString(1)) else None) + .takeWhile(_.isDefined) + .flatten + .toVector + finally rs.close() + } finally st.close() + } + /** All original-commit SHAs already recorded. Used to mark walk frontier. */ def allCommitOrigShas: Vector[String] = { val st = conn.createStatement() @@ -118,6 +196,206 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } + /** How many blob_map rows satisfy `predicate`. */ + private def countWhere(predicate: String): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(s"SELECT COUNT(*) FROM blob_map WHERE $predicate") + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + + /** Row counts of the four maps, for the mask-widening report. */ + def rowCounts: Mapping.RowCounts = { + def one(sql: String): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(sql) + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + Mapping.RowCounts( + blobTokenized = one(s"SELECT COUNT(*) FROM blob_map WHERE ${Mapping.TokenizedPredicate}"), + blobIdentity = one(s"SELECT COUNT(*) FROM blob_map WHERE NOT (${Mapping.TokenizedPredicate})"), + tree = one("SELECT COUNT(*) FROM tree_map"), + commit = one("SELECT COUNT(*) FROM commit_map"), + ref = one("SELECT COUNT(*) FROM ref_map") + ) + } + + /** The first `blob_map` path of a TOKENIZED row (orig_blob <> new_blob) that + * `newMask` no longer selects, or None when every one still matches. + * + * Restricted to tokenized rows on purpose, and that restriction is the whole + * correctness argument. `blob_map` also holds IDENTITY rows (orig == new) for + * every blob the walker merely passed through — `README.md`, `CMakeLists.txt`, + * a `.pdf` — written by Walker's `unchangedBlobs`. Those paths match no mask + * that ever existed, so a check over all rows refuses every project on this + * machine (measured: 198 of 198). A tokenized row, by contrast, exists only + * because the old mask selected its path, so "does the new mask still select + * it?" is exactly the superset question, asked of the data rather than of two + * regexes. + * + * Matched against the BASENAME, because that is what Walker matches + * (`fileMask.findFirstIn(tw.getNameString)`). Anchoring makes the two + * equivalent for every mask this pipeline has used, but replicating the + * walker is the point: the check must answer what the walker will do, not + * what a reader assumes it does. + * + * Streams the result set rather than materialising it: torvalds__linux holds + * 2.7 M tokenized rows. */ + def firstPathNarrowedBy(newMask: Regex): Option[String] = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(s"SELECT path FROM blob_map WHERE ${Mapping.TokenizedPredicate}") + try { + var offender: Option[String] = None + while (offender.isEmpty && rs.next()) { + val path = rs.getString(1) + if (newMask.findFirstIn(Mapping.basename(path)).isEmpty) offender = Some(path) + } + offender + } finally rs.close() + } finally st.close() + } + + /** Up to `size` (path, new_blob) pairs from the TOKENIZED rows, spread across + * the table rather than taken from its head, so the sample is not confined to + * whatever the first commit happened to touch. Deterministic: the same DB + * gives the same sample, so a refusal is reproducible. */ + def sampleTokenizedNewBlobs(size: Int, excludeExtensions: Set[String] = Set.empty): Vector[(String, String)] = { + require(size > 0, "sample size must be positive") + // The probe must ask about the rows that will be KEPT. Under --retokenize the + // rows for the named extensions are about to be deleted, and their new_blob + // ids may legitimately have been garbage-collected out of dst already; asking + // about those would refuse a correct invalidation. + val keep = + if (excludeExtensions.isEmpty) Mapping.TokenizedPredicate + else s"${Mapping.TokenizedPredicate} AND NOT ${Mapping.extensionPredicate(excludeExtensions)}" + val total = countWhere(keep) + if (total == 0L) return Vector.empty + val stride = math.max(1L, total / size) + val st = conn.prepareStatement( + s"""SELECT path, new_blob FROM ( + | SELECT path, new_blob, ROW_NUMBER() OVER (ORDER BY orig_blob, path) AS rn + | FROM blob_map WHERE $keep + |) WHERE (rn - 1) % ? = 0 LIMIT ?""".stripMargin) + try { + st.setLong(1, stride) + st.setInt(2, size) + val rs = st.executeQuery() + try Iterator + .continually(if (rs.next()) Some((rs.getString(1), rs.getString(2))) else None) + .takeWhile(_.isDefined) + .flatten + .toVector + finally rs.close() + } finally st.close() + } + + /** Discard everything a wider mask invalidates and record the new mask. + * + * Four deletions, in one transaction, each for its own reason: + * + * - `commit_map`: a commit names a tree, and every tree gains entries, so + * every rewritten commit id changes. + * - `ref_map`: its rows name commits. The FK cascades from `commit_map`, but + * deleting it explicitly means this does not depend on a per-connection + * pragma being on. + * - `tree_map`: trees gain entries. + * - `blob_map` IDENTITY rows only. This is the load-bearing one. An identity + * row says "this path was not selected, so its bytes pass through". Under a + * wider mask some of those paths ARE selected now, and `getBlob` would + * serve the identity row as a cache hit — putting RAW SOURCE into the + * tokenized repository, invisibly, for exactly the files this widening + * exists to tokenize. Measured on this corpus: 456,363 such rows across 88 + * of 198 projects. Identity rows on still-unselected paths are never read + * (`getBlob` is only called inside the mask branch), so dropping all of + * them is free and re-inserting them is a walk, not a tokenize. + * + * What survives is every row where orig_blob <> new_blob: a real + * tokenization, keyed by (blob, path), so same content and same extension and + * therefore same language and same tokens. That is 94.4% of the work. + * + * Returns the counts before and after, for the caller to report. */ + def applyMaskWidening(oldMask: String, newMask: String, atEpochSeconds: Long): Mapping.WideningResult = { + val before = rowCounts + inTx { + val st = conn.createStatement() + try { + st.executeUpdate("DELETE FROM commit_map") + st.executeUpdate("DELETE FROM ref_map") + st.executeUpdate("DELETE FROM tree_map") + st.executeUpdate(s"DELETE FROM blob_map WHERE NOT (${Mapping.TokenizedPredicate})") + } finally st.close() + setMeta("mask", newMask) + setMeta(Mapping.MaskWidenedFromKey, oldMask) + setMeta(Mapping.MaskWidenedAtKey, atEpochSeconds.toString) + } + Mapping.WideningResult(before, rowCounts) + } + + /** Discard everything a changed tokenizer invalidates for `extensions`, and + * record the new identity for them. + * + * Four deletions and the identity update, in ONE transaction, because a + * partially invalidated map is worse than either a stale one or a clean one: + * it is a state no flag describes and no later run can detect. + * + * - `blob_map` TOKENIZED rows for these extensions. These are the poisoned + * entries: same bytes, same path, but produced by the tokenizer that has + * since been corrected. + * - `tree_map`. A tree names the blobs under it, so a tree containing a + * re-tokenized file has a different id; and a retained `tree_map` row + * short-circuits the re-walk of that whole subtree (buildTreePlan consults + * `getTree` before it looks at anything below), so keeping it would leave + * the tree pointing at the old token blob and invalidate nothing. + * - `commit_map`, because commits name trees, and `ref_map`, because refs + * name commits. ref_map would cascade from commit_map's FK, but only if the + * per-connection pragma is on, so it is deleted explicitly. + * + * What survives is every tokenized row for every OTHER extension. That is the + * whole point: re-tokenizing a corpus costs 88% of total pipeline time, and a + * `.rs`-only defect has no business touching the C work. Trees and commits are + * rebuilt by a walk, which is the cheap part. + * + * Identity pass-through rows (orig == new) are kept. They record "this path was + * not selected, its bytes pass through", the mask has not changed here, and so + * the same paths are still not selected. (That is exactly the case + * `--mask-widened` must handle differently, which is why the two flags refuse + * to run together.) */ + def applyRetokenize( + extensions: Set[String], + identity: TokenizerIdentity, + memo: TokenizerMemo.PurgeReport, + atEpochSeconds: Long + ): Mapping.RetokenizeResult = { + val before = rowCounts + val invalidated = countTokenizedRowsForExtensions(extensions) + val previousIds = extensions.toVector.sorted.flatMap(e => storedTokenizerId(e).map(v => s"$e=$v")) + inTx { + val st = conn.createStatement() + try { + st.executeUpdate("DELETE FROM commit_map") + st.executeUpdate("DELETE FROM ref_map") + st.executeUpdate("DELETE FROM tree_map") + st.executeUpdate( + s"DELETE FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + } finally st.close() + // Every extension the run knows about gets its identity recorded, not just + // the invalidated ones: an extension with nothing recorded is a hole the + // next run cannot detect a change through. + identity.byExtension.foreach { case (ext, value) => + if (extensions.contains(ext) || storedTokenizerId(ext).isEmpty) setTokenizerId(ext, value) + } + setMeta(Mapping.RetokenizedAtKey, atEpochSeconds.toString) + setMeta(Mapping.RetokenizedExtensionsKey, extensions.toVector.sorted.mkString(",")) + setMeta(Mapping.RetokenizedFromKey, previousIds.mkString(",")) + } + Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo) + } + /** Run `body` inside a transaction; commit on success, rollback on throw. */ def inTx[A](body: => A): A = { conn.setAutoCommit(false) @@ -148,6 +426,131 @@ object Mapping { /** Mismatch between the recorded command/mask and the values passed in. */ final class MetaMismatchException(message: String) extends RuntimeException(message) + /** `--mask-widened` was passed but the new mask does not select something the + * old one did, so it is not a widening. Distinct from MetaMismatchException + * because the cause is different — the flag was given and the data refused it, + * rather than the flag being absent — but Main maps both to exit 3, which + * already means "this memo does not match this run". */ + final class MaskNarrowedException(message: String) extends RuntimeException(message) + + /** `--mask-widened` was passed and the map is reusable, but the `new_blob` ids + * it holds do not resolve in dst, so reusing it would dangle every reference. */ + final class DanglingNewBlobException(message: String) extends RuntimeException(message) + + /** A recorded tokenizer identity does not match this run's, and the operator + * did not ask for the affected entries to be invalidated. Reusing them would + * reproduce the tokens of the tokenizer that has since been corrected — + * silently, because nothing downstream can tell one token stream from + * another. Same exit status as a meta mismatch (3): the cause is the same + * class of thing, "this memo does not match this run". */ + final class TokenizerChangedException(message: String) extends RuntimeException(message) + + /** `--retokenize` was passed and would have invalidated nothing. + * + * This exception is the whole safety property of the flag. An invalidation + * that quietly invalidates nothing and exits 0 is indistinguishable, in a log, + * from one that worked — and the operator then publishes a dataset believing + * the defect is out of it. Every path that reaches "nothing changed" raises + * this instead of returning. */ + final class NothingInvalidatedException(message: String) extends RuntimeException(message) + + /** A recorded identity that differs from the one this run reports. */ + final case class IdentityChange(extension: String, stored: String, requested: String) + + /** What distinguishes a real tokenization from a pass-through. A tokenized row + * exists only because the mask selected its path; an identity row exists + * because it did not. Every decision in the mask-widening path turns on this + * one predicate, so it is written once. */ + private[blobexec] val TokenizedPredicate = "orig_blob <> new_blob" + + /** meta keys recording that a widening happened, so the provenance of a reused + * blob_map is in the database rather than only in a log. */ + private[blobexec] val MaskWidenedFromKey = "mask_widened_from" + private[blobexec] val MaskWidenedAtKey = "mask_widened_at" + + /** meta key prefix for the per-extension tokenizer identity: one row per + * extension, `tokenizer_id.rs`, `tokenizer_id.c`. Per extension rather than + * one combined value, because `--retokenize` has to be able to invalidate one + * language without disturbing the recorded state of the others. */ + private[blobexec] val TokenizerIdKeyPrefix = "tokenizer_id." + + /** meta keys recording that an invalidation happened, and what it replaced, so + * a reused blob map carries its own provenance. */ + private[blobexec] val RetokenizedAtKey = "retokenized_at" + private[blobexec] val RetokenizedExtensionsKey = "retokenized_extensions" + private[blobexec] val RetokenizedFromKey = "retokenized_from" + + /** "Does this path carry one of these extensions?", as SQL. + * + * `lower(path) LIKE '%.rs'` rather than a suffix computed in Scala, so the + * whole selection stays one statement over a table that holds 2.7 M rows for + * the largest project. Case-insensitive because the mask is (`(?i)` in + * CregitLanguages::file_mask) and `.C` and `.H` are real files in this corpus. + * + * The dot is part of the pattern, which is what stops `.c` from selecting + * `.cc` and `.cpp`. There is nothing to escape: [[TokenizerIdentity]] admits + * only `[a-z0-9+]+`, an alphabet with no quote, no `%` and no `_`. The require + * is there so that stays true if a future caller builds the set some other + * way. */ + private[blobexec] def extensionPredicate(extensions: Set[String]): String = { + require(extensions.nonEmpty, "extensionPredicate needs at least one extension") + extensions.foreach(e => require( + e.matches(TokenizerIdentity.ExtensionPattern), + s"not a usable extension: [$e] (must match ${TokenizerIdentity.ExtensionPattern})")) + extensions.toVector.sorted.map(e => s"lower(path) LIKE '%.$e'").mkString("(", " OR ", ")") + } + + /** How many `new_blob` ids to probe in dst. Every retained row's id must exist + * there, but reading 2.7 M of them is minutes of object lookup before the walk + * starts; a spread sample catches the failure this guards against, which is a + * dst that was wiped or never written, not one blob gone missing. */ + private[blobexec] val ReachabilitySampleSize = 256 + + final case class RowCounts(blobTokenized: Long, blobIdentity: Long, tree: Long, + commit: Long, ref: Long) + + final case class WideningResult(before: RowCounts, after: RowCounts) + + final case class RetokenizeResult( + before: RowCounts, + after: RowCounts, + extensions: Set[String], + blobsInvalidated: Long, + memo: TokenizerMemo.PurgeReport) + + /** The opt-in behind `--retokenize`, and the four things it has to be given. + * + * `extensions` is what makes it surgical. Re-tokenizing a corpus is 88% of + * total pipeline time, so a defect in one language's tokenizer must invalidate + * one language's entries. + * + * `newBlobResolves` answers "does this object id exist in dst?", for the rows + * that are KEPT. Not optional and with no default, for the same reason as in + * [[MaskWidening]]: `blob_map`'s `new_blob` ids live in dst and nowhere else. + * + * `purgeMemo` is handed the original blob ids being invalidated and must delete + * their entries from the content-addressed memo. It is not optional either, and + * that is the point: the memo is keyed on `sha1(contents)` alone + * (tokenizeByBlobId/tokenBySha.pl:76), with no tokenizer in the key, so + * dropping a `blob_map` row on its own just moves the stale answer one layer + * down. Two layers, one flag, no way to do one without the other. + * + * `report` receives the human-readable account of what was invalidated and + * what was kept. Injected so the decision is testable without capturing + * stdout. */ + final case class Retokenize( + extensions: Set[String], + newBlobResolves: String => Boolean, + purgeMemo: Vector[String] => TokenizerMemo.PurgeReport, + report: String => Unit, + memoDirIsTokenizerMemo: Boolean = false, + nowEpochSeconds: () => Long = () => System.currentTimeMillis() / 1000L, + sampleSize: Int = ReachabilitySampleSize) + + /** The part of `path` Walker matches the mask against. */ + private[blobexec] def basename(path: String): String = + path.substring(path.lastIndexOf('/') + 1) + // processed_at stored INTEGER (epoch seconds) for numeric filters. FK // ref_map.orig_commit -> commit_map ON DELETE CASCADE cleans up refs with their // commit; blob_map/tree_map have no FK (a blob/tree spans many commits). @@ -210,10 +613,59 @@ object Mapping { } finally st.close() } + /** The opt-in behind `--mask-widened`, and the two things it has to be given. + * + * `newBlobResolves` answers "does this object id exist in dst?". It is not + * optional and has no default, because `blob_map`'s `new_blob` ids live in + * dst and nowhere else: reusing the map against a dst that was wiped, or was + * never written, leaves every retained reference dangling and the failure + * surfaces as a corrupt repository several steps later. A default would let a + * caller skip the check by forgetting about it. + * + * `report` receives the human-readable account of what was kept and what was + * dropped. Injected so the decision is testable without capturing stdout. */ + final case class MaskWidening(newBlobResolves: String => Boolean, + report: String => Unit, + nowEpochSeconds: () => Long = () => System.currentTimeMillis() / 1000L, + sampleSize: Int = ReachabilitySampleSize) + /** Open/create the SQLite DB at `path` (schema created if absent) and validate * `command`/`mask` against `meta`: mismatch throws MetaMismatchException, - * absence records them. */ - def open(path: Path, command: String, mask: String, warm: Option[Path] = None): Mapping = { + * absence records them. + * + * `maskWidening` is the ONLY way past the mask refusal, and it is deliberately + * narrow. `command` is still checked exactly as before: a different tokenizer + * means different tokens for the same bytes, and no amount of mask reasoning + * covers that. */ + def open(path: Path, command: String, mask: String, warm: Option[Path] = None, + maskWidening: Option[MaskWidening] = None, + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenize: Option[Retokenize] = None): Mapping = { + // Refused rather than composed. Both invalidations are verified against the + // rows before they touch anything, and running them together would mean + // verifying each against a state the other was about to change: a widening + // decides what to do with identity rows on the assumption the tokenizations + // are valid, and an invalidation decides which tokenizations to drop on the + // assumption the mask has not moved. One at a time, each with its own + // verified report, and the second run is a resume. + require(!(maskWidening.isDefined && retokenize.isDefined), + "--mask-widened and --retokenize cannot be used in the same run: each verifies its own " + + "precondition against the rows, and together each would be verifying against a state the " + + "other is about to change. Run the widening first, then resume with --retokenize.") + retokenize.foreach { r => + require(tokenizerIdentity.nonEmpty, + "--retokenize needs a tokenizer identity to record. Without one the entries would be " + + "invalidated and nothing would be written to detect the next change with, so the very " + + "next run would reuse the new tokenizations without ever knowing which tokenizer made " + + "them.") + val unknown = r.extensions -- tokenizerIdentity.extensions + require(unknown.isEmpty, + s"--retokenize names extension(s) the tokenizer identity does not cover: " + + s"${unknown.toVector.sorted.mkString(", ")}. There is no identity to record for them, so " + + "the invalidation could not be accounted for. Known: " + + s"${tokenizerIdentity.extensions.toVector.sorted.mkString(", ")}.") + } + val url = s"jdbc:sqlite:${path.toAbsolutePath}" val conn = DriverManager.getConnection(url) enableForeignKeys(conn) @@ -223,11 +675,214 @@ object Mapping { val warmConn = warm.map(p => openWarm(p, command, mask)) val m = new Mapping(conn, warmConn) + // command first, and unconditionally: it gates the widening too. checkOrSetMeta(m, "command", command) - checkOrSetMeta(m, "mask", mask) + maskWidening match { + case Some(w) => widenOrCheckMask(m, mask, w) + case None => checkOrSetMeta(m, "mask", mask) + } + // Last, because it is the only check that can delete rows, and it must not do + // so against a map whose command or mask this run has already refused. + retokenize match { + case Some(r) => retokenizeOrRefuse(m, tokenizerIdentity, r) + case None => if (tokenizerIdentity.nonEmpty) checkOrRecordTokenizerIdentity(m, tokenizerIdentity) + } m } + /** The default path: record what is not recorded yet, and refuse if anything + * recorded disagrees. + * + * Refusing is not the same thing as invalidating, and the distinction is the + * whole reason this is safe to make the default. Nothing is deleted here. The + * 186 already-published projects are untouched — a project that is never run + * again is never refused either — and a project that IS re-run with a changed + * tokenizer stops instead of quietly reproducing the old tokens. + * + * An extension with nothing recorded is not a change. That is what every blob + * map in this corpus looks like today, and calling it a change would refuse + * every project at once on the strength of no evidence at all. */ + private def checkOrRecordTokenizerIdentity(m: Mapping, identity: TokenizerIdentity): Unit = { + val changes = m.cachePoisoningIdentityChanges(identity) + if (changes.nonEmpty) throw new TokenizerChangedException(tokenizerChangedMessage(changes)) + // Nothing recorded yet, or recorded but with no rows to poison: record the + // current value either way. Leaving an extension unrecorded is the hole this + // whole mechanism exists to close, and leaving a superseded value in place for + // an extension with no rows would refuse the next run for no reason. + identity.byExtension.foreach { case (ext, value) => + if (!m.storedTokenizerId(ext).contains(value)) m.setTokenizerId(ext, value) + } + } + + /** The `--retokenize` path. Five outcomes, in this order, and the order is the + * safety property: nothing is deleted until every check has passed. + * + * 1. A changed tokenizer the flag did not name: refuse, naming it. Invalidating + * the named extensions first and then refusing would leave a map no flag + * describes — part redone under a new tokenizer, part stale, and the + * recorded identity a mixture of both. + * 2. No tokenized row carries any of the named extensions: refuse. This is the + * no-op case, and a no-op that exits 0 is the failure this flag exists to + * prevent. + * 3. A sampled `new_blob` of a RETAINED row does not resolve in dst: refuse. + * Those ids live only there, so the map is reusable only alongside it. + * 4. The memo purge. It runs BEFORE the transaction because the two failure + * directions are not symmetric: a memo entry deleted for nothing costs one + * tokenizer invocation, a memo entry kept serves the stale tokens the whole + * exercise is meant to remove. If the purge throws, nothing has been + * invalidated and the run stops. + * 5. The purge found no entries at all: refuse. With a non-empty blob_map the + * tokenized rows were produced by tokenBySha.pl, which memoizes every one + * of them, so "none of them are in this directory" means the directory is + * not the memo — and the real memo would answer the re-tokenization with + * the stale tokens. + * + * Only after all five does anything change, and then it changes in one + * transaction. */ + private def retokenizeOrRefuse(m: Mapping, identity: TokenizerIdentity, r: Retokenize): Unit = { + val uncovered = m.cachePoisoningIdentityChanges(identity).filterNot(c => r.extensions.contains(c.extension)) + if (uncovered.nonEmpty) + throw new TokenizerChangedException( + "--retokenize refused: " + tokenizerChangedMessage(uncovered) + + s"\n --retokenize named only [${r.extensions.toVector.sorted.mkString(",")}], so those " + + "entries would have been invalidated while the ones above stayed stale, and the recorded " + + "identity would have become a mixture of two tokenizers. Nothing has been changed in the " + + "blob map. Name every changed extension in one run.") + + val affected = m.countTokenizedRowsForExtensions(r.extensions) + if (affected == 0L) + throw new NothingInvalidatedException( + s"--retokenize=${r.extensions.toVector.sorted.mkString(",")} refused: this blob map holds no " + + "tokenized row on a path with any of those extensions, so the flag would have invalidated " + + "NOTHING and the run would have exited 0 having reused every cached tokenization. Check " + + "the extension spelling (lowercase, no dot, as in tokenize/CregitLanguages.pm) and that " + + "this is the right project's blob map. Nothing has been changed.") + + val sample = m.sampleTokenizedNewBlobs(r.sampleSize, excludeExtensions = r.extensions) + sample.find { case (_, newBlob) => !r.newBlobResolves(newBlob) }.foreach { + case (path, newBlob) => + throw new DanglingNewBlobException( + s"--retokenize refused: blob_map's new_blob $newBlob (for '$path') does not exist in the " + + "destination repository. That row is one this invalidation KEEPS, and those ids live " + + "only in dst, so the map is only reusable alongside it. This is a --from-step 2 resume " + + "that keeps the work directory, never a fresh run (which deletes it first). Nothing has " + + s"been changed in the blob map. Sampled ${sample.size} retained tokenized rows.") + } + + val blobs = m.tokenizedOrigBlobsForExtensions(r.extensions) + val memo = r.purgeMemo(blobs) + if (memo.examined > 0L && memo.deleted == 0L) { + if (!r.memoDirIsTokenizerMemo) + throw new NothingInvalidatedException( + s"--retokenize refused: the memo held none of the ${memo.examined} affected blob(s) " + + s"(${memo.render}), and --memo-dir is not the directory in $$BFG_MEMO_DIR, the memo " + + "tokenBySha.pl reads. The --memo-dir is almost certainly not this project's. That " + + "matters because the memo is keyed on sha1(contents) with no tokenizer in the key: " + + "dropping the blob_map rows while the real memo keeps its entries just serves the same " + + "stale tokens through the other door. Nothing has been changed in the blob map.") + r.report( + s"--retokenize: the memo held none of the ${memo.examined} affected blob(s) " + + s"(${memo.render}). Accepted, because --memo-dir is $$BFG_MEMO_DIR, the memo " + + "tokenBySha.pl reads: no stale entry is left to serve, and every affected blob will run " + + "its tokenizer again.") + } + + val res = m.applyRetokenize(r.extensions, identity, memo, r.nowEpochSeconds()) + r.report( + s"--retokenize: invalidating the entries of [${res.extensions.toVector.sorted.mkString(",")}] " + + "because their tokenizer changed.\n" + + s" tokenizer identity now: ${identity.render}\n" + + s" DROPPED ${res.blobsInvalidated} tokenized blob_map rows on those extensions, and purged " + + s"their memo entries (${memo.render}). Both layers, because the memo is keyed on " + + "sha1(contents) alone and would otherwise answer the re-tokenization with the old tokens.\n" + + s" KEPT ${res.after.blobTokenized} tokenized blob_map rows on every other extension, and " + + s"${res.after.blobIdentity} identity rows. Re-tokenizing those is 88% of the pipeline and the " + + "defect has nothing to do with them.\n" + + s" DROPPED ${res.before.tree} tree_map rows (a tree names its blobs, and a retained tree row " + + s"short-circuits the re-walk), ${res.before.commit} commit_map rows (commits name trees) and " + + s"${res.before.ref} ref_map rows (refs name commits).\n" + + s" CHECKED ${math.min(r.sampleSize.toLong, res.after.blobTokenized)} of the retained new_blob " + + "ids resolve in dst.") + } + + private def tokenizerChangedMessage(changes: Vector[IdentityChange]): String = { + val exts = changes.map(_.extension).sorted + val detail = changes.map(c => + s"\n .${c.extension}: recorded ${c.stored}, this run reports ${c.requested}").mkString + s"the tokenizer for ${exts.map("." + _).mkString(", ")} is not the one that produced the " + + s"cached tokens in this blob map.$detail" + + "\n Reusing those rows would reproduce the OLD tokenizer's output for every cached file, and " + + "nothing downstream can tell one token stream from another: that is how a 16-day-old " + + "rustTokenizer binary put a `line:col` prefix into 741,869 .rs entries across 45 " + + "projects with no error anywhere.\n" + + " Nothing has been changed. Either restore the tokenizer this map was built with, or " + + s"invalidate exactly those entries:\n --retokenize=${exts.mkString(",")}\n" + + " That drops the affected blob_map rows and their memo entries, keeps every other " + + "extension's tokenizations, and requires --memo-dir and a step-2 resume." + } + + /** The `--mask-widened` path. Four outcomes, in this order, and the order is + * the safety property: nothing is written until both checks have passed. + * + * 1. No stored mask (a fresh DB): record it. Nothing to reuse, nothing to + * check — the flag is a no-op and says so. + * 2. Stored mask equals the new one: an ordinary resume. The flag is a no-op. + * 3. A tokenized row's path is no longer selected: this is a NARROWING, not a + * widening. Refuse, naming the path, having written nothing. + * 4. A sampled `new_blob` does not resolve in dst: refuse, naming it, having + * written nothing. + * + * Only after 3 and 4 both pass does anything change. */ + private def widenOrCheckMask(m: Mapping, mask: String, w: MaskWidening): Unit = + m.getMeta("mask") match { + case None => + m.setMeta("mask", mask) + w.report("--mask-widened: no mask recorded yet (fresh blob map), so there is nothing " + + "to reuse and nothing to check. Proceeding as a normal first run.") + case Some(stored) if stored == mask => + w.report(s"--mask-widened: the recorded mask already equals this run's [$mask], so this " + + "is an ordinary resume and the flag changes nothing.") + case Some(stored) => + val counts = m.rowCounts + m.firstPathNarrowedBy(mask.r).foreach { offending => + throw new MaskNarrowedException( + s"--mask-widened refused: [$mask] does not select '$offending', which was tokenized " + + s"under the recorded mask [$stored]. That makes the change a NARROWING, not a " + + "widening, so the rows this flag would reuse are not all valid. Nothing has been " + + "changed in the blob map. Verified against the data, one regex test per tokenized " + + s"row (${counts.blobTokenized} of them), not by comparing the two regexes." + ) + } + val sample = m.sampleTokenizedNewBlobs(w.sampleSize) + sample.find { case (_, newBlob) => !w.newBlobResolves(newBlob) }.foreach { + case (path, newBlob) => + throw new DanglingNewBlobException( + s"--mask-widened refused: blob_map's new_blob $newBlob (for '$path') does not exist " + + "in the destination repository. Those ids live only in dst, so the map is only " + + "reusable alongside it. This is a --from-step 2 resume that KEEPS the work " + + "directory, never a fresh run (which deletes it first). Nothing has been changed " + + s"in the blob map. Sampled ${sample.size} of ${counts.blobTokenized} tokenized rows." + ) + } + val r = m.applyMaskWidening(stored, mask, w.nowEpochSeconds()) + w.report( + s"--mask-widened: reusing the blob map across a mask widening.\n" + + s" old mask: $stored\n" + + s" new mask: $mask\n" + + s" KEPT ${r.after.blobTokenized} tokenized blob_map rows (same blob, same path, so " + + "same content and same extension, so the same tokens)\n" + + s" DROPPED ${r.before.blobIdentity} identity blob_map rows (orig == new). Under the old " + + "mask these recorded 'not selected, bytes pass through'. Some of those paths ARE " + + "selected now, and keeping them would serve raw source as a cache hit.\n" + + s" DROPPED ${r.before.tree} tree_map rows (trees gain entries) and " + + s"${r.before.commit} commit_map rows (commits name trees), and " + + s"${r.before.ref} ref_map rows (refs name commits).\n" + + s" CHECKED every one of the ${r.after.blobTokenized} retained paths still matches the new " + + s"mask, and ${math.min(w.sampleSize.toLong, r.after.blobTokenized)} of their new_blob ids " + + "resolve in dst." + ) + } + /** Open a frozen prior-run DB read-only (query_only + busy_timeout) as a lookup * fallback; never written by us, so concurrent shard readers are safe. Rejects a * warm DB whose command/mask differ from this run -- its ids would be foreign. */ diff --git a/blobExec/src/main/scala/cregit/blobexec/Saturating.scala b/blobExec/src/main/scala/cregit/blobexec/Saturating.scala deleted file mode 100644 index 97fdd918e..000000000 --- a/blobExec/src/main/scala/cregit/blobexec/Saturating.scala +++ /dev/null @@ -1,11 +0,0 @@ -package cregit.blobexec - -private[blobexec] object Saturating { - - def sum(a: Int, b: Int): Int = clampToInt(a.toLong + b.toLong) - - def product(a: Int, b: Int): Int = clampToInt(a.toLong * b.toLong) - - private def clampToInt(value: Long): Int = - math.min(math.max(value, 0L), Int.MaxValue.toLong).toInt -} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala new file mode 100644 index 000000000..0aa8bbbd9 --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala @@ -0,0 +1,98 @@ +package cregit.blobexec + +/** Which tokenizer produced the tokens for each file extension. + * + * The problem this type exists for. `blob_map` reuse was decided on `command` + * and `mask` alone (see `Mapping.open`), and `command` is the fixed path + * `tokenizeByBlobId/tokenBySha.pl` — a constant for the life of the pipeline. + * So when the Rust tokenizer's output format was corrected in 729643e, nothing + * about the recorded metadata changed, every cached `.rs` row stayed a cache + * hit, and a run with the corrected tokenizer reproduced the old, wrong tokens + * exactly. The mask cannot stand in for this: the mask decides WHICH files are + * tokenized and never HOW. + * + * The identity is per EXTENSION rather than per language, because the extension + * is what the cache rows can be selected by: `blob_map` stores paths, not + * languages. `CregitLanguages.pm` maps extension to language and language to + * parser, so the pipeline collapses that chain into one value per extension + * before handing it over (see `tokenize/tokenizerIdentity.pl`). + * + * The value is opaque here on purpose. This code never needs to know whether it + * is a hash of a binary, a version string the tokenizer reports, or a digest of + * a whole toolchain — only whether it is the same string as last time. */ +final case class TokenizerIdentity(byExtension: Map[String, String]) { + + def isEmpty: Boolean = byExtension.isEmpty + def nonEmpty: Boolean = byExtension.nonEmpty + def extensions: Set[String] = byExtension.keySet + def get(extension: String): Option[String] = byExtension.get(extension) + + /** Canonical, sorted rendering: the same map always prints the same string, so + * it can go in a log or a refusal message and be compared by eye. */ + def render: String = + byExtension.toVector.sorted.map { case (e, id) => s"$e=$id" }.mkString(",") +} + +object TokenizerIdentity { + + val empty: TokenizerIdentity = TokenizerIdentity(Map.empty) + + /** Extensions as `CregitLanguages.pm` spells them: lowercase, no leading dot. + * `c++` and `h++` are real entries in that table, hence the `+`. The pattern + * is also a safety property: these strings are interpolated into a SQL LIKE + * pattern in [[Mapping]], and this alphabet contains no quote, no `%` and no + * `_`, so there is nothing to escape and nothing to inject. */ + private[blobexec] val ExtensionPattern = "[a-z0-9+]+" + + /** The identity value: hex, and long enough to be a digest rather than a + * guess. Bounded above so a malformed argument cannot become a meta row of + * arbitrary size. */ + private[blobexec] val ValuePattern = "[0-9a-f]{8,128}" + + private val EntryRe = s"($ExtensionPattern)=($ValuePattern)".r + + /** Parse `ext=value[,ext=value]...`. + * + * Left on anything malformed, naming the offending entry. Refusing a + * malformed identity matters more than it looks: a value this code does not + * understand would be recorded verbatim, compare unequal to the next run's, + * and turn every subsequent run into a refusal nobody can explain. */ + def parse(spec: String): Either[String, TokenizerIdentity] = { + val trimmed = spec.trim + if (trimmed.isEmpty) + return Left("a tokenizer identity must name at least one extension, as ext=value") + val entries = trimmed.split(",", -1).toVector + val parsed = entries.map { + case EntryRe(ext, value) => Right((ext, value)) + case other => + Left( + s"cannot read [$other] as a tokenizer identity entry. Wanted " + + s"=, extension matching $ExtensionPattern (lowercase, no " + + s"leading dot, as in CregitLanguages.pm) and value matching $ValuePattern." + ) + } + parsed.collectFirst { case Left(why) => Left(why) }.getOrElse { + val pairs = parsed.collect { case Right(p) => p } + val duplicated = pairs.groupBy(_._1).collect { case (e, vs) if vs.size > 1 => e } + if (duplicated.nonEmpty) + Left(s"extension(s) named more than once in the tokenizer identity: " + + duplicated.toVector.sorted.mkString(", ")) + else Right(TokenizerIdentity(pairs.toMap)) + } + } + + /** Parse a `--retokenize=` extension list. Same alphabet as above, so the two + * cannot disagree about what an extension looks like. */ + def parseExtensions(spec: String): Either[String, Set[String]] = { + val trimmed = spec.trim + if (trimmed.isEmpty) + return Left("--retokenize must name at least one extension, as in --retokenize=rs") + val parts = trimmed.split(",", -1).toVector.map(_.trim) + val bad = parts.filterNot(_.matches(ExtensionPattern)) + if (bad.nonEmpty) + Left(s"not an extension: ${bad.map(b => s"[$b]").mkString(", ")}. Wanted lowercase " + + s"names without a leading dot, matching $ExtensionPattern (as in CregitLanguages.pm): " + + "--retokenize=rs, --retokenize=c,h") + else Right(parts.toSet) + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala new file mode 100644 index 000000000..f1f60d78a --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala @@ -0,0 +1,91 @@ +package cregit.blobexec + +import java.nio.file.{Files, Path} +import java.security.MessageDigest + +/** The content-addressed memo `tokenizeByBlobId/tokenBySha.pl` keeps, described + * from this side so it can be invalidated. + * + * That script is the `` blobExec runs per blob. Before invoking a + * tokenizer it hashes the bytes on its stdin and looks for + * `$BFG_MEMO_DIR/xx/yy/`; on a hit it prints that file and no tokenizer + * runs at all. The key is `sha1_hex($contents)` and NOTHING else — + * tokenBySha.pl:76,83-84. No tokenizer version, no extension, no language. + * + * So dropping a `blob_map` row is not enough to force a re-tokenization. The + * walker re-runs the command for that blob, the command finds the memo entry, + * and the same stale tokens come back. Both layers have to go together, which + * is why `--retokenize` will not run without `--memo-dir`. + * + * Deleting a memo entry is always safe in the direction that matters: a miss + * costs one tokenizer invocation and produces correct tokens, while a kept + * entry produces wrong ones. That asymmetry is why the purge runs BEFORE the + * database transaction — if the purge fails, nothing has been invalidated and + * the run refuses; if the transaction fails afterwards, some cache entries were + * thrown away and nothing is wrong. */ +object TokenizerMemo { + + /** What the purge did. `missing` is not an error on its own: a blob excluded as + * oversized or denylisted never reached the tokenizer, so it never had an + * entry. `deleted == 0` with `examined > 0` IS, and [[Mapping]] refuses on it + * — see there. */ + final case class PurgeReport(examined: Long, deleted: Long, missing: Long, unreadable: Long) { + def render: String = + s"examined=$examined deleted=$deleted missing=$missing unreadable=$unreadable" + } + + object PurgeReport { + val empty: PurgeReport = PurgeReport(0L, 0L, 0L, 0L) + } + + /** `sha1_hex` of the bytes, matching Perl's `Digest::SHA::sha1_hex`. */ + def sha1Hex(bytes: Array[Byte]): String = { + val md = MessageDigest.getInstance("SHA-1") + val digest = md.digest(bytes) + val sb = new StringBuilder(digest.length * 2) + digest.foreach(b => sb.append("%02x".format(b))) + sb.toString + } + + /** The two-level fan-out tokenBySha.pl builds: `substr(0,2)/substr(2,2)/sha1` + * (tokenBySha.pl:83-84). Replicated rather than abstracted over, because the + * two have to agree byte for byte and a shared helper across a Perl/Scala + * boundary is not available. `TokenizerMemoSpec` pins it against a literal + * sha1 and, when perl is on PATH, against Digest::SHA itself. */ + def relativePathFor(sha1Hex: String): String = { + require(sha1Hex.length >= 4, s"not a sha1: [$sha1Hex]") + s"${sha1Hex.substring(0, 2)}/${sha1Hex.substring(2, 4)}/$sha1Hex" + } + + def entryFor(root: Path, sha1Hex: String): Path = + root.resolve(relativePathFor(sha1Hex)) + + def isTokenizerMemo(memoDir: Path, bfgMemoDir: Option[String]): Boolean = + bfgMemoDir.filter(_.nonEmpty).exists { env => + try Files.isSameFile(memoDir, java.nio.file.Paths.get(env)) + catch { case _: java.io.IOException | _: java.nio.file.InvalidPathException => false } + } + + /** Delete the memo entries of `origBlobs`. + * + * `readBytes` turns an original blob sha into its bytes, or None when the + * object cannot be materialised (oversized). Injected so this is testable + * without a repository, and so the caller decides how the bytes are read. + * + * One blob at a time and nothing retained: the corpus figure for a `.rs` + * invalidation is 741,869 rows, and holding their contents is not an option. */ + def purge(root: Path, origBlobs: Vector[String], readBytes: String => Option[Array[Byte]]): PurgeReport = { + var deleted = 0L + var missing = 0L + var unreadable = 0L + origBlobs.foreach { sha => + readBytes(sha) match { + case None => unreadable += 1L + case Some(bytes) => + val entry = entryFor(root, sha1Hex(bytes)) + if (Files.deleteIfExists(entry)) deleted += 1L else missing += 1L + } + } + PurgeReport(origBlobs.size.toLong, deleted, missing, unreadable) + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala new file mode 100644 index 000000000..47e178c86 --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala @@ -0,0 +1,142 @@ +package cregit.blobexec + +import java.io.{BufferedOutputStream, DataInputStream, EOFException, IOException} +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} +import java.util.concurrent.{ArrayBlockingQueue, ConcurrentHashMap, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters._ + +/** Persistent tokenizer processes that speak tokenizeByBlobId/WORKER_PROTOCOL.md, + * one request in flight per worker. */ +final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoCloseable { + import TokenizerWorkerPool._ + + private final class Worker(val index: Int, val process: Process) { + val stdin = new BufferedOutputStream(process.getOutputStream) + val stdout = new DataInputStream(process.getInputStream) + } + + private val idle = new ArrayBlockingQueue[Worker](size) + private val workers = ConcurrentHashMap.newKeySet[Worker]() + private val closed = new AtomicBoolean(false) + + try (0 until size).foreach(addWorker) + catch { case failure: Throwable => close(); throw failure } + + /** Same result as [[BlobExec.invoke]]: (exit, stdout, stderr), and + * [[BlobExec.TimeoutExitCode]] for a blob that ran out of time. */ + def invoke( + bytes: Array[Byte], + filename: String, + timeoutSeconds: Int + ): (Int, Array[Byte], String) = { + val worker = idle.take() + val backstop = timeoutSeconds + BackstopSlackSeconds + within(backstop)(request(worker, bytes, filename, timeoutSeconds)) match { + case None => + System.err.println( + s"blobExec: tokenizer worker ${worker.index} gave no response within ${backstop}s; replacing it") + replace(worker) + Timeout + case Some(Left(failure)) => + replace(worker) + (1, Array.emptyByteArray, s"tokenizer worker ${worker.index} failed: $failure\n") + case Some(Right(response)) => + if (worker.process.isAlive) idle.put(worker) else replace(worker) + if (response._1 == WorkerTimeoutExitCode) Timeout else response + } + } + + override def close(): Unit = + if (closed.compareAndSet(false, true)) { + val all = workers.asScala.toVector + all.foreach(w => try w.stdin.close() catch { case _: IOException => () }) + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(CloseSeconds) + all.foreach { w => + if (!w.process.waitFor(math.max(0L, deadline - System.nanoTime()), TimeUnit.NANOSECONDS)) kill(w.process) + } + } + + private def addWorker(index: Int): Unit = { + val process = new ProcessBuilder(workerPath).redirectError(ProcessBuilder.Redirect.INHERIT).start() + val worker = new Worker(index, process) + within(ReadySeconds)(readLine(worker.stdout)) match { + case Some(Right("READY")) => + workers.add(worker) + idle.put(worker) + case notReady => + kill(process) + throw new IllegalStateException(s"tokenizer worker [$workerPath] did not start: $notReady") + } + } + + private def replace(worker: Worker): Unit = { + workers.remove(worker) + kill(worker.process) + if (!closed.get()) addWorker(worker.index) + } + + private def request(worker: Worker, bytes: Array[Byte], filename: String, timeoutSeconds: Int) = { + val name = filename.getBytes(UTF_8) + worker.stdin.write(s"REQ ${bytes.length} ${name.length} $timeoutSeconds\n".getBytes(UTF_8)) + worker.stdin.write(name) + worker.stdin.write(bytes) + worker.stdin.flush() + readLine(worker.stdout).split(' ') match { + case Array("RES", exit, outLength, errLength) => + val out = readBytes(worker.stdout, outLength.toInt) + (exit.toInt, out, new String(readBytes(worker.stdout, errLength.toInt), UTF_8)) + case header => throw new IOException(s"malformed response header [${header.mkString(" ")}]") + } + } +} + +object TokenizerWorkerPool { + /** What a worker answers when it killed the tokenizer for running out of time. */ + val WorkerTimeoutExitCode = 124 + private val Timeout = (BlobExec.TimeoutExitCode, Array.emptyByteArray, "") + /** The worker kills its own tokenizer at the budget. This only catches a wedged worker. */ + private val BackstopSlackSeconds = 5L + private val ReadySeconds = 30L + private val CloseSeconds = 2L + private val MaxHeaderBytes = 8192 + + /** Kill the children first: once the worker is gone they are no longer its descendants. */ + private def kill(process: Process): Unit = { + process.descendants().forEach(child => { child.destroyForcibly(); () }) + process.destroyForcibly() + () + } + + /** Runs `read` on a daemon thread, so that a worker which never answers cannot block the caller. */ + private def within[A](seconds: Long)(read: => A): Option[Either[Throwable, A]] = { + val result = new AtomicReference[Either[Throwable, A]]() + val done = new CountDownLatch(1) + val reader = new Thread(() => { + try result.set(Right(read)) + catch { case failure: Throwable => result.set(Left(failure)) } + finally done.countDown() + }, "tokenizer-worker-read") + reader.setDaemon(true) + reader.start() + if (done.await(seconds, TimeUnit.SECONDS)) Some(result.get()) else None + } + + private def readLine(in: DataInputStream): String = { + val line = new java.io.ByteArrayOutputStream + var next = in.read() + while (next != '\n') { + if (next == -1) throw new EOFException("tokenizer worker closed its stdout") + if (line.size() >= MaxHeaderBytes) throw new IOException(s"header longer than $MaxHeaderBytes bytes") + line.write(next) + next = in.read() + } + line.toString(UTF_8) + } + + private def readBytes(in: DataInputStream, length: Int): Array[Byte] = { + val bytes = new Array[Byte](length) + in.readFully(bytes) + bytes + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index a9c9f4741..502a6579e 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -20,15 +20,18 @@ final case class WalkStats( blobsCacheHit: Int, refsProjected: Int, aborted: Boolean, - /** Blobs whose command was killed for exceeding its budget. Each one is a - * file left holding raw source instead of tokens, so a non-zero count must - * reach the caller rather than living only in stderr. */ + /** Blobs whose command was killed for exceeding its budget, and so + * excluded from the rewrite as failed. */ blobsTimedOut: Long, - /** Mask-matched blobs JGit would not materialise. Reported, not gating. */ + /** Distinct mask-matched blobs excluded because JGit will not materialise an + * object that large: see [[Walker.MaxBlobBytes]]. */ blobsOversized: Long, /** Distinct mask-matched blobs excluded because they are on the shipped blob - * denylist. Reported, not gating. */ + * denylist ([[BlobDenylist]]). */ blobsDenylisted: Long, + /** Blobs whose tokenizer reported [[BlobExec.ParserCrashExitCode]] + * (srcML died on a signal, or the token stream came back empty), and so + * excluded from the rewrite as failed. */ blobsParserCrashed: Long = 0L, blobCommandExecutions: Long, originalBlobCopyRequests: Long, @@ -57,6 +60,10 @@ final class Walker( destinationMayContainObjects: Boolean = true, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, + workerPool: Option[TokenizerWorkerPool] = None, + // The shipped list by default, so a caller cannot forget it and hand a known + // non-terminating blob to srcml. A parameter only so a test can supply its + // own fixture; nothing at run time chooses a different list. denylist: BlobDenylist = BlobDenylist.shipped ) { import Walker._ @@ -81,9 +88,16 @@ final class Walker( // -- stall watchdog ------------------------------------------------------ // - // `Await.result` is unbounded on purpose: a duration budget cannot tell a wedged - // run from an honestly large one. The watchdog asserts completed work instead, - // which is safe because ChildRunner bounds every child. + // The consumer's `Await.result` calls are unbounded again, and deliberately. + // A duration budget cannot tell a wedged run from an honestly large one: the + // predecessor of this watchdog computed 3,806,400s for tencentkona-21's + // first-import commit and clamped to 30 days, so on the very run that + // motivated this work it would never have fired. Instead of predicting how + // long honest work takes, assert that work is *happening*: every blob, tree, + // commit and original-blob copy stamps `lastProgressNanos`, and if nothing at + // all completes within `stallTimeoutSeconds` the watchdog kills the process. + // Safe because GNU `timeout` hard-bounds every child at `blobTimeoutSeconds`, + // so a healthy run always completes *something* well inside the window. private val lastProgressNanos = new AtomicLong(System.nanoTime()) private val lastProgressWhat = new AtomicReference[String]("startup") @@ -134,21 +148,23 @@ final class Walker( } /** Run `body` under the stall watchdog. The watchdog is a daemon so it can - * never keep the JVM alive, and it halts rather than exits: a shutdown hook - * could block on the same wedged thread. */ + * never keep the JVM alive, and it halts rather than exiting: a stalled run + * may well have a shutdown hook that would block on the same wedged thread, + * and an exit path that can hang is not a fix for a hang. */ private def withStallWatchdog[A](body: => A): A = { val done = new AtomicBoolean(false) val tick = math.max(1L, math.min(30L, math.max(1, stallTimeoutSeconds).toLong / 4L)) val watchdog = new Thread( () => { - var interrupted = false - while (!done.get() && !interrupted) { - // Exit on interrupt: a set flag makes every later sleep throw at once. + while (!done.get()) { try Thread.sleep(tick * 1000L) - catch { case _: InterruptedException => interrupted = true } - if (!done.get() && !interrupted) stalled().foreach { stalledNanos => + catch { case _: InterruptedException => Thread.currentThread().interrupt() } + if (!done.get()) stalled().foreach { stalledNanos => System.err.println(watchdogReport(stalledNanos)) - // Halting reaps nothing, so kill the descendant tree first. + // Halting leaves nothing behind to reap the children, and an + // orphaned srcml tree is what survived its parent by 51 hours in + // the incident this work comes from. Kill the whole descendant + // tree first — `timeout`, the shell, the tokenizer, all of it. val killed = killDescendants() System.err.println(s"blobExec: killed $killed leftover child process(es) before exiting") System.err.flush() @@ -295,14 +311,13 @@ final class Walker( val procWalk = new RevWalk(src) var aborted = false - var timedOut = false var commits = 0 var blobsRun = 0 var blobsHit = 0 try { val it = sliceIds.iterator - while (it.hasNext && !aborted && !timedOut) { + while (it.hasNext && !aborted) { val rc = procWalk.parseCommit(it.next()) val origTreeId = rc.getTree.getId @@ -314,9 +329,6 @@ final class Walker( if (misses.abort) { aborted = true - } else if (misses.timedOutKeys.nonEmpty) { - timedOut = true - persistRetryableBlobs(misses, rc.getId.name) } else { val subtreeMap = scala.collection.mutable.Map.empty[String, String] val newTreeId = assemble(plan, resolved, treeInserter, subtreeMap) @@ -352,33 +364,6 @@ final class Walker( (commits, blobsRun, blobsHit, aborted) } - /** Persist only what a commit containing a timed-out blob may leave behind. - * - * Three writes would each hide the retry: the blob's `blob_map` row, any - * `tree_map` row above it (a tree hit short-circuits the whole subtree), and - * the commit's `commit_map` row. All three are suppressed. Successful - * tokenizations are kept, because they are content-addressed and already in - * dst; identity rows are not, because their bytes are copied during the tree - * assembly this path skips. - */ - private def persistRetryableBlobs(misses: MissResolution, origCommitSha: String): Unit = { - val keep = misses.persistable - dbLock.synchronized { - mapping.inTx { - keep.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } - } - } - System.err.println( - s"blobExec: commit $origCommitSha contains ${misses.timedOutKeys.size} timed-out blob(s): " + - s"kept ${keep.size} good blob row(s), recorded no tree and no commit for it. " + - "Another run over this memo retries just those blobs (from the pipeline: resume at " + - "step 2, never step 1); nothing needs clearing by hand." - ) - misses.timedOutKeys.toVector.sorted.foreach { case (sha, path) => - System.err.println(s"blobExec: will retry on the next run: $sha ($path)") - } - } - // -- ref preparation ----------------------------------------------------- /** For every commit sha already in `commit_map` that still exists in src, @@ -386,7 +371,8 @@ final class Walker( private def markUninteresting(revWalk: RevWalk): Unit = { mapping.allCommitOrigShas.foreach { sha => val id = ObjectId.fromString(sha) - // The frontier scan is slow but healthy; stamp it. + // Scanning a large memo's frontier happens before any blob runs, and on a + // long history it is not fast; it must not look like a stall. progress(s"frontier $sha") try { val rc = revWalk.parseCommit(id) @@ -429,14 +415,13 @@ final class Walker( val commitInserter = dst.newObjectInserter() var aborted = false // local mutation across the walk loop is unavoidable - var timedOut = false // a blob in this commit was killed on its budget var commits = 0 var blobsRun = 0 var blobsHit = 0 try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted && !timedOut) { + while (iter.hasNext && !aborted) { val rc = iter.next() val origCommitSha = rc.getId.name @@ -449,11 +434,6 @@ final class Walker( if (misses.abort) { aborted = true - } else if (misses.timedOutKeys.nonEmpty) { - // Keep the successful tokenizations; persist no tree and no commit, and - // stop, so no descendant is folded onto a parent we did not record. - timedOut = true - persistRetryableBlobs(misses, origCommitSha) } else { // Assemble the new tree id (bottom-up). `subtreeMap` collects every // freshly built (sub)tree's orig->new id so the transaction below @@ -529,8 +509,6 @@ final class Walker( val inFlight = new ConcurrentHashMap[(String, String), Future[BlobResult]]() val aborted = new AtomicBoolean(false) - // Stops the producer and drains the consumer, without aborting the run. - val timedOutStop = new AtomicBoolean(false) val producerError = new AtomicReference[Throwable](null) // -- producer: iterate the walk in order, plan trees, submit blob work -- @@ -538,7 +516,7 @@ final class Walker( def run(): Unit = { try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted.get() && !timedOutStop.get()) { + while (iter.hasNext && !aborted.get()) { val rc = iter.next() val data = snapshotCommit(rc) // decouple consumer from the shared RevWalk val (plan, artifacts) = buildTreePlan(rc.getTree.getId, pathPrefix = "") @@ -578,31 +556,16 @@ final class Walker( queue.take() match { case EndOfWalk => stopped = true case CommitItem(data, plan, artifacts, missFutures) => - if (aborted.get() || timedOutStop.get()) { - // Draining after abort / producer-error / a timed-out commit: - // discard until EndOfWalk. Folding a later commit here would - // reference a parent this run deliberately did not record. + if (aborted.get()) { + // Draining after abort / producer-error: discard until EndOfWalk. } else { // Await only this commit's futures (typically already complete). // Unbounded on purpose: the stall watchdog, not a guessed // duration, is what stops a wedged run (see withStallWatchdog). val results = missFutures.map { case (k, f) => k -> Await.result(f, Duration.Inf) } - val timedOutIds: IMap[(String, String), ObjectId] = - results.iterator.collect { case (k, BlobResult.Unusable(id)) => k -> id }.toMap results.values.collectFirst { case a: BlobResult.Aborted => a } match { case Some(_) => aborted.set(true) // stop the producer; drain the remainder - case None if timedOutIds.nonEmpty => - // Retryable: keep the good blob rows, write no tree and no - // commit, and stop the walk so nothing is folded onto a parent - // this run did not record. See persistRetryableBlobs. - val good: IMap[(String, String), ObjectId] = - results.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap - persistRetryableBlobs( - MissResolution(good ++ timedOutIds, abort = false, timedOutKeys = timedOutIds.keySet), - data.origCommitSha - ) - timedOutStop.set(true) case None => val resolved: IMap[(String, String), ObjectId] = results.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap @@ -679,7 +642,6 @@ final class Walker( val inFlight = new ConcurrentHashMap[(String, String), Future[BlobResult]]() val aborted = new AtomicBoolean(false) - val timedOutStop = new AtomicBoolean(false) // see walkCommitsPipelined val producerError = new AtomicReference[Throwable](null) // -- producer: plan trees, submit blob work, chain tree assembly -------- @@ -687,7 +649,7 @@ final class Walker( def run(): Unit = { try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted.get() && !timedOutStop.get()) { + while (iter.hasNext && !aborted.get()) { val rc = iter.next() val data = snapshotCommit(rc) val (plan, artifacts) = buildTreePlan(rc.getTree.getId, pathPrefix = "") @@ -733,25 +695,15 @@ final class Walker( queue.take() match { case EndOfTreeWalk => stopped = true case CommitTreeItem(data, artifacts, blobKeys, treeFuture) => - if (aborted.get() || timedOutStop.get()) { - // Draining after abort / producer-error / a timed-out commit: - // discard until EndOfTreeWalk. + if (aborted.get()) { + // Draining after abort / producer-error: discard until EndOfTreeWalk. } else { // The assembly (and any abort it observed) is already done off-thread. // Unbounded: the stall watchdog is the backstop, not a budget. Await.result(treeFuture, Duration.Inf) match { case TreeResult.Aborted(_, _) => aborted.set(true) // stop the producer; drain the remainder - case TreeResult.Built(_, resolved, _, tok) if tok.nonEmpty => - // Retryable: the tree was assembled off-thread and is usable, - // but recording it (or the commit) would hide the timeout from - // the next run. Keep the good blob rows only, and stop here. - persistRetryableBlobs( - MissResolution(resolved, abort = false, timedOutKeys = tok), - data.origCommitSha - ) - timedOutStop.set(true) - case TreeResult.Built(newTreeId, resolved, subtrees, _) => + case TreeResult.Built(newTreeId, resolved, subtrees) => val parents = data.parentOrigShas.map { pn => dbLock.synchronized(mapping.getCommit(pn)).getOrElse( throw new IllegalStateException( @@ -809,18 +761,14 @@ final class Walker( blobResults.iterator.map(_._2).collectFirst { case a: BlobResult.Aborted => a } match { case Some(a) => TreeResult.Aborted(a.stderr, a.exitCode) case None => - // `resolved` is what the consumer persists, so a timed-out blob is kept - // out of it; `forTree` is what the tree references, so it is kept in. val resolved: IMap[(String, String), ObjectId] = blobResults.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap - val timedOut: IMap[(String, String), ObjectId] = - blobResults.iterator.collect { case (k, BlobResult.Unusable(id)) => k -> id }.toMap val inserter = dst.newObjectInserter() try { val subtreeMap = scala.collection.mutable.Map.empty[String, String] - val newTreeId = assemble(plan, resolved ++ timedOut, inserter, subtreeMap) + val newTreeId = assemble(plan, resolved, inserter, subtreeMap) inserter.flush() - TreeResult.Built(newTreeId, resolved, subtreeMap.toMap, timedOut.keySet) + TreeResult.Built(newTreeId, resolved, subtreeMap.toMap) } finally inserter.close() } } @@ -831,7 +779,11 @@ final class Walker( * into dst). Never touches `mapping`, keeping the hot path lock-free. */ private def executeBlobTask(task: BlobMissTask): BlobResult = readBlob(task) match { - case None => BlobResult.Oversized(task.origId) + // None means "excluded from the rewrite": too large for jgit to + // materialise, or on the blob denylist. Both take the same downstream path + // — no id, no tree entry, no blob_map row — which is why one result covers + // them. readBlob has already counted and explained whichever it was. + case None => BlobResult.Excluded(task.origId) case Some(bytes) => executeBlobTask(task, bytes) } @@ -841,7 +793,7 @@ final class Walker( inFlightBlobs.put(label, System.nanoTime()) try { blobCommandExecutions.increment() - val unusable = new AtomicBoolean(false) + val failure = new AtomicReference[String]() val outcome = BlobExec.run( bytes = bytes, origSha = task.origId.name, @@ -851,14 +803,14 @@ final class Walker( abortOnError = abortOnError, inserter = workerInserter, timeoutSeconds = blobTimeoutSeconds, - onTimeout = () => { blobsTimedOut.increment(); unusable.set(true) }, - onParserCrash = () => { blobsParserCrashed.increment(); unusable.set(true) } + onTimeout = () => { blobsTimedOut.increment(); failure.set("timeout") }, + onParserCrash = () => { blobsParserCrashed.increment(); failure.set("parser-crash") }, + workerPool = workerPool ) val res = outcome match { - case BlobExec.Outcome.Skip if unusable.get() => - // The tree must still reference something, so keep the original bytes - ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) - BlobResult.Unusable(task.origId) + case BlobExec.Outcome.Skip if failure.get() != null => + noteFailed(task, failure.get()) + BlobResult.Excluded(task.origId) case BlobExec.Outcome.Skip => ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) BlobResult.Resolved(task.origId) @@ -972,28 +924,28 @@ final class Walker( // -- parallel blob resolution ------------------------------------------- /** Run each miss through the external command on the pool. Returns the ids the - * tree should reference, an aborted flag, and the keys whose command was - * killed on its budget — those are in the id map (the tree needs them) but - * must be kept out of `blob_map`, so the caller can retry them. */ + * tree should reference and an aborted flag. Excluded blobs have no id. */ private def resolveMisses( misses: Vector[BlobMissTask], pool: java.util.concurrent.ExecutorService )(implicit ec: ExecutionContext): MissResolution = { - if (misses.isEmpty) return MissResolution(IMap.empty, abort = false, timedOutKeys = Set.empty) + if (misses.isEmpty) return MissResolution(IMap.empty, abort = false) // De-dup so we don't run the command twice for the same key within a // single commit (e.g. the same (blob, path) reached via two subtrees). val unique = misses.map(m => (m.origId.name, m.fullPath) -> m).toMap.values.toVector + // An oversized blob yields None: it contributes no id, so no tree entry, no + // blob_map row and no dataset row. See readBlob and resolveEntry. val futures = unique.map { task => Future { - readBlob(task).map { bytes => + readBlob(task).flatMap { bytes => val workerInserter = dst.newObjectInserter() val label = s"${task.origId.name} (${task.fullPath})" inFlightBlobs.put(label, System.nanoTime()) try { blobCommandExecutions.increment() - val unusable = new AtomicBoolean(false) + val failure = new AtomicReference[String]() val outcome = BlobExec.run( bytes = bytes, origSha = task.origId.name, @@ -1003,17 +955,24 @@ final class Walker( abortOnError = abortOnError, inserter = workerInserter, timeoutSeconds = blobTimeoutSeconds, - onTimeout = () => { blobsTimedOut.increment(); unusable.set(true) }, - onParserCrash = () => { blobsParserCrashed.increment(); unusable.set(true) } + onTimeout = () => { blobsTimedOut.increment(); failure.set("timeout") }, + onParserCrash = () => { blobsParserCrashed.increment(); failure.set("parser-crash") } ) - // A Skip keeps the original id, so those bytes must exist in dst. + val failed = Option(failure.get()) + failed.foreach(noteFailed(task, _)) + // For Skip outcomes (identical output, or a non-zero exit with + // abortOnError=false) we keep the original blob id, so the dst tree will + // reference it — meaning the bytes must exist in dst. For Replace + // outcomes the worker has already inserted the new blob. For Abort we do + // nothing (caller short-circuits). A failed blob is excluded. outcome match { - case BlobExec.Outcome.Skip => ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) - case _ => () + case BlobExec.Outcome.Skip if failed.isEmpty => + ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) + case _ => () } workerInserter.flush() progress(s"blob $label") - (task, outcome, unusable.get()) + if (failed.isDefined) None else Some((task, outcome)) } finally { inFlightBlobs.remove(label) workerInserter.close() @@ -1022,82 +981,160 @@ final class Walker( } } - // Unbounded: the stall watchdog is the backstop, not a budget. + // Unbounded: the stall watchdog is the backstop, not a budget. `flatten` + // drops the excluded blobs: oversized, denylisted or failed. val results = Await.result(Future.sequence(futures), Duration.Inf).flatten - val abort = results.exists { case (_, o, _) => o.isInstanceOf[BlobExec.Outcome.Abort] } - if (abort) MissResolution(IMap.empty, abort = true, timedOutKeys = Set.empty) + val abort = results.exists { case (_, o) => o.isInstanceOf[BlobExec.Outcome.Abort] } + if (abort) MissResolution(IMap.empty, abort = true) else { val ids = results.iterator.collect { - case (task, BlobExec.Outcome.Replace(newId), _) => + case (task, BlobExec.Outcome.Replace(newId)) => (task.origId.name, task.fullPath) -> newId - case (task, BlobExec.Outcome.Skip, _) => - // identical, non-zero-exit (with abortOnError=false), or timed out + case (task, BlobExec.Outcome.Skip) => + // identical, or non-zero-exit (with abortOnError=false) (task.origId.name, task.fullPath) -> task.origId }.toMap - val timedOutKeys = results.iterator.collect { - case (task, _, true) => (task.origId.name, task.fullPath) - }.toSet - MissResolution(ids, abort = false, timedOutKeys = timedOutKeys) + MissResolution(ids, abort = false) } } + /** Read one mask-matched blob, or `None` when it is too large for JGit to + * materialise and must therefore be excluded from the rewrite. + * + * `qualcomm/qcom-embedded-power-measurement` ended with rc 1 here: + * + * org.eclipse.jgit.errors.LargeObjectException: 205b0f65... exceeds size limit + * at cregit.blobexec.Walker.readBlob(Walker.scala:1047) + * + * on `src/libraries/libexcel/excel.cpp`, 102,897,757 bytes and 2,398,232 + * lines. The mask *did* select it (`.cpp`), so unlike the pass-through path + * this is not a binary that slipped through. It is machine-generated: its own + * header says it is a `dumpcpp` dump of Microsoft Excel's COM type library. + * It carries no contributor-behaviour signal, and the tokenizer cannot be run + * on it in any case, so it is excluded — and the size alone is not the reason + * worth writing down, the provenance is. + * + * Two layers, because one is not enough: + * - the size check is a fast path, at the threshold JGit itself uses; + * - the `LargeObjectException` catch is the real mechanism, and covers any + * blob that sits between [[Walker.MaxBlobBytes]] and a JGit configured + * lower than its own default. A bare size check with a hand-picked + * constant above JGit's threshold would still throw in that band. + * + * Streaming the bytes instead (as the pass-through path does) is not an + * option here: the tokenizer would then be handed 98 MB of generated C++ on + * a box with about 5 GiB to spare. + */ private def readBlob(task: BlobMissTask): Option[Array[Byte]] = { + // Before anything is read or opened: a denylisted blob is one srcML cannot be + // trusted with, so the cheapest possible check is the right one. This is a map + // lookup on a small map, and it is what turns a 600 s timeout into a + // microsecond and a logged exclusion. val denied = denylist.entryFor(task.origId.name) if (denied.isDefined) { noteDenylisted(task, denied.get); return None } + if (failedKeys.contains((task.origId.name, task.fullPath))) return None val r = src.newObjectReader() try { val loader = r.open(task.origId, OBJ_BLOB) - try Some(loader.getBytes) - catch { - case _: org.eclipse.jgit.errors.LargeObjectException => - noteOversized(task, loader.getSize) - None - } + val size = loader.getSize + if (Walker.isOversized(size)) { noteOversized(task, size); None } + else + try Some(loader.getBytes) + catch { + case _: org.eclipse.jgit.errors.LargeObjectException => + noteOversized(task, size) + None + } } finally r.close() } - /** `(origSha, fullPath)` of blobs excluded as oversized; [[resolveEntry]] drops - * their tree entries. */ + /** `(origSha, fullPath)` of every blob excluded as oversized. Read by + * [[resolveEntry]], which drops those entries from the rewritten tree: the + * key's absence from `resolved` is what omits the file, and this set is what + * distinguishes a deliberate omission from a missing-key bug. */ private val oversizedKeys = ConcurrentHashMap.newKeySet[(String, String)]() - /** `(origSha, fullPath)` of blobs excluded by the denylist; kept apart from - * [[oversizedKeys]] so the two counts can never be confused. */ + /** `(origSha, fullPath)` of every blob excluded by the denylist. Read by + * [[resolveEntry]] for the same reason as [[oversizedKeys]]: the key's absence + * from `resolved` is what omits the path, and this set is what separates a + * deliberate omission from a missing-key bug. Kept separate from + * `oversizedKeys` so the two exclusions can never be confused in the counts. */ private val denylistedKeys = ConcurrentHashMap.newKeySet[(String, String)]() + /** `(origSha, fullPath)` of every blob whose tokenizer timed out or crashed in + * this run. Excluded like a denylisted blob, and checked in [[readBlob]] so + * that one failing blob costs one tokenizer run, not one per commit. */ + private val failedKeys = ConcurrentHashMap.newKeySet[(String, String)]() + + private def noteFailed(task: BlobMissTask, reason: String): Unit = + if (failedKeys.add((task.origId.name, task.fullPath))) + System.err.println( + s"blobExec: EXCLUDED failed blob: sha=${task.origId.name} path=${task.fullPath} " + + s"reason=$reason. The blob is left out of the rewritten tree, so it produces no " + + "blame and no dataset row. The walk carries on.") + + /** Count and explain one denylisted blob, once per `(sha, path)`. The sha, the + * path, the reason and the citation are all in the line, because this line is + * the only per-blob record of what the dataset does not contain, and "we could + * not parse it" is not a defensible sentence in a paper. */ private def noteDenylisted(task: BlobMissTask, entry: BlobDenylist.Entry): Unit = { val key = (task.origId.name, task.fullPath) + // Counted and logged once per (sha, path), exactly like an oversized blob, so + // the two counters mean the same thing. One blob reached through two paths is + // therefore two, which is the honest figure for "paths the dataset is missing" + // — but note the four shipped entries are ONE file's history, so a count of 4 + // is not four distinct files. if (denylistedKeys.add(key)) { blobsDenylisted.increment() System.err.println( s"blobExec: EXCLUDED denylisted blob: sha=${task.origId.name} " + s"path=${task.fullPath} reason=${entry.reason} citation=${entry.citation}. " + - s"See ${BlobDenylist.EntriesSource}. Reported, never fatal." + "The blob is left out of the rewritten tree, so it produces no blame and no dataset " + + "row, and the file is absent from the tokenized repository rather than present as raw " + + "source. The exclusion is deterministic and cited; the walk carries on and the run " + + "still exits 0. See " + + s"${BlobDenylist.ResourcePath} in the blobExec jar for the list itself." ) } } + /** Count and explain one exclusion, once per `(sha, path)`. The path and the + * size are in the line on purpose: a bare sha cannot be cited in a paper, + * and this line is the only durable record of what the dataset is missing. */ private def noteOversized(task: BlobMissTask, sizeBytes: Long): Unit = { val key = (task.origId.name, task.fullPath) if (oversizedKeys.add(key)) { blobsOversized.increment() System.err.println( s"blobExec: EXCLUDED oversized blob: sha=${task.origId.name} " + - s"path=${task.fullPath} size=${sizeBytes}B. JGit will not materialise an object " + - "this large, so the path is absent from the rewritten tree. Reported, not fatal." + s"path=${task.fullPath} size=${sizeBytes}B limit=${Walker.MaxBlobBytes}B. " + + "JGit will not materialise an object this large, and a source file this size is " + + "machine-generated rather than authored. The blob is left out of the rewritten " + + "tree, so it produces no blame and no dataset row, and the file is absent from " + + "the tokenized repository rather than present as raw source. The walk carries on " + + "and the run still exits 0." ) } } /** Copy one original blob from src into dst by streaming it. Returns (id, size). * - * Used instead of readBlob on the pass-through path, where a blob keeps the - * size the project committed: `ObjectLoader.getBytes` throws - * LargeObjectException above JGit's stream threshold, and raising the - * threshold would still hold the whole blob in the heap. + * Replaces readBlob on the pass-through path. `ObjectLoader.getBytes` throws + * LargeObjectException above JGit's stream threshold, and a blob that does not + * match the mask is copied verbatim, so its size is whatever the project + * committed rather than the size of a source file. redis/redis failed this way + * after 1,684s of work: + * + * org.eclipse.jgit.errors.LargeObjectException: 12e1ac54... exceeds size limit + * at cregit.blobexec.Walker.ensureOriginalBlobAvailable(Walker.scala:912) + * + * Streaming also removes the memory spike. Raising the threshold instead would + * still hold the whole blob, and three concurrent projects on a 30 GiB box + * cannot each afford a large fixture. * - * The reader stays open for the whole copy. Closing it before the stream is + * The reader stays open for the whole copy: closing it before the stream is * consumed would invalidate the stream. */ private def streamBlobInto(id: ObjectId, inserter: ObjectInserter): (ObjectId, Long) = { @@ -1126,6 +1163,8 @@ final class Walker( case TreeExisting(id) => id case TreeBuild(origId, entries) => val tf = new org.eclipse.jgit.lib.TreeFormatter + // flatMap, not map: an oversized blob resolves to no entry at all, so the + // rewritten tree simply does not contain that path. val resolvedEntries = entries.flatMap(resolveEntry(_, resolved, inserter, subtreeAcc)) // jgit requires sorted entries (git tree order). The TreeWalk visited // them in tree order already, so we keep that order. @@ -1160,11 +1199,17 @@ final class Walker( val key = (origId.name, fullPath) resolved.get(key) match { case Some(newId) => Some(ResolvedEntry(name, mode, newId, copyBytes = false)) - case None if oversizedKeys.contains(key) || denylistedKeys.contains(key) => None + // Excluded as oversized or by the denylist (see readBlob): drop the entry, + // so the file is absent from the rewritten tree rather than present as raw + // source. + case None if oversizedKeys.contains(key) || denylistedKeys.contains(key) || + failedKeys.contains(key) => None + // Anything else missing is a bug, and used to surface as a bare + // NoSuchElementException. Keep it fatal and say which blob it was. case None => throw new IllegalStateException( s"blob ${origId.name} ($fullPath) is a mask-matched miss with no resolution " + - "and was excluded neither as oversized nor by the blob denylist") + "and was excluded neither as oversized, by the blob denylist, nor as failed") } } @@ -1447,6 +1492,30 @@ final class Walker( object Walker { + /** Size at which a mask-matched blob stops being tokenizable and is excluded. + * + * Deliberately not a number of our own choosing. It is read from JGit, which + * refuses to materialise any object at or above its stream-file threshold + * (`core.streamFileThreshold`, default 50 MiB) — `ObjectLoader.getBytes` + * throws `LargeObjectException` instead. Picking a larger constant, as an + * earlier draft of this fix did with 64 MiB, leaves a band (50-64 MiB) in + * which the size check passes and `getBytes` then throws anyway; picking a + * smaller one would silently drop files JGit could have handled. + * + * Reading a *fresh* `WindowCacheConfig` gives JGit's default rather than + * whatever is currently installed, which cannot be read back through public + * API. That is why this is only a fast path and [[Walker#readBlob]] also + * catches `LargeObjectException`: with the threshold lowered below the + * default, the catch is what handles the band. + * + * `>=`, not `>`, because that is JGit's own comparison: `Pack.load` returns a + * streaming (non-materialisable) loader once `size >= streamFileThreshold`. + */ + private[blobexec] val MaxBlobBytes: Long = + new org.eclipse.jgit.storage.file.WindowCacheConfig().getStreamFileThreshold.toLong + + /** Pure form of the exclusion decision, so it can be checked without a repo. */ + private[blobexec] def isOversized(sizeBytes: Long): Boolean = sizeBytes >= MaxBlobBytes private val OriginalBlobCacheSize = 1 << 16 private val OriginalBlobCopyLockCount = 256 @@ -1500,43 +1569,22 @@ object Walker { * memory stays modest. */ private val PipelineWindow = 32 + /** Window with no completed work of any kind after which the run is declared + * stalled. Generous because it is a watchdog, not a schedule: every child is + * separately hard-bounded at `--blob-timeout`. */ + private[blobexec] val DefaultStallTimeoutSeconds = 1800 + /** Exit status when the watchdog kills a stalled run. Distinct from 4 (a blob * timed out but the walk finished) so the runner can tell them apart. */ private[blobexec] val StalledExitStatus = 5 - private val WatchdogTickMarginSeconds = 60 - - private val StallTimeoutMultiple = 3 + /** Headroom for the watchdog's tick (30 s at most) and the reap of a killed child. */ + private val StallFloorMarginSeconds = 60 + /** Smallest stall window that cannot fire while one blob is still inside its own + * lifetime: a commit whose last blob is slow completes nothing until it ends. */ private[blobexec] def stallFloorFor(blobTimeoutSeconds: Int): Int = - Saturating.sum(ChildRunner.maxLifetimeSeconds(blobTimeoutSeconds), WatchdogTickMarginSeconds) - - private[blobexec] def stallTimeoutFor(blobTimeoutSeconds: Int): Int = - math.max( - Saturating.product(math.max(1, blobTimeoutSeconds), StallTimeoutMultiple), - stallFloorFor(blobTimeoutSeconds) - ) - - private[blobexec] val DefaultStallTimeoutSeconds: Int = - stallTimeoutFor(BlobExec.DefaultTimeoutSeconds) - - private[blobexec] def resolveStallTimeout( - blobTimeoutSeconds: Int, - stallTimeoutSeconds: Int, - stallExplicit: Boolean - ): Either[String, Int] = { - val floor = stallFloorFor(blobTimeoutSeconds) - if (stallTimeoutSeconds >= floor) Right(stallTimeoutSeconds) - else if (stallExplicit) - Left( - s"--stall-timeout=$stallTimeoutSeconds must be at least ${floor}s to go with " + - s"--blob-timeout=$blobTimeoutSeconds. A blob completing or being killed is the only " + - "progress a pure-blob commit makes, and killing one takes the budget plus the kill " + - "grace, so a smaller stall window kills runs whose blobs are merely slow. Try " + - s"--stall-timeout=${stallTimeoutFor(blobTimeoutSeconds)} with --blob-timeout=$blobTimeoutSeconds." - ) - else Right(stallTimeoutFor(blobTimeoutSeconds)) - } + math.min(BlobExec.maxChildLifetimeSeconds(blobTimeoutSeconds).toLong + StallFloorMarginSeconds, Int.MaxValue.toLong).toInt /** Pure form of the watchdog's decision, so it can be tested without halting * a JVM: has more than `stallTimeoutSeconds` passed with no progress? */ @@ -1559,17 +1607,8 @@ object Walker { fullMessage: String ) - /** What [[Walker.resolveMisses]] hands back: `ids` is what the tree - * references, `timedOutKeys` is the subset that must not be persisted. */ - final case class MissResolution( - ids: IMap[(String, String), ObjectId], - abort: Boolean, - timedOutKeys: Set[(String, String)] - ) { - /** The rows that are genuinely correct and worth keeping: content-addressed - * tokenizations of blobs that did not time out. */ - def persistable: IMap[(String, String), ObjectId] = ids -- timedOutKeys - } + /** What [[Walker.resolveMisses]] hands back: the ids the tree references. */ + final case class MissResolution(ids: IMap[(String, String), ObjectId], abort: Boolean) /** Per-blob worker result handed from a pool thread back to the consumer. */ sealed trait BlobResult @@ -1578,13 +1617,11 @@ object Walker { final case class Resolved(newId: ObjectId) extends BlobResult /** abort-on-error tripped by a non-zero command exit. */ final case class Aborted(stderr: String, exitCode: Int) extends BlobResult - /** The command was killed on its budget. The tree still references the - * original blob (dst stays self-consistent), but this is deliberately not - * a `Resolved`: nothing about this blob, the trees containing it, or its - * commit may be persisted, or a re-run would treat raw source as done. */ - final case class Unusable(origId: ObjectId) extends BlobResult - /** Too large for JGit to materialise; contributes no id. */ - final case class Oversized(origId: ObjectId) extends BlobResult + /** Excluded from the rewrite: oversized, denylisted, or its tokenizer failed. + * It is deliberately not `Resolved`: it contributes no id, so + * every `collect` that builds a tree/blob_map map drops it, `resolveEntry` + * omits the path, and the run is still publishable. */ + final case class Excluded(origId: ObjectId) extends BlobResult } /** Items flowing producer -> consumer across the bounded queue. */ @@ -1608,11 +1645,7 @@ object Walker { final case class Built( newTreeId: ObjectId, resolved: IMap[(String, String), ObjectId], - subtrees: IMap[String, String], - /** Blobs in this commit whose command was killed on its budget. The tree - * is usable for this run's output, but if this is non-empty neither it - * nor the commit may be recorded, or the next run skips the retry. */ - timedOutKeys: Set[(String, String)] + subtrees: IMap[String, String] ) extends TreeResult final case class Aborted(stderr: String, exitCode: Int) extends TreeResult } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala index 88488fb5e..c4ab0174e 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala @@ -3,49 +3,288 @@ package cregit.blobexec import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -/** What a published dataset does NOT contain is part of its contract, so the shipped - * entries are asserted rather than trusted. */ +import java.nio.file.Files + +/** The denylist is a DATA file, because the list of blobs a published dataset + * omits is a property of the dataset and a paper has to cite it. That makes the + * file's contents part of the contract, so they are asserted here and not only + * parsed. */ class BlobDenylistSpec extends AnyFunSuite with Matchers { - // Four historical versions of one file at two paths: the list is keyed on content. - private val KnownShas = Set( + // CLASS 1 — the Java parser on type-annotated array types, upstream + // srcML/srcML#2361. Eight blobs, all in tencent__tencentkona-21's bare clone, + // all OpenJDK langtools regression tests. They are the historical versions of + // TWO files, one of which lives at two paths after a repository reorganisation — + // which is why the list is keyed on content, not on path. The second file + // carries a distinct sufficient construct within the same defect. + private val JavaAnnotationShas = Set( + // TestNewCastArray.java "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65", "303c1a109f1fdc4b9d84e15111c56931202844ac", "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca", - "c91924d1e87cb82d372a73b24579ccc34609efaa" + "c91924d1e87cb82d372a73b24579ccc34609efaa", + // CheckErrorsForSource7.java + "56c8adf588e697a3e42b600b2e5ec54710592c61", + "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5", + "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7", + "afbd81e5789ba060147b0de3be4f01759963299d" + ) + + // CLASS 2 — the C parser on C++ inside a `.h`, because CregitLanguages maps that + // extension to language C. A DIFFERENT defect in a DIFFERENT language, with NO + // upstream issue, so it cites the analysis document instead. Asserted separately + // because the one thing a reader must not do is read this list as one defect. + // Four historical versions of eden/fs/utils/StatTimes.h in facebook__sapling; + // the file's other four versions return by value and parse in milliseconds. + private val CParserShas = Set( + "5460d43f1884e3be230e02215151b9b963844a48", + "720914b27e170b73b38ef662fb04239273dd0a34", + "99c191e9a208547f5c9b1224c290f106a640de59", + "dbb885e0bf046755c4fcfbddfc2454f90c550c62" ) - test("the shipped list holds exactly the known blobs, so an addition is deliberate") { - // Also catches a sha written twice: a Map literal silently keeps the last. - BlobDenylistEntries.NonTerminating.keySet shouldEqual KnownShas + // CLASS 3 — the C and C++ parsers CRASHING ON A SIGNAL (SIGSEGV or SIGABRT) + // under `--position`. A THIRD defect, and the only one that is not a hang: srcml + // is dead in milliseconds, so no timeout budget is ever involved. 197 blobs, the + // whole history of 36 files across 18 projects, found by running the tokenizer's + // real chain over all 904 historical versions of those paths (707 of the 904 + // parse cleanly — this is content-triggered, not a property of the path). + // + // Asserted by SIZE and by spot-checked members rather than by listing 197 + // literals: a 197-line literal set duplicated here would be maintained by + // copy-paste and would stop being a check. What must not drift is the count, the + // citation, and the fact that class 3 never claims to be #2361. + private val ParserCrashCount = 197 + + // Spot checks, one per distinguishing property, so the assertions below are + // anchored on real entries rather than on whatever happens to be first. + // - the sweep's own control file, 1,263 B, which rules out a size limit + // - a SIGABRT entry, where the signal is NOT a segfault + // - a blob vendored byte-identically into TWO projects, which is why the list + // is keyed on content and not on path + private val CrashPositionOnlySha = "4a62830c807a5a2aff73f474891c38ed913d39c4" + private val CrashAbortSha = "37cf5d5b1985a7510ad9c952897ef253231e80f7" + private val CrashSharedBlobSha = "012ea8148e8d21cf515a9ff47d4af719d049b276" + + private val KnownHangShas = JavaAnnotationShas ++ CParserShas + + test("the shipped list is on the classpath and holds the twelve hangs plus 197 crashes") { + // A jar built without the resource would hand all of these back to srcml. The + // twelve would hang for 600s each and then exit 4; the 197 would segfault or + // abort, and blobExec would exit 6 on `blobsParserCrashed`. Either way the + // project cannot publish, so "the resource is present" is itself a requirement. + BlobDenylist.shipped.size shouldEqual 12 + ParserCrashCount + JavaAnnotationShas.size shouldEqual 8 + CParserShas.size shouldEqual 4 + KnownHangShas.foreach { sha => + withClue(s"$sha is missing from the shipped denylist: ") { + BlobDenylist.shipped.entryFor(sha) should not be empty + } + } + } + + test("class 3 holds exactly the 197 signal-death blobs, and no hang is counted among them") { + // The two groups are separated by citation, which is the only thing in the data + // file that distinguishes them, so this also pins that the citations did not + // get pasted across classes. + val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) + crashes.size shouldEqual ParserCrashCount + crashes.map(_.sha).toSet.size shouldEqual ParserCrashCount // no duplicates + crashes.map(_.sha).toSet intersect KnownHangShas shouldBe empty } - test("every key is a 40-character lowercase hex git blob id") { - // Not the memo's content hash, and not a path. - BlobDenylistEntries.NonTerminating.keys.foreach { sha => - withClue(s"$sha: ") { sha should fullyMatch regex "[0-9a-f]{40}" } + test("a class 3 entry says CRASH, names its signal, and disclaims #2361") { + // "we could not parse it" is indefensible in a paper; so is calling a segfault + // a hang. Every class 3 reason has to carry the defect, the signal and the + // srcML version, and has to say which invocation produced it. + val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) + crashes.foreach { e => + withClue(s"${e.sha}: ") { + e.citation should not include "2361" + e.reason should include("srcML 1.1.0") + e.reason should include("--position") + e.reason.toUpperCase should include("CRASH") + e.reason should (include("SIGSEGV (shell status 139)") or include("SIGABRT (shell status 134)")) + // the flag is not optional, and the reason has to say why + e.reason should include("token format carries line:col positions") + // a crash is not the hang, and must not be mistaken for it + e.reason should include("not the non-termination") + } } } - test("every entry cites a defect and says what it is") { - BlobDenylistEntries.NonTerminating.foreach { case (sha, e) => - withClue(s"$sha: ") { - e.citation should not be empty + test("each class 3 entry records whether --position is the trigger, measured per blob") { + // silent-empty-sweep.md claims --position is the trigger for all of them. It is + // not: measured per blob, 132 are position-only, 49 die either way, and 16 exit + // 1 without the flag. Inheriting the sweep's single claim would have put a + // false statement on 65 entries, so each one states its own measurement and + // this holds the three groups to their counts. + val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) + val positionOnly = crashes.count(_.reason.contains("--position IS the trigger")) + val eitherWay = crashes.count(_.reason.contains("--position is NOT the trigger")) + val noHelp = crashes.count(_.reason.contains("dropping --position would not recover")) + + positionOnly shouldEqual 132 + eitherWay shouldEqual 49 + noHelp shouldEqual 16 + positionOnly + eitherWay + noHelp shouldEqual ParserCrashCount + } + + test("the spot-checked class 3 blobs are present, with the property each was chosen for") { + // crc32_small.c, 1,263 B — the sweep's control, and the file the committed + // reproducer tests/t/fixtures/srcml-position-crash.c is derived from. + val posOnly = BlobDenylist.shipped.entryFor(CrashPositionOnlySha).getOrElse( + fail(s"$CrashPositionOnlySha (crc32_small.c) is missing from the shipped denylist")) + posOnly.reason should include("--position IS the trigger") + posOnly.reason should include("SIGSEGV (shell status 139)") + + // An abort, not a segfault. The defect is broader than "segfault", and an + // abort evades a timeout-shaped denylist just as thoroughly. + val abort = BlobDenylist.shipped.entryFor(CrashAbortSha).getOrElse( + fail(s"$CrashAbortSha (util-linux swaplabel.c) is missing from the shipped denylist")) + abort.reason should include("SIGABRT (shell status 134)") + + // One blob, two projects: zlib's deflate.c is byte-identical in ossec__ossec-hids + // and azerothcore__azerothcore-wotlk. A path-keyed list would need two entries + // and would be wrong the moment a third project vendored the same bytes. + BlobDenylist.shipped.entryFor(CrashSharedBlobSha) should not be empty + } + + test("every entry carries a one-line reason and a citation for its own defect") { + // Without both, an exclusion is indistinguishable from data loss. The partner + // rejected "we could not parse it" as indefensible in a paper, and this is the + // assertion that keeps the answer in the artefact. + // + // The citation is per-CLASS, not global. Citing #2361 for the C parser hang + // would be a false attribution in a published artefact: it is a different + // language and a different construct, and no upstream issue exists for it. + // The same applies to class 3, which is not even a hang. + BlobDenylist.shipped.entries.foreach { e => + withClue(s"${e.sha}: ") { + if (JavaAnnotationShas.contains(e.sha)) e.citation shouldEqual "srcML/srcML#2361" + else if (CParserShas.contains(e.sha)) e.citation should include("srcml-nontermination-analysis.md") + else e.citation should include("silent-empty-sweep.md") + e.reason should not be empty e.reason should not include "\n" + e.reason.toLowerCase should include("srcml") + } + } + } + + test("every citation belongs to exactly one of the three classes, so none is unattributed") { + // A pasted or empty citation would leave an omission in a published dataset with + // nothing to look up. Partitioning the whole list proves there is no fourth, + // accidental citation string hiding in the file. + val byClass = BlobDenylist.shipped.entries.groupBy { e => + if (e.citation == "srcML/srcML#2361") "class1" + else if (e.citation.contains("srcml-nontermination-analysis.md")) "class2" + else if (e.citation.contains("silent-empty-sweep.md")) "class3" + else "UNATTRIBUTED: " + e.citation + } + byClass.keys.filter(_.startsWith("UNATTRIBUTED")) shouldBe empty + byClass("class1").size shouldEqual 8 + byClass("class2").size shouldEqual 4 + byClass("class3").size shouldEqual ParserCrashCount + } + + test("the C parser entries do not claim to be srcML#2361, and carry their reproducer") { + // The evidence for these is a minimised reproducer in the artefact rather than + // an upstream issue number, so the reproducer is part of the contract. + CParserShas.foreach { sha => + val e = BlobDenylist.shipped.entryFor(sha).getOrElse( + fail(s"the C parser blob $sha is missing from the shipped denylist")) + withClue(s"$sha: ") { + e.citation should not include "2361" + e.reason should include("namespace n{struct T&f(const struct S);}") } } } test("lookup is by git blob id, in either case, and misses everything else") { - val sha = KnownShas.head - BlobDenylist.shipped.entryFor(sha) should not be empty - BlobDenylist.shipped.entryFor(sha.toUpperCase) should not be empty + val sha = KnownHangShas.head + BlobDenylist.shipped.entryFor(sha).map(_.sha) shouldEqual Some(sha) + BlobDenylist.shipped.entryFor(sha.toUpperCase).map(_.sha) shouldEqual Some(sha) BlobDenylist.shipped.entryFor("0" * 40) shouldEqual None BlobDenylist.shipped.entryFor("") shouldEqual None } test("an empty list matches nothing, so a test fixture cannot exclude by accident") { BlobDenylist.empty.isEmpty shouldBe true - BlobDenylist.empty.entryFor(KnownShas.head) shouldEqual None + BlobDenylist.empty.entryFor(KnownHangShas.head) shouldEqual None + } + + // -- the parser -------------------------------------------------------------- + + private val sha1 = "a" * 40 + private val sha2 = "b" * 40 + + test("comments, blank lines and surrounding whitespace are ignored") { + val list = BlobDenylist.parse( + Vector( + "# a comment", + "", + " ", + s" $sha1\tsrcML/srcML#2361\tnon-termination ", + s"$sha2\tsrcML/srcML#2361\tnon-termination\t" // a trailing tab is tolerated + ), + "fixture") + list.size shouldEqual 2 + list.entryFor(sha1).map(_.reason) shouldEqual Some("non-termination") + } + + test("a line that is not three fields is refused, naming the line") { + // Silently skipping a malformed line would re-admit a blob that hangs the run + // for 600s and then blocks publication — the exact failure this list removes. + val bad = intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector(s"$sha1\tsrcML/srcML#2361"), "fixture") + } + bad.getMessage should include("fixture:1") + intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector(s"$sha1\t\treason"), "fixture") // empty citation + } + intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector(s"$sha1\tcitation\t"), "fixture") // empty reason + } + } + + test("a key that is not a 40-hex git blob sha is refused") { + // The memo's key is sha1 of the file contents with no git header, so a + // plausible-looking wrong hash is an easy mistake to make. A 39-character or + // non-hex key would simply never match, silently. + intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector(s"${"a" * 39}\tc\tr"), "fixture") + } + intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector(s"${"z" * 40}\tc\tr"), "fixture") + } + intercept[IllegalArgumentException] { + BlobDenylist.parse(Vector("TestNewCastArray.java\tc\tr"), "fixture") + } + } + + test("a duplicated sha is refused: two reasons for one blob means one is wrong") { + val bad = intercept[IllegalArgumentException] { + BlobDenylist.parse( + Vector(s"$sha1\tc\tfirst reason", s"${sha1.toUpperCase}\tc\tsecond reason"), + "fixture") + } + bad.getMessage should include("twice") + } + + test("the same parser reads a file, which is how a fixture list is built") { + val f = Files.createTempFile("denylist-", ".tsv") + try { + Files.writeString(f, s"# header\n$sha1\tsrcML/srcML#2361\tnon-termination\n") + val list = BlobDenylist.fromFile(f) + list.shas shouldEqual Set(sha1) + list.entryFor(sha1).map(_.describe) shouldEqual + Some("non-termination [srcML/srcML#2361]") + } finally Files.deleteIfExists(f) + } + + test("a missing resource is fatal rather than an empty list") { + intercept[IllegalStateException] { + BlobDenylist.fromResource("/cregit/blobexec/no-such-denylist.tsv") + } } } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala index dbef5ecb6..ef77f040a 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala @@ -101,6 +101,12 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } } + // CHANGED ASSERTION. This test used to be "zero exit, empty stdout against + // non-empty input → Replace with empty blob", and it pinned the defect: a + // crashed srcML exits 0 through the wrapper with empty stdout, and that empty + // stdout was inserted as the file's tokenization. 36 files across 19 projects + // were published as 0-byte blobs that way, with nothing counting them. Zero + // output from non-empty input is now a counted failure, never a blob. test("zero exit, empty stdout against non-empty input → Skip, counted, no blob") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) val cmd = shellScript("cat > /dev/null; true") @@ -111,6 +117,9 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { crashes.get shouldEqual 1 } + // The other half of the distinction the fix has to make: an input that is + // genuinely empty may legitimately produce empty output, and must stay a plain + // Skip that is NOT counted as a crash. test("zero exit, empty stdout against empty input → Skip, not counted a crash") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) val cmd = shellScript("cat > /dev/null; true") @@ -123,6 +132,9 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { test("the parser-crash exit status is Skip, counted, and never a Replace") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) + // Writes plausible-looking output first, so the test proves the status is what + // rejects it rather than the emptiness check: a crashed srcML can emit a + // truncated prefix before dying, and that prefix must never become a blob. val cmd = shellScript(s"echo 'partial tokens'; exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, abortOnError = false, inserter, @@ -132,6 +144,9 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } test("a parser crash skips one blob even with abortOnError, like a timeout") { + // The hostile case: one crashing blob must not take down a run that has + // already folded thousands of commits. It gates publication through the exit + // status instead (Main.exitStatus), which is where a timeout gates it too. val cmd = shellScript(s"exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, abortOnError = true, inserter) shouldBe BlobExec.Outcome.Skip @@ -154,21 +169,31 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { timeouts.get shouldEqual 1 crashes.get shouldEqual 1 // a timeout is not a crash + // A healthy blob and an ordinary failure must leave both counts alone, or a + // project would be held back from publication for nothing. run(shellScript("tr a-z A-Z")) run(shellScript("exit 7")) crashes.get shouldEqual 1 timeouts.get shouldEqual 1 } - test("a child that never exits is killed at its budget, not awaited forever") { - // `sleep 30` is the wedged srcml chain: it reads nothing, writes nothing, and - // holds its stdout open so the reader parks in pipe_read. + test("a child that never exits is killed at the timeout, not awaited forever") { + // `sleep 30` stands in for the wedged srcml chain: it reads nothing and + // writes nothing, and it holds its stdout open so the reader thread parks + // in pipe_read. With a 1 second budget the call must return quickly. val cmd = shellScript("sleep 30") val started = System.currentTimeMillis() - val outcome = new ChildRunner(1).run(cmd, "irrelevant".getBytes(UTF_8), Nil) + val (exit, _, _) = BlobExec.invoke( + "irrelevant".getBytes(UTF_8), + "0" * 40, + "input.java", + "input.java", + cmd, + timeoutSeconds = 1 + ) val elapsed = System.currentTimeMillis() - started - assert(elapsed < 10000, s"run took ${elapsed}ms; the budget did not fire") - outcome shouldBe a[ChildRunner.Outcome.Killed] + assert(elapsed < 10000, s"invoke took ${elapsed}ms; the timeout did not fire") + exit shouldEqual BlobExec.TimeoutExitCode } test("a timed-out blob is skipped, never tokenized and never an abort") { @@ -181,57 +206,20 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { out shouldBe BlobExec.Outcome.Skip } - test("an orphan grandchild holding the pipes cannot outlive the budget") { - // The real process shape, tokenBySha.pl -> sh -> srcml: killing only the - // direct child leaves the background `sleep` holding this JVM's pipes. + test("an orphan grandchild holding the pipes cannot outlive the timeout") { + // The tencentkona-21 shape: tokenBySha.pl -> sh -> srcml, where killing the + // direct child leaves grandchildren holding the JVM's stdout/stderr pipes, + // the reader threads blocked, and `exitValue()`'s join unreturnable. The + // background `sleep` is that grandchild. Only a process-group kill returns + // near the budget; a destroy() of the direct child alone would park here + // until the JVM-side backstop latch, far past this assertion. val cmd = shellScript("sleep 30 & sleep 30") val started = System.currentTimeMillis() - val outcome = new ChildRunner(1).run(cmd, "irrelevant".getBytes(UTF_8), Nil) + val (exit, _, _) = BlobExec.invoke( + "irrelevant".getBytes(UTF_8), "0" * 40, "input.java", "input.java", cmd, timeoutSeconds = 1) val elapsed = System.currentTimeMillis() - started - assert(elapsed < 10000, s"run took ${elapsed}ms; the descendant tree was not killed") - outcome shouldBe a[ChildRunner.Outcome.Killed] - } - - test("a descendant holding the pipe cannot park the call, nor forge output") { - // perl's fork() guarantees the child inherits fd 1. A shell's `&` does not - // reliably, which is why this does not use one. - val cmd = shellScript( - """exec perl -e 'if (fork() == 0) { sleep 10; exit 0 } print "partial"; exit 0'""") - val started = System.currentTimeMillis() - val outcome = new ChildRunner(30, drainGraceSeconds = 1).run(cmd, Array.emptyByteArray, Nil) - val elapsed = System.currentTimeMillis() - started - - // The holder lives 10s. Either branch below is safe, and neither may wait for it. - assert(elapsed < 5000, s"run took ${elapsed}ms: it waited for the descendant") - outcome match { - // EOF was reached, so the buffer holds everything the child wrote. - case ChildRunner.Outcome.Exited(status, stdout, _) => - status shouldEqual 0 - new String(stdout, UTF_8) shouldEqual "partial" - // The reader was still blocked, so no output is offered at all. - case ChildRunner.Outcome.Killed(why) => - why should include("still open") - } - } - - test("a blob whose command leaves a descendant on the pipe is never an abort") { - val cmd = shellScript( - """exec perl -e 'if (fork() == 0) { sleep 10; exit 0 } print "partial"; exit 0'""") - val started = System.currentTimeMillis() - val outcome = BlobExec.run("original".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, - abortOnError = true, inserter, timeoutSeconds = 30) - assert(System.currentTimeMillis() - started < 20000, "the walk waited for the descendant") - outcome should not be a[BlobExec.Outcome.Abort] - } - - test("the environment reaches the child, and the exit status comes back") { - val cmd = shellScript("""printf '%s' "$BFG_PATH"; exit 3""") - inside(new ChildRunner(30).run(cmd, Array.emptyByteArray, Seq("BFG_PATH" -> "src/x.c"))) { - case ChildRunner.Outcome.Exited(status, stdout, _) => - status shouldEqual 3 - new String(stdout, UTF_8) shouldEqual "src/x.c" - case other => fail(s"expected Exited, got $other") - } + assert(elapsed < 10000, s"invoke took ${elapsed}ms; the process group was not killed") + exit shouldEqual BlobExec.TimeoutExitCode } test("a timeout is reported to the caller exactly once, and only on a timeout") { diff --git a/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala new file mode 100644 index 000000000..5028d6ab4 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala @@ -0,0 +1,161 @@ +package cregit.blobexec + +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.{ObjectId, Repository} +import org.eclipse.jgit.revwalk.RevWalk +import org.eclipse.jgit.storage.file.FileRepositoryBuilder +import org.eclipse.jgit.treewalk.TreeWalk +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +/** A blob whose tokenizer times out or crashes is excluded from the rewritten + * tree, like a denylisted blob: the walk carries on, the other blobs keep their + * tokens, and the run is not aborted. */ +class FailedBlobSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val workRoot: Path = Files.createTempDirectory("failed-blob-") + + override def afterAll(): Unit = deleteRecursive(workRoot) + + private def tokenizer(dir: Path, onB: String): String = { + val f = dir.resolve("tok.sh") + Files.writeString( + f, + s"""|#!/bin/sh + |if [ "$$BFG_FILENAME" = "b.c" ]; then + | $onB + |else + | tr a-z A-Z + |fi + |""".stripMargin + ) + f.toFile.setExecutable(true) + f.toAbsolutePath.toString + } + + private val hang = "sleep 300 & sleep 300" + private val crash = s"cat >/dev/null; exit ${BlobExec.ParserCrashExitCode}" + + private def run(src: Repository, dstPath: Path, dbPath: Path, command: String, mode: String): WalkStats = { + val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] + dst.create(true) + val mapping = Mapping.open(dbPath, command, """\.c$""") + try + new Walker( + src, dst, mapping, """\.c$""".r, command, + abortOnError = false, + parallelism = 2, + pipeline = mode == "pipeline", + pipelineTrees = mode == "pipeline-trees", + blobTimeoutSeconds = 1 + ).run() + finally { + mapping.close() + dst.close() + } + } + + private final case class Fixture(src: Repository, commits: Vector[ObjectId], dir: Path) + + private def fixture(extraCommitInDeep: Boolean): Fixture = { + val dir = Files.createTempDirectory(workRoot, "fixture-") + val srcDir = dir.resolve("src") + Files.createDirectories(srcDir.resolve("deep")) + val git = Git.init().setDirectory(srcDir.toFile).call() + def commit(msg: String) = + git.commit().setMessage(msg).setAuthor("Tester", "t@example.org") + .setCommitter("Tester", "t@example.org").call().getId + Files.writeString(srcDir.resolve("a.c"), "alpha\n") + Files.writeString(srcDir.resolve("deep/b.c"), "beta\n") + git.add().addFilepattern(".").call() + val first = commit("two blobs, one of which fails") + val rest = + if (!extraCommitInDeep) Vector.empty + else { + Files.writeString(srcDir.resolve("deep/c.c"), "gamma\n") + git.add().addFilepattern(".").call() + Vector(commit("a sibling of the failing blob")) + } + Fixture(git.getRepository, first +: rest, dir) + } + + for (mode <- Seq("serial", "pipeline", "pipeline-trees")) { + for ((kind, onB) <- Seq("timed-out" -> hang, "crashed" -> crash)) + test(s"[$mode] a $kind blob is excluded and the commit is still folded") { + val f = fixture(extraCommitInDeep = false) + val dstPath = f.dir.resolve("dst.git") + val dbPath = f.dir.resolve("blobmap.db") + val command = tokenizer(f.dir, onB) + + val stats = run(f.src, dstPath, dbPath, command, mode) + + stats.aborted shouldBe false + stats.commitsProcessed shouldEqual 1 + (stats.blobsTimedOut + stats.blobsParserCrashed) shouldEqual 1 + Main.exitStatus(stats) shouldEqual 0 + val m = Mapping.open(dbPath, command, """\.c$""") + try { + val newCommit = m.getCommit(f.commits.head.name) + newCommit shouldBe defined + filesIn(dstPath, newCommit.get) shouldEqual Map("a.c" -> "ALPHA\n") + m.getBlob(blobIdAt(f.src, f.commits.head, "deep/b.c").name, "deep/b.c") shouldEqual None + } finally m.close() + } + + test(s"[$mode] a failed blob is tried once per run, not once per commit") { + val f = fixture(extraCommitInDeep = true) + val dstPath = f.dir.resolve("dst.git") + val dbPath = f.dir.resolve("blobmap.db") + val command = tokenizer(f.dir, hang) + + val stats = run(f.src, dstPath, dbPath, command, mode) + + stats.commitsProcessed shouldEqual 2 + stats.blobsTimedOut shouldEqual 1 + val m = Mapping.open(dbPath, command, """\.c$""") + try { + val second = m.getCommit(f.commits(1).name) + second shouldBe defined + filesIn(dstPath, second.get) shouldEqual Map("a.c" -> "ALPHA\n", "deep/c.c" -> "GAMMA\n") + } finally m.close() + } + } + + // -- helpers --------------------------------------------------------------- + + private def blobIdAt(repo: Repository, commitId: ObjectId, path: String): ObjectId = { + val rw = new RevWalk(repo) + try { + val tw = TreeWalk.forPath(repo, path, rw.parseCommit(commitId).getTree) + try tw.getObjectId(0) finally tw.close() + } finally rw.close() + } + + private def filesIn(dstPath: Path, commitSha: String): Map[String, String] = { + val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] + val rw = new RevWalk(dst) + val tw = new TreeWalk(dst) + try { + tw.addTree(rw.parseCommit(ObjectId.fromString(commitSha)).getTree) + tw.setRecursive(true) + val out = Map.newBuilder[String, String] + while (tw.next()) + out += tw.getPathString -> new String(dst.open(tw.getObjectId(0)).getBytes, UTF_8) + out.result() + } finally { tw.close(); rw.close(); dst.close() } + } + + private def deleteRecursive(p: Path): Unit = { + if (Files.isDirectory(p)) { + val s = Files.list(p) + try s.forEach(deleteRecursive) finally s.close() + } + Files.deleteIfExists(p) + () + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index 2ae5f1aad..f40df421f 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -39,50 +39,54 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.parsePositiveSeconds("10.5") shouldEqual None } - test("timeout and stall have distinct exit statuses, and neither collides") { - Main.TimedOutExitStatus shouldEqual 4 + test("the stall status collides with no other exit status") { Walker.StalledExitStatus shouldEqual 5 - Main.TimedOutExitStatus should not equal Walker.StalledExitStatus - Set(0, 1, 2, 3) should not contain Main.TimedOutExitStatus Set(0, 1, 2, 3) should not contain Walker.StalledExitStatus } // -- the two timeouts are coupled ------------------------------------------- + // + // A pure-blob commit's only progress stamp is a blob finishing or being killed, + // so the watchdog window must exceed the per-blob budget. The defaults (600 and + // 1800) satisfy it; following the advice to raise --blob-timeout to 1800 without + // touching the window would not, and a legitimately slow blob would then race + // its own watchdog. test("a window at or above the floor is accepted unchanged") { - Walker.resolveStallTimeout(600, 1800, stallExplicit = false) shouldEqual Right(1800) - Walker.resolveStallTimeout(600, 1800, stallExplicit = true) shouldEqual Right(1800) - Walker.resolveStallTimeout(600, Walker.stallFloorFor(600), stallExplicit = true) shouldEqual + Main.resolveStallTimeout(600, 1800, stallExplicit = false) shouldEqual Right(1800) + Main.resolveStallTimeout(600, 1800, stallExplicit = true) shouldEqual Right(1800) + Main.resolveStallTimeout(600, Walker.stallFloorFor(600), stallExplicit = true) shouldEqual Right(Walker.stallFloorFor(600)) } test("the floor is the child's whole lifetime, not just the budget") { - Walker.stallFloorFor(600) should be > ChildRunner.maxLifetimeSeconds(600) - Walker.resolveStallTimeout(600, 601, stallExplicit = true).isLeft shouldBe true - Walker.resolveStallTimeout(600, 630, stallExplicit = true).isLeft shouldBe true + Walker.stallFloorFor(600) should be > BlobExec.maxChildLifetimeSeconds(600) + Main.resolveStallTimeout(600, 601, stallExplicit = true).isLeft shouldBe true + Main.resolveStallTimeout(600, 630, stallExplicit = true).isLeft shouldBe true } test("a refusal names the floor it wants, and the suggestion clears it") { - val why = Walker.resolveStallTimeout(600, 601, stallExplicit = true).swap.getOrElse("") + val why = Main.resolveStallTimeout(600, 601, stallExplicit = true).swap.getOrElse("") why should include(Walker.stallFloorFor(600).toString) - val suggested = Walker.stallTimeoutFor(600) - Walker.resolveStallTimeout(600, suggested, stallExplicit = true) shouldEqual Right(suggested) + val suggested = Main.widenedStall(600) + Main.resolveStallTimeout(600, suggested, stallExplicit = true) shouldEqual Right(suggested) } - test("a small budget is widened past the fixed kill overhead, not just tripled") { - Walker.resolveStallTimeout(5, 3, stallExplicit = false) - .getOrElse(0) should be >= Walker.stallFloorFor(5) - Walker.resolveStallTimeout(1, 1, stallExplicit = false) - .getOrElse(0) should be >= Walker.stallFloorFor(1) + test("a small --blob-timeout no longer lets the watchdog stop a healthy run") { + val oneBlobLifetimeNanos = BlobExec.maxChildLifetimeSeconds(5).toLong * 1000000000L + Main.resolveStallTimeout(5, 15, stallExplicit = true).isLeft shouldBe true + val window = Main.resolveStallTimeout(5, 3, stallExplicit = false).getOrElse(0) + Walker.isStalled(oneBlobLifetimeNanos, 0L, window) shouldBe false + Main.resolveStallTimeout(1, 1, stallExplicit = false).getOrElse(0) should be >= Walker.stallFloorFor(1) } test("the floor cannot overflow into a guard that always passes") { Walker.stallFloorFor(Int.MaxValue) should be > 0 - ChildRunner.maxLifetimeSeconds(Int.MaxValue) should be > 0 + BlobExec.maxChildLifetimeSeconds(Int.MaxValue) should be > 0 } test("the defaults satisfy the relationship") { - Walker.resolveStallTimeout( + Main.resolveStallTimeout( BlobExec.DefaultTimeoutSeconds, Walker.DefaultStallTimeoutSeconds, stallExplicit = false @@ -90,75 +94,35 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("a defaulted window is widened to fit a raised --blob-timeout") { - Walker.resolveStallTimeout(1800, 1800, stallExplicit = false) shouldEqual Right(5400) - Walker.resolveStallTimeout(3600, 1800, stallExplicit = false) shouldEqual Right(10800) + // The exact collision the printed advice used to create: --blob-timeout=1800 + // against the 1800 default window. + Main.resolveStallTimeout(1800, 1800, stallExplicit = false) shouldEqual Right(5400) + Main.resolveStallTimeout(3600, 1800, stallExplicit = false) shouldEqual Right(10800) } test("an explicit window that is too small is refused, naming both values") { - val bad = Walker.resolveStallTimeout(1800, 1800, stallExplicit = true) + val bad = Main.resolveStallTimeout(1800, 1800, stallExplicit = true) bad.isLeft shouldBe true val why = bad.swap.getOrElse("") why should include("--stall-timeout=1800") why should include("--blob-timeout=1800") - Walker.resolveStallTimeout(600, 60, stallExplicit = true).isLeft shouldBe true + Main.resolveStallTimeout(600, 60, stallExplicit = true).isLeft shouldBe true } test("widening cannot overflow Int") { - Walker.resolveStallTimeout(Int.MaxValue, 1800, stallExplicit = false) shouldEqual Right(Int.MaxValue) + Main.resolveStallTimeout(Int.MaxValue, 1800, stallExplicit = false) shouldEqual Right(Int.MaxValue) } - // -- the stall window's floor ----------------------------------------------- - - test("the floor clears one blob's whole lifetime, kill path included") { - Walker.stallFloorFor(600) should be > ChildRunner.maxLifetimeSeconds(600) - Walker.stallFloorFor(1) should be > ChildRunner.maxLifetimeSeconds(1) - } - - test("the default window is derived from the blob budget, not a second literal") { - Walker.DefaultStallTimeoutSeconds shouldEqual Walker.stallTimeoutFor(BlobExec.DefaultTimeoutSeconds) - } - - test("a defaulted window follows a raised budget past the floor") { - Walker.stallTimeoutFor(1800) shouldEqual 5400 - Walker.stallTimeoutFor(5) should be >= Walker.stallFloorFor(5) - Walker.stallTimeoutFor(1) should be >= Walker.stallFloorFor(1) - } - - test("a blob budget past the default window raises the floor above it") { - Walker.stallFloorFor(3600) should be > Walker.DefaultStallTimeoutSeconds - } - - test("the floor grows with the blob budget, and never underflows") { - Walker.stallFloorFor(60) should be < Walker.stallFloorFor(600) - Walker.stallFloorFor(0) should be > 0 - Walker.stallFloorFor(-5) should be > 0 - } - - test("a run at exactly the floor is not stalled by its own slowest blob") { - val blobBudget = 3600 - val floor = Walker.stallFloorFor(blobBudget) - val quiet = ChildRunner.maxLifetimeSeconds(blobBudget).toLong * 1000000000L - Walker.isStalled(nowNanos = quiet, lastProgressNanos = 0L, floor) shouldBe false - } - - test("the accepted range of a timeout keeps every derived window inside Int") { - Main.parsePositiveSeconds((Main.MaxTimeoutSeconds + 1).toString) shouldEqual None - Main.parsePositiveSeconds(Int.MaxValue.toString) shouldEqual None - Walker.stallTimeoutFor(Main.MaxTimeoutSeconds) should be > 0 - ChildRunner.maxLifetimeSeconds(Main.MaxTimeoutSeconds) should be > 0 - } - - // -- which counters gate publication ---------------------------------------- + // -- which counters change the exit status ---------------------------------- private def stats( aborted: Boolean = false, blobsTimedOut: Long = 0L, blobsOversized: Long = 0L, blobsDenylisted: Long = 0L, - blobsParserCrashed: Long = 0L, - blobsTokenized: Int = 1 + blobsParserCrashed: Long = 0L ) = WalkStats( - commitsProcessed = 1, commitsAlreadyMapped = 0, blobsRunThroughCommand = blobsTokenized, + commitsProcessed = 1, commitsAlreadyMapped = 0, blobsRunThroughCommand = 1, blobsCacheHit = 0, refsProjected = 1, aborted = aborted, blobsTimedOut = blobsTimedOut, blobsOversized = blobsOversized, blobsDenylisted = blobsDenylisted, blobsParserCrashed = blobsParserCrashed, @@ -183,10 +147,11 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.exitStatus(stats(blobsOversized = 3, blobsDenylisted = 4)) shouldEqual 0 } - test("a timeout still blocks publication, even alongside a denylisted blob") { - Main.exitStatus(stats(blobsTimedOut = 1)) shouldEqual Main.TimedOutExitStatus - Main.exitStatus(stats(blobsTimedOut = 1, blobsDenylisted = 4)) shouldEqual - Main.TimedOutExitStatus + test("a failed blob does not change the exit status: it is excluded") { + Main.exitStatus(stats(blobsTimedOut = 1)) shouldEqual 0 + Main.exitStatus(stats(blobsParserCrashed = 36)) shouldEqual 0 + Main.exitStatus(stats(blobsTimedOut = 1, blobsParserCrashed = 1, blobsDenylisted = 4, + blobsOversized = 3)) shouldEqual 0 } test("an abort still wins over everything") { @@ -195,44 +160,31 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.exitStatus(stats(aborted = true, blobsParserCrashed = 1)) shouldEqual 2 } - test("a crash rate past tolerance blocks publication, with its own status") { - Main.exitStatus(stats(blobsParserCrashed = 2, blobsTokenized = 100)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("a crash rate within tolerance still publishes: the blob is excluded either way") { - Main.exitStatus(stats(blobsParserCrashed = 1, blobsTokenized = 100)) shouldEqual 0 - Main.exitStatus(stats(blobsParserCrashed = 500, blobsTokenized = 500000)) shouldEqual 0 - } - - test("one crash in a tiny project does not gate, so rounding cannot zero the tolerance") { - Main.exitStatus(stats(blobsParserCrashed = 1, blobsTokenized = 1)) shouldEqual 0 - Main.exitStatus(stats(blobsParserCrashed = 2, blobsTokenized = 1)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("a crash past tolerance still blocks alongside explained exclusions") { - Main.exitStatus(stats(blobsParserCrashed = 9, blobsTokenized = 100, - blobsDenylisted = 4, blobsOversized = 3)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("the parser-crash status collides with no other blobExec exit status") { + // The ineffective-invalidation status must be distinct: run_pipeline_process.sh maps blobExec's statuses onto different + // remedies and different marker files, so a collision sends the operator to the + // wrong one. This status specifically must not be 0 — "--retokenize did + // nothing" reported as success is the failure the flag exists to prevent — and + // must not be 3, which means the run never started. + test("the ineffective-retokenize status collides with no other blobExec exit status") { val others = Map( - "clean" -> 0, - "usage" -> 1, - "aborted" -> 2, - "maskChanged" -> 3, - "timedOut" -> Main.TimedOutExitStatus, - "stalled" -> Walker.StalledExitStatus + "clean" -> 0, + "usage" -> 1, + "aborted" -> 2, + "maskChanged" -> 3, + "stalled" -> Walker.StalledExitStatus ) others.foreach { case (name, status) => - withClue(s"parser-crash status must differ from $name ($status): ") { - Main.ParserCrashedExitStatus should not equal status + withClue(s"ineffective-retokenize status must differ from $name ($status): ") { + Main.RetokenizeIneffectiveExitStatus should not equal status } } } + test("an ineffective --retokenize is not reported as a clean walk") { + // Belt and braces on the one collision that would actually lose data. + Main.RetokenizeIneffectiveExitStatus should not equal 0 + } + // -- the watchdog's decision ------------------------------------------------ private val second = 1000000000L diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala index e24139c99..1b00aafc6 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala @@ -126,8 +126,15 @@ class MappingSpec extends AnyFunSuite with Matchers { } finally Files.deleteIfExists(tmp) } - // A mask change invalidates tree_map: rows assembled under the old mask omit - // every newly selected file, and the trees still look complete afterwards. + // The mask widening (per-language masks -> one universal mask) is exactly the + // change this refusal exists for, so pin it with the real strings rather than + // ".*" vs "\.c$". A project whose mask changes MUST be rebuilt: its tree_map + // rows were assembled from the files the old mask selected, so reusing them + // would produce a repository that silently omits every newly selected file — + // and the omission is invisible afterwards, because the trees look complete. + // + // If someone later "fixes" this into a warning to make a resume work, this test + // is what says no. The rows below are the ones that would be reused. test("a widened mask is refused on resume, not silently reused") { val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" @@ -147,7 +154,8 @@ class MappingSpec extends AnyFunSuite with Matchers { ex.getMessage should include(oldMask) ex.getMessage should include(newMask) - // Same mask still resumes. + // Same mask still resumes, and the rows are still there — the refusal is + // about the mask changing, not about resuming at all. val m2 = Mapping.open(tmp, "/bin/cat", oldMask) try { m2.getTree("origtree") shouldBe Some("newtree") @@ -157,7 +165,9 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the warm-DB fallback refuses a widened mask too") { - // shard_build.sh passes a prior run's DB as --warm; openWarm has its own check. + // shard_build.sh passes a prior run's DB as --warm. Its blob ids were minted + // under that run's mask, so a mask change makes them foreign; openWarm has + // its own check and it must not drift from checkOrSetMeta's. val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" val warm = Files.createTempFile("mapping-warm-", ".db") @@ -177,6 +187,290 @@ class MappingSpec extends AnyFunSuite with Matchers { } } + // -- --mask-widened --------------------------------------------------------- + // + // The opt-in beside the refusal above. Everything here exists because reusing + // blob_map across a widening is only safe under conditions that have to be + // CHECKED, not assumed, and one of those conditions is not obvious: blob_map + // holds identity rows for blobs the old mask did NOT select, and a wider mask + // selects some of them. Serving one of those as a cache hit puts raw source + // into the tokenized repository, which is the one outcome that silently + // corrupts the dataset's meaning. + + private val OldCMask = """\.[ch]$""" + private val OldCppMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" + private val UniversalMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" + + /** A widening that says every id resolves, collecting the report. */ + private def widening(resolves: String => Boolean = _ => true) = { + val log = new StringBuilder + (Mapping.MaskWidening( + newBlobResolves = resolves, + report = m => { log.append(m); log.append('\n'); () }, + nowEpochSeconds = () => 1700000000L + ), log) + } + + /** A blob map as a real run leaves it: tokenized rows for the paths the mask + * selected, identity rows for every other blob the walker passed through, plus + * trees, a commit and a ref. */ + private def seedUnderOldMask(tmp: java.nio.file.Path, mask: String): Unit = { + val m = Mapping.open(tmp, "/bin/cat", mask) + try { + m.putBlob("b1", "src/a.c", "tok1") + m.putBlob("b2", "src/b.h", "tok2") + // Identity rows. These are the dangerous ones: under the universal mask + // a.cpp and T.java ARE selected, so a reused row would serve raw source. + m.putBlob("b3", "src/a.cpp", "b3") + m.putBlob("b4", "src/T.java", "b4") + // And these are paths no mask will ever select. + m.putBlob("b5", "README.md", "b5") + m.putBlob("b6", "CMakeLists.txt", "b6") + m.putTree("t1", "nt1") + m.putCommit("c1", "nc1") + m.putRef(Mapping.RefRow("refs/heads/main", "head", "c1", "nc1", "c1", "nc1")) + } finally m.close() + } + + test("without the flag a mask change is still refused: the default does not move") { + val tmp = Files.createTempFile("mapping-widen-default-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val ex = intercept[Mapping.MetaMismatchException] { + Mapping.open(tmp, "/bin/cat", UniversalMask) + } + ex.getMessage should include("mask") + // ...and it changed nothing, so the flag can still be used afterwards. + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + m.getMeta("mask") shouldBe Some(OldCMask) + m.getTree("t1") shouldBe Some("nt1") + m.getCommit("c1") shouldBe Some("nc1") + m.getBlob("b3", "src/a.cpp") shouldBe Some("b3") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("with the flag: tokenized rows are kept, identity rows and trees and commits go") { + val tmp = Files.createTempFile("mapping-widen-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, log) = widening() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + try { + // Kept: the real tokenizations. + m.getBlob("b1", "src/a.c") shouldBe Some("tok1") + m.getBlob("b2", "src/b.h") shouldBe Some("tok2") + // Gone: every identity row, so the newly selected paths are cache MISSES + // and get tokenized instead of passing through as raw source. + m.getBlob("b3", "src/a.cpp") shouldBe None + m.getBlob("b4", "src/T.java") shouldBe None + m.getBlob("b5", "README.md") shouldBe None + m.getBlob("b6", "CMakeLists.txt") shouldBe None + // Gone: trees, commits and the refs that name them. + m.getTree("t1") shouldBe None + m.getCommit("c1") shouldBe None + m.getRef("refs/heads/main") shouldBe None + m.allCommitOrigShas shouldBe empty + // The stored mask is now the new one, with the old one kept as provenance. + m.getMeta("mask") shouldBe Some(UniversalMask) + m.getMeta(Mapping.MaskWidenedFromKey) shouldBe Some(OldCMask) + m.getMeta(Mapping.MaskWidenedAtKey) shouldBe Some("1700000000") + } finally m.close() + log.toString should include("KEPT 2 tokenized blob_map rows") + log.toString should include("DROPPED 4 identity blob_map rows") + log.toString should include("DROPPED 1 tree_map rows") + } finally Files.deleteIfExists(tmp) + } + + test("after a widening the same run resumes normally, with no flag needed") { + val tmp = Files.createTempFile("mapping-widen-again-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, _) = widening() + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)).close() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask) + try m.getBlob("b1", "src/a.c") shouldBe Some("tok1") finally m.close() + } finally Files.deleteIfExists(tmp) + } + + // THE check. A narrowing is refused on the evidence of the rows, and the + // databases are left exactly as they were — the operator can still resume + // under the old mask afterwards, which is the proof that nothing was touched. + test("a NARROWING mask is refused, names the path, and leaves the databases untouched") { + val tmp = Files.createTempFile("mapping-narrow-", ".db") + try { + // Tokenized under the C++ mask, so src/a.cpp is a real tokenization. + val seed = Mapping.open(tmp, "/bin/cat", OldCppMask) + try { + seed.putBlob("b1", "src/a.c", "tok1") + seed.putBlob("b2", "src/a.cpp", "tok2") + seed.putBlob("b3", "README.md", "b3") + seed.putTree("t1", "nt1") + seed.putCommit("c1", "nc1") + seed.putRef(Mapping.RefRow("refs/heads/main", "head", "c1", "nc1", "c1", "nc1")) + } finally seed.close() + + val (w, log) = widening() + val ex = intercept[Mapping.MaskNarrowedException] { + // C only. It drops .cpp, so it is a narrowing wearing the flag's name. + Mapping.open(tmp, "/bin/cat", OldCMask, None, Some(w)) + } + ex.getMessage should include("src/a.cpp") + ex.getMessage should include(OldCMask) + ex.getMessage should include(OldCppMask) + ex.getMessage should include("NARROWING") + log.toString shouldBe "" + + // Untouched: every row and the recorded mask. + val m = Mapping.open(tmp, "/bin/cat", OldCppMask) + try { + m.getMeta("mask") shouldBe Some(OldCppMask) + m.getMeta(Mapping.MaskWidenedFromKey) shouldBe None + m.getBlob("b1", "src/a.c") shouldBe Some("tok1") + m.getBlob("b2", "src/a.cpp") shouldBe Some("tok2") + m.getBlob("b3", "README.md") shouldBe Some("b3") + m.getTree("t1") shouldBe Some("nt1") + m.getCommit("c1") shouldBe Some("nc1") + m.getRef("refs/heads/main").map(_.newCommit) shouldBe Some("nc1") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + // An identity row on a path no mask selects must NOT be read as a narrowing. + // Every blob map on this machine holds such rows (README.md, CMakeLists.txt, + // .pdf), so a check over all rows rather than tokenized ones refuses every + // project and the feature never fires once. + test("identity rows on never-selected paths do not count as a narrowing") { + val tmp = Files.createTempFile("mapping-identity-ok-", ".db") + try { + val seed = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + seed.putBlob("b1", "src/a.c", "tok1") + seed.putBlob("b5", "README.md", "b5") + seed.putBlob("b6", "docs/manual.pdf", "b6") + seed.putBlob("b7", "Makefile", "b7") + } finally seed.close() + val (w, _) = widening() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + try m.getMeta("mask") shouldBe Some(UniversalMask) finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the narrowing check reads the basename, the way Walker matches the mask") { + // Walker matches against tw.getNameString, not the full path. Replicate it, + // or the check answers a question the walker never asks. + Mapping.basename("src/kernel/a.c") shouldEqual "a.c" + Mapping.basename("a.c") shouldEqual "a.c" + Mapping.basename("dir.c/plain") shouldEqual "plain" + } + + test("a new_blob missing from dst is refused, and nothing is changed") { + val tmp = Files.createTempFile("mapping-dangling-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, log) = widening(resolves = _ => false) + val ex = intercept[Mapping.DanglingNewBlobException] { + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + } + ex.getMessage should include("does not exist in the destination repository") + ex.getMessage should include("step 2") + log.toString shouldBe "" + + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + m.getMeta("mask") shouldBe Some(OldCMask) + m.getTree("t1") shouldBe Some("nt1") + m.getBlob("b3", "src/a.cpp") shouldBe Some("b3") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the reachability check probes only retained rows, never identity ones") { + // An identity row's new_blob is the ORIGINAL blob id, which lives in src and + // need not be in dst at all until tree assembly copies it. Probing one would + // refuse a perfectly good resume. + val tmp = Files.createTempFile("mapping-probe-set-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val probed = scala.collection.mutable.Set.empty[String] + val (w, _) = widening(resolves = id => { probed += id; true }) + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)).close() + probed shouldEqual Set("tok1", "tok2") + } finally Files.deleteIfExists(tmp) + } + + test("the command is still checked under the flag: a different tokenizer is refused") { + // The widening argument is about WHICH blobs, and says nothing about a + // different program producing different tokens from the same bytes. + val tmp = Files.createTempFile("mapping-widen-cmd-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, _) = widening() + val ex = intercept[Mapping.MetaMismatchException] { + Mapping.open(tmp, "/bin/echo", UniversalMask, None, Some(w)) + } + ex.getMessage should include("command") + } finally Files.deleteIfExists(tmp) + } + + test("the flag is a no-op on a fresh blob map and on an unchanged mask") { + val fresh = Files.createTempFile("mapping-widen-fresh-", ".db") + Files.deleteIfExists(fresh) + try { + val (w1, log1) = widening() + val m1 = Mapping.open(fresh, "/bin/cat", UniversalMask, None, Some(w1)) + try m1.getMeta("mask") shouldBe Some(UniversalMask) finally m1.close() + log1.toString should include("nothing to reuse") + log1.toString should include("nothing to check") + + val (w2, log2) = widening() + val m2 = Mapping.open(fresh, "/bin/cat", UniversalMask, None, Some(w2)) + try m2.getMeta(Mapping.MaskWidenedFromKey) shouldBe None finally m2.close() + log2.toString should include("ordinary resume") + } finally Files.deleteIfExists(fresh) + } + + test("rowCounts separates tokenized rows from identity rows") { + withMapping() { m => + m.putBlob("b1", "a.c", "tok1") + m.putBlob("b2", "b.c", "tok2") + m.putBlob("b3", "README.md", "b3") + m.putTree("t1", "nt1") + m.putCommit("c1", "nc1") + val c = m.rowCounts + c.blobTokenized shouldEqual 2L + c.blobIdentity shouldEqual 1L + c.tree shouldEqual 1L + c.commit shouldEqual 1L + c.ref shouldEqual 0L + } + } + + test("the sample is spread across the table, bounded, and deterministic") { + val tmp = Files.createTempFile("mapping-sample-", ".db") + try { + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + (0 until 100).foreach(i => m.putBlob(f"orig$i%03d", f"f$i%03d.c", f"tok$i%03d")) + m.sampleTokenizedNewBlobs(10) should have size 10 + // Spread, not the first ten: a head sample only sees the first commit. + m.sampleTokenizedNewBlobs(10).map(_._2) shouldEqual + m.sampleTokenizedNewBlobs(10).map(_._2) + m.sampleTokenizedNewBlobs(10).map(_._2).toSet should contain("tok090") + // Asking for more than exists returns what exists. + m.sampleTokenizedNewBlobs(1000) should have size 100 + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the sample of an all-identity blob map is empty rather than an error") { + withMapping() { m => + m.putBlob("b1", "README.md", "b1") + m.sampleTokenizedNewBlobs(16) shouldBe empty + } + } + test("allCommitOrigShas returns all rows") { withMapping() { m => m.putCommit("a", "x") diff --git a/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala new file mode 100644 index 000000000..34992c000 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala @@ -0,0 +1,482 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} + +/** Tokenizer-identity recording, the refusal it makes possible, and the + * `--retokenize` invalidation, at the level of the blob map itself. + * + * The defect these exist for: `Mapping.open` decided reuse on `command` and + * `mask`, and `command` is the constant path `tokenizeByBlobId/tokenBySha.pl`. + * Correcting the Rust tokenizer (729643e) therefore changed nothing either + * check looks at, every cached `.rs` row stayed a hit, and a run with the fixed + * binary reproduced the broken tokens. */ +class RetokenizeSpec extends AnyFunSuite with Matchers { + + private val cmd = "/bin/cat" + private val mask = """(?i)\.(c|h|rs)$""" + + private val rustV1 = "1111111111111111" + private val rustV2 = "2222222222222222" + private val cV1 = "aaaaaaaaaaaaaaaa" + private val cV2 = "bbbbbbbbbbbbbbbb" + + private def id(pairs: (String, String)*) = TokenizerIdentity(pairs.toMap) + + private def withDb[A](body: Path => A): A = { + val d = Files.createTempDirectory("retok-") + try body(d.resolve("map.sqlite")) + finally { + def rm(p: Path): Unit = { + if (Files.isDirectory(p)) { val s = Files.list(p); try s.forEach(rm) finally s.close() } + Files.deleteIfExists(p) + () + } + rm(d) + } + } + + /** A blob map that looks like a finished run: two tokenized `.rs` rows, two + * tokenized `.c` rows, one identity (pass-through) row, and the tree, commit + * and ref rows above them. */ + private def seed(db: Path, identity: TokenizerIdentity): Unit = { + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = identity) + try { + m.putBlob("rsblob1", "src/one.rs", "newrs1") + m.putBlob("rsblob2", "src/two.rs", "newrs2") + m.putBlob("cblob1", "src/one.c", "newc1") + m.putBlob("cblob2", "lib/two.h", "newc2") + m.putBlob("mdblob", "README.md", "mdblob") // identity row: not selected + m.putTree("tree1", "newtree1") + m.putCommit("commit1", "newcommit1") + m.putRef(Mapping.RefRow("refs/heads/master", "head", "commit1", "newcommit1", "commit1", "newcommit1")) + } finally m.close() + } + + private def resolvesEverything: String => Boolean = _ => true + + private def retokenize( + exts: Set[String], + purge: Vector[String] => TokenizerMemo.PurgeReport = + blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L), + resolves: String => Boolean = resolvesEverything, + report: String => Unit = _ => (), + memoDirIsTokenizerMemo: Boolean = false + ): Mapping.Retokenize = + Mapping.Retokenize( + extensions = exts, + newBlobResolves = resolves, + purgeMemo = purge, + report = report, + memoDirIsTokenizerMemo = memoDirIsTokenizerMemo, + nowEpochSeconds = () => 1700000000L) + + private val emptyMemo: Vector[String] => TokenizerMemo.PurgeReport = + blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, 0L, blobs.size.toLong, 0L) + + // -- recording and refusing ------------------------------------------------ + + test("an identity absent from meta is recorded, so the first run establishes it") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1, "c" -> cV1)) + try { + m.storedTokenizerId("rs") shouldBe Some(rustV1) + m.storedTokenizerId("c") shouldBe Some(cV1) + } finally m.close() + } + } + + test("an unchanged identity is an ordinary resume and changes nothing") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.rowCounts.blobTokenized shouldEqual 4L + finally m.close() + } + } + + test("a CHANGED identity refuses the run; this is the bug, and the refusal is the fix") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val ex = intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1)) + } + // An operator has to be able to act on it without reading this source. + ex.getMessage should include("rs") + ex.getMessage should include(rustV1) + ex.getMessage should include(rustV2) + ex.getMessage should include("--retokenize=rs") + // The extension whose tokenizer did NOT change must not be named, or the + // hint tells the operator to throw away work that is still valid. + ex.getMessage should not include "--retokenize=c,rs" + } + } + + test("a refusal writes nothing: the map is exactly as it was") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV2)) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("no identity passed at all leaves behaviour exactly as it was") { + // The 186 already-published projects must not become refusals or + // invalidations because this mechanism now exists. + withDb { db => + seed(db, TokenizerIdentity.empty) + val m = Mapping.open(db, cmd, mask) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.storedTokenizerId("rs") shouldBe None + } finally m.close() + } + } + + test("a changed identity for an extension with no cached rows is recorded, not refused") { + // The pipeline reports one identity per extension the checkout can parse, for + // every project, because the identity describes the checkout and not the + // project. A Java-only project carries a `c` identity it has no rows for. + // Refusing it when the C toolchain moves would be a refusal with no remedy: + // --retokenize=c would then fail too, correctly, for having nothing to + // invalidate. Nothing can be served from a cache with no entries. + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "9999999999999999")) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "8888888888888888")) + try { + m.storedTokenizerId("java") shouldBe Some("8888888888888888") + m.rowCounts.blobTokenized shouldEqual 4L + } finally m.close() + } + } + + test("a changed identity IS refused as soon as one cached row carries the extension") { + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "9999999999999999")) + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m0.putBlob("javablob", "src/One.java", "newjava") finally m0.close() + intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "8888888888888888")) + } + } + } + + // -- the invalidation ------------------------------------------------------ + + test("--retokenize drops only the named extension's tokenized rows") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + var reported = "" + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), report = s => reported = s))) + try { + // The assertion the whole flag exists for. + m.getBlob("rsblob1", "src/one.rs") shouldBe None + m.getBlob("rsblob2", "src/two.rs") shouldBe None + // And the assertion that makes it worth having rather than --drop-memo: + // the C tokenizations, which are 88% of the pipeline's time, are intact. + m.getBlob("cblob1", "src/one.c") shouldBe Some("newc1") + m.getBlob("cblob2", "lib/two.h") shouldBe Some("newc2") + m.storedTokenizerId("rs") shouldBe Some(rustV2) + m.storedTokenizerId("c") shouldBe Some(cV1) + reported should include("rs") + } finally m.close() + } + } + + test("--retokenize drops tree_map, commit_map and ref_map, because trees name blobs") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try { + // A retained tree_map row would short-circuit the re-walk of the subtree + // holding the .rs file, and the tree would keep pointing at the old + // token blob — a cache invalidation that invalidates nothing. + m.getTree("tree1") shouldBe None + m.getCommit("commit1") shouldBe None + m.getRef("refs/heads/master") shouldBe None + } finally m.close() + } + } + + test("--retokenize keeps the identity pass-through rows, which are not tokenizations") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try m.getBlob("mdblob", "README.md") shouldBe Some("mdblob") + finally m.close() + } + } + + test("--retokenize purges the memo for exactly the invalidated blobs, before touching the map") { + withDb { db => + seed(db, id("rs" -> rustV1)) + var purged: Vector[String] = Vector.empty + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), purge = { blobs => + purged = blobs + TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L) + }))) + try { + purged.sorted shouldEqual Vector("rsblob1", "rsblob2") + // The memo is keyed on sha1(contents) with no tokenizer in the key + // (tokenBySha.pl:76), so a dropped blob_map row alone would be answered + // by the memo with the same stale tokens. + purged should not contain "cblob1" + } finally m.close() + } + } + + test("a memo purge that fails aborts before the map is changed") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[RuntimeException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), + purge = _ => throw new RuntimeException("memo is read-only")))) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("a memo that held none of the affected blobs refuses: the --memo-dir is wrong") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val ex = intercept[Mapping.NothingInvalidatedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), + purge = blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, 0L, blobs.size.toLong, 0L)))) + } + ex.getMessage should include("--memo-dir") + ex.getMessage should include("$BFG_MEMO_DIR") + // And it changed nothing. + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + finally m.close() + } + } + + test("an empty memo is accepted when --memo-dir is the memo the tokenizer reads") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val reports = Vector.newBuilder[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), purge = emptyMemo, report = reports += _, + memoDirIsTokenizerMemo = true))) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe None + m.getBlob("rsblob2", "src/two.rs") shouldBe None + m.getBlob("cblob1", "src/one.c") shouldBe Some("newc1") + m.storedTokenizerId("rs") shouldBe Some(rustV2) + val log = reports.result().mkString("\n") + log should include("the memo held none of the 2 affected blob(s)") + log should include("Accepted, because --memo-dir is $BFG_MEMO_DIR") + log should include("DROPPED 2 tokenized blob_map rows") + } finally m.close() + } + } + + test("a memo that held entries is purged without the empty-memo notice") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val reports = Vector.newBuilder[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), report = reports += _, memoDirIsTokenizerMemo = true))) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe None + reports.result().mkString("\n") should not include "Accepted, because" + } finally m.close() + } + } + + // -- the ways it must refuse rather than no-op ----------------------------- + + test("--retokenize on an extension with no tokenized rows refuses, it does not exit quietly") { + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "cccccccccccccccc")) + val ex = intercept[Mapping.NothingInvalidatedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "dddddddddddddddd"), + retokenize = Some(retokenize(Set("java")))) + } + ex.getMessage should include("java") + // Nothing was dropped on the way to discovering it. + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.getTree("tree1") shouldBe Some("newtree1") + } finally m.close() + } + } + + test("--retokenize that does not cover every changed tokenizer refuses first, changing nothing") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val ex = intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV2), + retokenize = Some(retokenize(Set("rs")))) + } + ex.getMessage should include("c") + // If this had invalidated .rs and then refused, the map would be in a state + // no flag describes: .rs redone under a new tokenizer, .c stale, and the + // recorded identity a mixture. Half a cache invalidation is a corruption. + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1, "c" -> cV1)) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("naming every changed tokenizer at once is accepted") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1, "h" -> cV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV2, "h" -> cV2), + retokenize = Some(retokenize(Set("rs", "c", "h")))) + try { + m.rowCounts.blobTokenized shouldEqual 0L + m.storedTokenizerId("rs") shouldBe Some(rustV2) + m.storedTokenizerId("c") shouldBe Some(cV2) + } finally m.close() + } + } + + test("--retokenize refuses when a retained new_blob does not resolve in dst") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + // blob_map's new_blob ids live in dst and nowhere else. Reusing the .c rows + // beside a dst that was wiped leaves every retained reference dangling. + intercept[Mapping.DanglingNewBlobException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), resolves = _ => false))) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + finally m.close() + } + } + + test("the reachability probe asks about the rows that are KEPT, never the ones dropped") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + var asked = Vector.empty[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), resolves = { b => asked :+= b; true }))) + try { + asked should contain allOf ("newc1", "newc2") + // Probing a dropped row would refuse a correct invalidation whenever the + // old token blobs had already been garbage-collected out of dst. + asked should not contain "newrs1" + } finally m.close() + } + } + + // -- selection details ----------------------------------------------------- + + test("extensions are matched case-insensitively, as the mask is") { + withDb { db => + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m0.putBlob("upper", "src/Shouty.RS", "newupper") finally m0.close() + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try m.getBlob("upper", "src/Shouty.RS") shouldBe None + finally m.close() + } + } + + test("a longer extension is not matched by a shorter one's suffix") { + withDb { db => + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("c" -> cV1)) + try { + m0.putBlob("ccblob", "src/x.cc", "newcc") + m0.putBlob("cppblob", "src/x.cpp", "newcpp") + m0.putBlob("cblob", "src/x.c", "newc") + } finally m0.close() + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("c" -> cV2), + retokenize = Some(retokenize(Set("c")))) + try { + m.getBlob("cblob", "src/x.c") shouldBe None + // `.cc` and `.cpp` end in neither `.c`; a naive LIKE '%c' would take both. + m.getBlob("ccblob", "src/x.cc") shouldBe Some("newcc") + m.getBlob("cppblob", "src/x.cpp") shouldBe Some("newcpp") + } finally m.close() + } + } + + test("the invalidation is recorded in meta, so a reused map says where it came from") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try { + m.getMeta(Mapping.RetokenizedAtKey) shouldBe Some("1700000000") + m.getMeta(Mapping.RetokenizedExtensionsKey) shouldBe Some("rs") + m.getMeta(Mapping.RetokenizedFromKey) shouldBe Some(s"rs=$rustV1") + } finally m.close() + } + } + + test("--mask-widened and --retokenize together are refused, not composed") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, """(?i)\.(c|h|rs|java)$""", + maskWidening = Some(Mapping.MaskWidening(newBlobResolves = _ => true, report = _ => ())), + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + } + } + } + + test("--retokenize without an identity to record is refused") { + withDb { db => + seed(db, TokenizerIdentity.empty) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, mask, retokenize = Some(retokenize(Set("rs")))) + } + } + } + + test("--retokenize naming an extension the identity does not cover is refused") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs", "java")))) + } + } + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala b/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala deleted file mode 100644 index a32e445c5..000000000 --- a/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala +++ /dev/null @@ -1,179 +0,0 @@ -package cregit.blobexec - -import org.eclipse.jgit.api.Git -import org.eclipse.jgit.internal.storage.file.FileRepository -import org.eclipse.jgit.lib.{ObjectId, Repository} -import org.eclipse.jgit.revwalk.RevWalk -import org.eclipse.jgit.storage.file.FileRepositoryBuilder -import org.eclipse.jgit.treewalk.TreeWalk -import org.scalatest.BeforeAndAfterAll -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers - -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Path} - -/** - * A blob whose tokenizer was killed on its budget must be *retryable*: the run - * reports it and refuses to publish, but simply running again retries the blob - * with no surgery on the mapping database. - * - * Three writes would each hide such a blob from the next run: its `blob_map` row, - * any `tree_map` row above it, and its commit's `commit_map` row. All three are - * asserted absent, and the commit's successful tokenizations asserted present. - */ -class TimeoutRetrySpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { - - private val workRoot: Path = Files.createTempDirectory("timeout-retry-") - - override def afterAll(): Unit = deleteRecursive(workRoot) - - /** The tokenizer under test: it hangs on `b.c` only while `hangMarker` exists, - * so both runs can use the identical command path (the memo's `meta` table - * refuses a command change, exactly as in production). */ - private def tokenizer(dir: Path, hangMarker: Path): String = { - val f = dir.resolve("tok.sh") - Files.writeString( - f, - s"""|#!/bin/sh - |if [ "$$BFG_FILENAME" = "b.c" ] && [ -e "${hangMarker.toAbsolutePath}" ]; then - | sleep 300 & sleep 300 - |else - | tr a-z A-Z - |fi - |""".stripMargin - ) - f.toFile.setExecutable(true) - f.toAbsolutePath.toString - } - - private def run( - src: Repository, - dstPath: Path, - dbPath: Path, - command: String, - mode: String - ): WalkStats = { - val dstExisted = Files.isDirectory(dstPath) - val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] - if (!dstExisted) dst.create(true) - val mapping = Mapping.open(dbPath, command, """\.c$""") - try - new Walker( - src, dst, mapping, """\.c$""".r, command, - abortOnError = false, - parallelism = 2, - pipeline = mode == "pipeline", - pipelineTrees = mode == "pipeline-trees", - destinationMayContainObjects = dstExisted, - blobTimeoutSeconds = 1 - ).run() - finally { - mapping.close() - dst.close() - } - } - - private def withMapping[A](dbPath: Path, command: String)(body: Mapping => A): A = { - val m = Mapping.open(dbPath, command, """\.c$""") - try body(m) finally m.close() - } - - // Each walker persists a commit its own way, so each must suppress all three. - for (mode <- Seq("serial", "pipeline", "pipeline-trees")) - test(s"[$mode] a timed-out blob is retried by simply running again, and good work is kept") { - val dir = Files.createTempDirectory(workRoot, "fixture-") - val srcDir = dir.resolve("src") - Files.createDirectories(srcDir) - val git = Git.init().setDirectory(srcDir.toFile).setBare(false).call() - Files.writeString(srcDir.resolve("a.c"), "alpha\n") - Files.createDirectories(srcDir.resolve("deep")) - Files.writeString(srcDir.resolve("deep/b.c"), "beta\n") - git.add().addFilepattern("a.c").call() - git.add().addFilepattern("deep/b.c").call() - val commit = git.commit() - .setMessage("two blobs, one of which will wedge") - .setAuthor("Tester", "t@example.org") - .setCommitter("Tester", "t@example.org") - .call() - - val src = git.getRepository - val commitSha = commit.getId.name - val topTreeSha = commit.getTree.getId.name - val aSha = blobIdAt(src, commit.getId, "a.c").name - val bSha = blobIdAt(src, commit.getId, "deep/b.c").name - val subTreeSha = blobIdAt(src, commit.getId, "deep").name - - val hangMarker = dir.resolve("HANG") - Files.writeString(hangMarker, "hang b.c\n") - val command = tokenizer(dir, hangMarker) - val dstPath = dir.resolve("dst.git") - val dbPath = dir.resolve("blobmap.db") - - // -- run 1: b.c wedges ----------------------------------------------------- - val first = run(src, dstPath, dbPath, command, mode) - - first.blobsTimedOut shouldEqual 1 - first.aborted shouldBe false // a timeout is not an abort - first.commitsProcessed shouldEqual 0 // the containing commit was not folded - - withMapping(dbPath, command) { m => - // The three writes that would each hide the retry, all absent: - m.getBlob(bSha, "deep/b.c") shouldEqual None - m.getTree(subTreeSha) shouldEqual None - m.getTree(topTreeSha) shouldEqual None - m.getCommit(commitSha) shouldEqual None - // ...and the work that was genuinely correct, kept: - m.getBlob(aSha, "a.c") shouldBe defined - m.getBlob(aSha, "a.c").get should not equal aSha // tokenized, not raw - } - - // -- run 2: same command, nothing cleared by hand -------------------------- - Files.delete(hangMarker) - val second = run(src, dstPath, dbPath, command, mode) - - second.blobsTimedOut shouldEqual 0 - second.aborted shouldBe false - second.commitsProcessed shouldEqual 1 - // a.c came from the memo rather than being re-tokenized: the retry costs one - // blob, not the whole commit. - second.blobsCacheHit should be >= 1 - - withMapping(dbPath, command) { m => - m.getCommit(commitSha) shouldBe defined - m.getTree(topTreeSha) shouldBe defined - val bNew = m.getBlob(bSha, "deep/b.c") - bNew shouldBe defined - bNew.get should not equal bSha // the retry tokenized it - readBlob(dstPath, bNew.get) shouldEqual "BETA\n" - } - } - - // -- helpers --------------------------------------------------------------- - - private def blobIdAt(repo: Repository, commitId: ObjectId, path: String): ObjectId = { - val rw = new RevWalk(repo) - try { - val tree = rw.parseCommit(commitId).getTree - val tw = TreeWalk.forPath(repo, path, tree) - try tw.getObjectId(0) finally tw.close() - } finally rw.close() - } - - private def readBlob(dstPath: Path, sha: String): String = { - val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] - try { - val r = dst.newObjectReader() - try new String(r.open(ObjectId.fromString(sha)).getBytes, UTF_8) finally r.close() - } finally dst.close() - } - - private def deleteRecursive(p: Path): Unit = { - if (Files.isDirectory(p)) { - val s = Files.list(p) - try s.forEach(deleteRecursive) finally s.close() - } - Files.deleteIfExists(p) - () - } -} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala new file mode 100644 index 000000000..51b0e5d08 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala @@ -0,0 +1,69 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class TokenizerIdentitySpec extends AnyFunSuite with Matchers { + + test("one extension round-trips") { + TokenizerIdentity.parse("rs=deadbeefcafe") shouldBe + Right(TokenizerIdentity(Map("rs" -> "deadbeefcafe"))) + } + + test("several extensions round-trip, and render is sorted and stable") { + val id = TokenizerIdentity.parse("h=aaaaaaaa,c=bbbbbbbb,rs=cccccccc").toOption.get + id.extensions shouldBe Set("c", "h", "rs") + id.render shouldEqual "c=bbbbbbbb,h=aaaaaaaa,rs=cccccccc" + // A rendering that depended on map order would make every log line and every + // refusal message a different string for the same fact. + TokenizerIdentity.parse("c=bbbbbbbb,rs=cccccccc,h=aaaaaaaa").toOption.get.render shouldEqual id.render + } + + test("c++ and h++ are extensions, because CregitLanguages.pm says so") { + val id = TokenizerIdentity.parse("c++=aaaaaaaa,h++=bbbbbbbb").toOption.get + id.extensions shouldBe Set("c++", "h++") + } + + test("an empty spec is refused, not read as 'no extensions'") { + // "no extensions" would make --tokenizer-identity a silent no-op, which is + // the whole failure mode this mechanism exists to remove. + TokenizerIdentity.parse("").isLeft shouldBe true + TokenizerIdentity.parse(" ").isLeft shouldBe true + } + + test("a leading dot is refused: the tables spell extensions without one") { + TokenizerIdentity.parse(".rs=deadbeefcafe").isLeft shouldBe true + } + + test("an uppercase extension is refused rather than silently lowercased") { + TokenizerIdentity.parse("RS=deadbeefcafe").isLeft shouldBe true + } + + test("a value that is not a long-enough hex digest is refused") { + TokenizerIdentity.parse("rs=zzzz").isLeft shouldBe true + TokenizerIdentity.parse("rs=ab").isLeft shouldBe true + TokenizerIdentity.parse("rs=").isLeft shouldBe true + TokenizerIdentity.parse("rs").isLeft shouldBe true + } + + test("a quote or a LIKE wildcard cannot get into an extension") { + // These strings reach a SQL LIKE pattern in Mapping. The alphabet is the + // defence, so it is asserted here rather than trusted there. + Seq("r's", "r%s", "r_s", "rs;", "rs)", "r s").foreach { bad => + TokenizerIdentity.parse(s"$bad=deadbeefcafe").isLeft shouldBe true + TokenizerIdentity.parseExtensions(bad).isLeft shouldBe true + } + } + + test("an extension named twice is refused: one of the two values would win silently") { + TokenizerIdentity.parse("rs=aaaaaaaa,rs=bbbbbbbb").isLeft shouldBe true + } + + test("extension lists for --retokenize parse and reject the same way") { + TokenizerIdentity.parseExtensions("rs") shouldBe Right(Set("rs")) + TokenizerIdentity.parseExtensions("c,h,cpp") shouldBe Right(Set("c", "h", "cpp")) + TokenizerIdentity.parseExtensions("") .isLeft shouldBe true + TokenizerIdentity.parseExtensions(".rs").isLeft shouldBe true + TokenizerIdentity.parseExtensions("c,,h").isLeft shouldBe true + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala new file mode 100644 index 000000000..739ec5d84 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala @@ -0,0 +1,151 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +/** The memo layout has to match tokenizeByBlobId/tokenBySha.pl exactly, and the + * two are written in different languages, so the agreement is asserted rather + * than assumed. A silent disagreement here would look like a successful + * invalidation that purged nothing. */ +class TokenizerMemoSpec extends AnyFunSuite with Matchers { + + private def withTempDir[A](body: Path => A): A = { + val d = Files.createTempDirectory("memo-spec-") + try body(d) + finally { + def rm(p: Path): Unit = { + if (Files.isDirectory(p)) { val s = Files.list(p); try s.forEach(rm) finally s.close() } + Files.deleteIfExists(p) + () + } + rm(d) + } + } + + test("sha1Hex matches a known Digest::SHA value") { + // printf 'int a;\n' | sha1sum + TokenizerMemo.sha1Hex("int a;\n".getBytes(UTF_8)) shouldEqual + "56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + // printf 'fn main() {}\n' | sha1sum + TokenizerMemo.sha1Hex("fn main() {}\n".getBytes(UTF_8)) shouldEqual + "c135f41eadc2f7248d34e03e64a04e8420bf5e4b" + } + + test("the path is /xx/yy/, the fan-out tokenBySha.pl builds") { + val sha = "56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + TokenizerMemo.relativePathFor(sha) shouldEqual + "56/f5/56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + // Both halves come from the FRONT of the sha (substr 0,2 then 2,2), which is + // easy to get wrong as 0,2 then -2. + TokenizerMemo.relativePathFor(sha) should startWith("56/f5/") + } + + test("sha1Hex agrees with perl's Digest::SHA on bytes perl might mangle") { + // The real inputs are arbitrary source bytes, including CRLF and high bytes. + // Perl reads the blob from stdin and hashes it; if either side treated the + // bytes as text the keys would diverge for exactly those files. + val perl = Option(System.getenv("PATH")).toVector + .flatMap(_.split(java.io.File.pathSeparator)) + .map(p => Path.of(p, "perl")) + .find(Files.isExecutable) + assume(perl.isDefined, "perl not on PATH") + + val samples = Vector( + "int a;\n", + "a\r\nb\r\n", + "no trailing newline", + "caf\u00e9 \u00fcber\n", + "" + ) + withTempDir { d => + samples.foreach { s => + val f = d.resolve("input.bin") + Files.write(f, s.getBytes(UTF_8)) + val pb = new ProcessBuilder( + perl.get.toString, "-MDigest::SHA=sha1_hex", "-e", + "local $/; open(F, '<', $ARGV[0]) or die; binmode F; print sha1_hex( // '')", + f.toString) + pb.redirectErrorStream(true) + val proc = pb.start() + val out = new String(proc.getInputStream.readAllBytes(), UTF_8).trim + proc.waitFor() + withClue(s"sample [${s.replace("\n", "\\n").replace("\r", "\\r")}] perl said [$out]: ") { + out shouldEqual TokenizerMemo.sha1Hex(s.getBytes(UTF_8)) + } + } + } + } + + test("purge deletes the entries of the named blobs and leaves the rest alone") { + withTempDir { root => + val contents = Map( + "blob-rs-1" -> "fn one() {}\n", + "blob-rs-2" -> "fn two() {}\n", + "blob-c-1" -> "int c;\n" + ) + contents.values.foreach { s => + val p = TokenizerMemo.entryFor(root, TokenizerMemo.sha1Hex(s.getBytes(UTF_8))) + Files.createDirectories(p.getParent) + Files.writeString(p, "TOKENS FOR " + s) + } + + val report = TokenizerMemo.purge( + root, + Vector("blob-rs-1", "blob-rs-2"), + sha => contents.get(sha).map(_.getBytes(UTF_8))) + + report.examined shouldEqual 2L + report.deleted shouldEqual 2L + report.missing shouldEqual 0L + report.unreadable shouldEqual 0L + + Files.exists(TokenizerMemo.entryFor(root, + TokenizerMemo.sha1Hex(contents("blob-rs-1").getBytes(UTF_8)))) shouldBe false + // The whole point of being selective: the C entry is still a valid + // tokenization and re-doing it would cost 88% of the pipeline for nothing. + Files.exists(TokenizerMemo.entryFor(root, + TokenizerMemo.sha1Hex(contents("blob-c-1").getBytes(UTF_8)))) shouldBe true + } + } + + test("a blob with no memo entry is counted missing, not deleted") { + withTempDir { root => + val report = TokenizerMemo.purge(root, Vector("absent"), _ => Some("x\n".getBytes(UTF_8))) + report.deleted shouldEqual 0L + report.missing shouldEqual 1L + } + } + + test("a blob whose bytes cannot be read is counted, not skipped in silence") { + withTempDir { root => + val report = TokenizerMemo.purge(root, Vector("oversized"), _ => None) + report.examined shouldEqual 1L + report.unreadable shouldEqual 1L + report.deleted shouldEqual 0L + } + } + + test("isTokenizerMemo holds for the same directory, by a symlink or a trailing slash too") { + withTempDir { root => + val memo = Files.createDirectory(root.resolve("memo")) + val link = Files.createSymbolicLink(root.resolve("link"), memo) + TokenizerMemo.isTokenizerMemo(memo, Some(memo.toString)) shouldBe true + TokenizerMemo.isTokenizerMemo(memo, Some(memo.toString + "/")) shouldBe true + TokenizerMemo.isTokenizerMemo(link, Some(memo.toString)) shouldBe true + } + } + + test("isTokenizerMemo is false for another directory, an unset or empty variable, or a missing path") { + withTempDir { root => + val memo = Files.createDirectory(root.resolve("memo")) + val other = Files.createDirectory(root.resolve("other")) + TokenizerMemo.isTokenizerMemo(memo, Some(other.toString)) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, None) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, Some("")) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, Some(root.resolve("gone").toString)) shouldBe false + } + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala new file mode 100644 index 000000000..4921b3fbb --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala @@ -0,0 +1,136 @@ +package cregit.blobexec + +import org.eclipse.jgit.internal.storage.dfs.{DfsRepositoryDescription, InMemoryRepository} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Paths} +import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val tmpDir = Files.createTempDirectory("tokenizer-worker-pool-") + private val worker = tmpDir.resolve("fake-worker.pl") + private val grandchildPidFile = Paths.get(s"$worker.grandchild") + + override def beforeAll(): Unit = { + Files.writeString( + worker, + """|#!/usr/bin/env perl + |use strict; + |use warnings; + |$| = 1; + |sub reply { my ($exit, $out) = @_; print "RES $exit ", length($out), " 0\n", $out } + |print "READY\n"; + |while (my $header = ) { + | my (undef, $bodyLength, $nameLength, $timeout) = split ' ', $header; + | read(STDIN, my $name, $nameLength); + | read(STDIN, my $body, $bodyLength) if $bodyLength; + | if ($name eq 'hang.c') { + | my $child = fork() // exit 3; + | exec 'sleep', '300' if $child == 0; + | open(my $pids, '>', "$0.grandchild") or exit 3; + | print {$pids} $child; + | close($pids); + | sleep 300; + | } + | exit 7 if $name eq 'die.c'; + | sleep 2 if $name eq 'slow.c'; + | if ($name eq 'crash.c') { reply(33, '') } + | elsif ($name eq 'timeout.c') { reply(124, '') } + | elsif ($name eq 'budget.c') { reply(0, $timeout) } + | else { reply(0, uc $body) } + |} + |""".stripMargin + ) + worker.toFile.setExecutable(true) + } + + override def afterAll(): Unit = + Files.walk(tmpDir).sorted(java.util.Comparator.reverseOrder()).forEach(p => Files.delete(p)) + + private def withPool[A](size: Int)(body: TokenizerWorkerPool => A): A = { + val pool = new TokenizerWorkerPool(worker.toString, size) + try body(pool) finally pool.close() + } + + private def stdout(pool: TokenizerWorkerPool, filename: String, body: String, timeoutSeconds: Int = 2) = { + val (exit, out, _) = pool.invoke(body.getBytes(UTF_8), filename, timeoutSeconds) + exit shouldEqual 0 + new String(out, UTF_8) + } + + test("returns the worker's output and sends each request's own budget") { + withPool(1) { pool => + stdout(pool, "hello.c", "hello") shouldEqual "HELLO" + stdout(pool, "budget.c", "x", timeoutSeconds = 7) shouldEqual "7" + stdout(pool, "budget.c", "x", timeoutSeconds = 21) shouldEqual "21" + } + } + + test("a worker's timeout and parser crash take BlobExec's timeout and crash paths") { + val inserter = new InMemoryRepository(new DfsRepositoryDescription("pool-spec")).newObjectInserter() + withPool(1) { pool => + def run(filename: String): (BlobExec.Outcome, String) = { + var called = "none" + val outcome = BlobExec.run( + "body".getBytes(UTF_8), "a" * 40, filename, s"src/$filename", "tokenBySha.pl", + abortOnError = true, inserter = inserter, + onTimeout = () => called = "timeout", onParserCrash = () => called = "parser-crash", + workerPool = Some(pool)) + (outcome, called) + } + run("timeout.c") shouldEqual ((BlobExec.Outcome.Skip, "timeout")) + run("crash.c") shouldEqual ((BlobExec.Outcome.Skip, "parser-crash")) + run("ok.c")._1 shouldBe a[BlobExec.Outcome.Replace] + } + } + + test("kills a wedged worker with its children, replaces it, and serves the next request") { + withPool(1) { pool => + val started = System.nanoTime() + pool.invoke("x".getBytes(UTF_8), "hang.c", 1)._1 shouldEqual BlobExec.TimeoutExitCode + (System.nanoTime() - started).nanos should be < 15.seconds + eventuallyStopped(Seq(Files.readString(grandchildPidFile).trim.toLong)) shouldBe true + stdout(pool, "next.c", "next") shouldEqual "NEXT" + } + } + + test("a worker that dies in a request fails that blob only") { + withPool(1) { pool => + pool.invoke("x".getBytes(UTF_8), "die.c", 2)._1 should not equal 0 + stdout(pool, "ok.c", "ok") shouldEqual "OK" + } + } + + test("serves pool-sized requests in parallel") { + implicit val ec: ExecutionContext = ExecutionContext.global + withPool(3) { pool => + val started = System.nanoTime() + val requests = (1 to 3).map(n => Future(stdout(pool, "slow.c", s"body$n", timeoutSeconds = 5))) + Await.result(Future.sequence(requests), 10.seconds).toSet shouldEqual Set("BODY1", "BODY2", "BODY3") + (System.nanoTime() - started).nanos should be < 4.seconds + } + } + + test("close stops every worker") { + val pool = new TokenizerWorkerPool(worker.toString, 2) + val pids = ProcessHandle.current().children().iterator().asScala + .filter(_.info().commandLine().orElse("").contains(worker.toString)) + .map(_.pid()).toVector + pids should have size 2 + pool.close() + eventuallyStopped(pids) shouldBe true + } + + private def eventuallyStopped(pids: Seq[Long]): Boolean = { + def alive(pid: Long) = ProcessHandle.of(pid).map[Boolean](_.isAlive).orElse(false) + val deadline = System.nanoTime() + 5.seconds.toNanos + while (System.nanoTime() < deadline && pids.exists(alive)) Thread.sleep(25) + !pids.exists(alive) + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala index 91205d835..26e94fe95 100644 --- a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala @@ -85,11 +85,40 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // Empty by default: a fixture blob must never be excluded because it happens // to collide with the shipped list, and the shipped list must never be // exercised by accident. The denylist tests pass their own. - denylist: BlobDenylist = BlobDenylist.empty + denylist: BlobDenylist = BlobDenylist.empty, + maskWidened: Boolean = false, + // Empty by default so every test written before the tokenizer-identity + // mechanism existed keeps its exact previous behaviour. + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenizeExtensions: Set[String] = Set.empty, + memoDir: Option[Path] = None ): WalkStats = { val destinationMayContainObjects = Files.isDirectory(dstPath) val dst = openBare(dstPath) - val mapping = Mapping.open(dbPath, command, mask) + val resolvesInDst: String => Boolean = id => { + val r = dst.newObjectReader() + try r.has(ObjectId.fromString(id)) finally r.close() + } + val widening = + if (!maskWidened) None + else Some(Mapping.MaskWidening(newBlobResolves = resolvesInDst, report = _ => ())) + val retokenize = + if (retokenizeExtensions.isEmpty) None + else Some(Mapping.Retokenize( + extensions = retokenizeExtensions, + newBlobResolves = resolvesInDst, + purgeMemo = blobs => memoDir match { + case None => TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L) + case Some(root) => TokenizerMemo.purge(root, blobs, sha => { + val r = srcRepo.newObjectReader() + try Some(r.open(ObjectId.fromString(sha), OBJ_BLOB).getBytes) + catch { case _: Exception => None } + finally r.close() + }) + }, + report = _ => () + )) + val mapping = Mapping.open(dbPath, command, mask, None, widening, tokenizerIdentity, retokenize) try { val walker = new Walker( srcRepo, dst, mapping, mask.r, command, abortOnError, parallelism, @@ -640,10 +669,20 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // -- shared dir helpers ---------------------------------------------------- + // -- large pass-through blob (regression) ---------------------------------- // - // Lowering the threshold lets a small blob cross it, so the regression - // reproduces in milliseconds instead of needing a 50 MB fixture. + // redis/redis failed after 1,684s of real work with: + // org.eclipse.jgit.errors.LargeObjectException: 12e1ac54... exceeds size limit + // at cregit.blobexec.Walker.ensureOriginalBlobAvailable + // + // A blob that does not match the mask is copied verbatim, so its size is + // whatever the project committed, not the size of a source file. + // ObjectLoader.getBytes refuses anything above JGit's stream threshold, so the + // copy path must stream instead of materialising. + // + // The threshold is lowered here so a small blob reproduces it in milliseconds + // rather than needing a 50 MB fixture. private def withStreamFileThreshold[T](bytes: Int)(body: => T): T = { val lowered = new WindowCacheConfig lowered.setStreamFileThreshold(bytes) @@ -703,7 +742,33 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- oversized MASK-MATCHED blob (regression) ------------------------------- + // + // qualcomm/qcom-embedded-power-measurement ended a whole project with rc 1: + // org.eclipse.jgit.errors.LargeObjectException: 205b0f65... exceeds size limit + // at cregit.blobexec.Walker.readBlob + // on src/libraries/libexcel/excel.cpp — 102,897,757 bytes, 2,398,232 lines, a + // dumpcpp dump of Microsoft Excel's COM type library. The mask DID select it + // (.cpp), so the streaming fix on the pass-through path does not apply: the + // tokenizer cannot be handed 98 MB, so the blob has to be excluded outright. + + test("the oversized limit is jgit's own threshold, not a constant above it") { + val jgitDefault = new WindowCacheConfig().getStreamFileThreshold.toLong + // Read from jgit, so the two cannot drift. An earlier draft of this fix used + // a hand-picked 64 MiB, which leaves 50-64 MiB passing the size check and + // then throwing inside getBytes anyway. + Walker.MaxBlobBytes shouldEqual jgitDefault + jgitDefault shouldEqual 50L * 1024 * 1024 + Walker.isOversized(200L * 1024 * 1024) shouldBe true + Walker.isOversized(55L * 1024 * 1024) shouldBe true // the band the 64 MiB constant missed + Walker.isOversized(4096L) shouldBe false + } + /** Run the exclusion end to end on `mode`, with jgit's threshold lowered so a + * 78 KB fixture reproduces a 98 MB blob in milliseconds. + * + * The fixture size is the point: 78 KB is far BELOW Walker.MaxBlobBytes, so + * the size fast path cannot see it and only the LargeObjectException catch + * can. That is exactly the band a bare size check gets wrong. */ private def checkOversizedExclusion(label: String, pipeline: Boolean, pipelineTrees: Boolean): Unit = { val dir = freshWorkDir("oversized-masked-" + label) val git = initSrc(dir.resolve("src")) @@ -715,6 +780,9 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte "add a normal source file and a generated one") } finally git.close() + // Proof that the fast path is not what fires here. + Walker.isOversized(generated.length.toLong) shouldBe false + val srcRepo = openBare(dir.resolve("src/.git")) val stats = try { withStreamFileThreshold(4 * 1024) { @@ -763,13 +831,14 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // -- DENYLISTED mask-matched blob ------------------------------------------- // - // srcML 1.1.0 does not terminate on the denylisted blobs: 0 bytes of output, one - // core busy, at any budget. Upstream srcML/srcML#2361 is open with no patch and - // v1.1.0 is the latest release, so there is nothing to upgrade to. The timeout - // path contains the hang but answers it with exit 4, "incomplete, do not - // publish", which holds the project back for a diagnosed third-party defect. + // tencent__tencentkona-21 holds four blobs on which srcML 1.1.0 does not + // terminate: 0 bytes of output, one core at 101%, at every budget from 5s to + // 600s. Upstream srcML/srcML#2361 is open with no patch and v1.1.0 is the latest + // release, so there is nothing to upgrade to. The timeout path contains the hang + // but answers it with exit 4, "incomplete, do not publish", which would leave + // that project permanently unpublishable over a diagnosed third-party defect. // - // So a denylisted blob takes Task 5's EXCLUSION path instead: dropped in + // So a denylisted blob takes the EXCLUSION path instead: dropped in // microseconds, never handed to the tokenizer, counted on its own, and NOT // gating the exit status. A timeout still gates it, because a timeout is a hang // nobody has explained yet. @@ -793,8 +862,9 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } finally git.close() val denied = blobIdOf(hangs) - val denylist = BlobDenylist(Map(denied.name -> - BlobDenylist.Entry("srcML/srcML#2361", "srcML 1.1.0 does not terminate on it"))) + val denylist = BlobDenylist.parse( + Vector(s"${denied.name}\tsrcML/srcML#2361\tsrcML 1.1.0 does not terminate on it"), + "fixture") // Every invocation of the tokenizer records the blob it was given. "Excluded in // milliseconds, never handed to the tokenizer" is only checkable from the @@ -849,7 +919,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } test("a denylisted blob is excluded without running the tokenizer (pipeline)") { - // The mode run_pipeline_process.sh uses for small and medium projects. + // The mode run_pipeline_process.sh actually uses for S/M projects, which is + // what tencent__tencentkona-21 runs as. checkDenylistExclusion("pipeline", pipeline = true, pipelineTrees = false) } @@ -882,6 +953,225 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte finally dst.close() } + // -- --mask-widened, end to end --------------------------------------------- + // + // The one test that matters for the mask widening, because it is the only one + // that asks what ends up in dst. Reusing blob_map has to do two things at once: + // skip the work already done, AND tokenize the files the wider mask newly + // selects. The second is the trap. Those files already have an identity row + // from the first run, saying "not selected, bytes pass through", and a naive + // reuse serves that row as a cache hit — so the file lands in the TOKENIZED + // repository holding RAW SOURCE, and nothing downstream can tell. + + test("a widened mask reuses the tokenized rows and still tokenizes the newly selected ones") { + val w = freshWorkDir("mask-widened") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val cmd = shellScript(w, "tr a-z A-Z") + val narrow = """\.[ch]$""" + val wide = """\.(c|cpp|h)$""" + + val git = initSrc(srcDir) + writeAndCommit(git, Map( + "a.c" -> "int a;\n", + "b.h" -> "int b;\n", + "c.cpp" -> "int c;\n", + "README.md" -> "leave-me-alone\n" + ), "only commit") + + // Run 1, narrow mask: two files tokenized, c.cpp merely passed through. + val first = runWalker(git.getRepository, dstDir, db, cmd, narrow) + first.blobCommandExecutions shouldEqual 2L + locally { + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + // Raw, and correctly so: the narrow mask did not select it. + fileAtHead(dst, "master", "c.cpp") shouldBe Some("int c;\n") + } finally dst.close() + } + + // Run 2, wider mask, same dst and same db, with the opt-in. + val second = runWalker(git.getRepository, dstDir, db, cmd, wide, maskWidened = true) + + // Exactly ONE tokenizer invocation: a.c and b.h came from blob_map, c.cpp is + // new. That ratio is the entire point of reusing the map. + second.blobCommandExecutions shouldEqual 1L + second.blobsCacheHit shouldEqual 2 + + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + fileAtHead(dst, "master", "b.h") shouldBe Some("INT B;\n") + // The assertion this whole task turns on. Under a reused blob_map without + // the identity-row purge this reads "int c;\n" — raw source, in the + // tokenized repository, invisibly. + fileAtHead(dst, "master", "c.cpp") shouldBe Some("INT C;\n") + fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") + } finally dst.close() + } + + test("without the opt-in the same second run refuses, and dst is left alone") { + val w = freshWorkDir("mask-widened-refused") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val cmd = shellScript(w, "tr a-z A-Z") + + val git = initSrc(srcDir) + writeAndCommit(git, Map("a.c" -> "int a;\n", "c.cpp" -> "int c;\n"), "only commit") + runWalker(git.getRepository, dstDir, db, cmd, """\.[ch]$""") + + intercept[Mapping.MetaMismatchException] { + runWalker(git.getRepository, dstDir, db, cmd, """\.(c|cpp|h)$""") + } + + val dst = openBare(dstDir) + try fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") finally dst.close() + } + + // -- tokenizer identity ---------------------------------------------------- + // + // The defect, end to end. blob_map reuse was decided on `command` + `mask`, and + // `command` is the constant path tokenizeByBlobId/tokenBySha.pl. So correcting a + // tokenizer changed nothing either check looks at: every cached row for that + // language stayed a cache hit, and a run with the corrected tokenizer + // reproduced the OLD tokens exactly, with no error anywhere. Measured on the + // corpus for the Rust fix (729643e): 741,869 .rs (blob, path) pairs in 45 + // projects would have been served from the poisoned cache. + // + // These two tests are the deliverable. The first says a changed tokenizer can + // no longer be silent. The second says the opt-in actually re-tokenizes the + // affected files, and ONLY those — re-tokenizing everything costs 88% of total + // pipeline time, so a `.rs`-only fix must not touch the C work. + + test("a changed tokenizer identity refuses the incremental run instead of reusing its tokens") { + val w = freshWorkDir("tokid-refuse") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val mask = """\.(c|rs)$""" + val v1 = shellScript(w, "tr a-z A-Z") + + val git = initSrc(srcDir) + writeAndCommit(git, Map("a.c" -> "int a;\n", "b.rs" -> "fn b() {}\n"), "only commit") + runWalker(git.getRepository, dstDir, db, v1, mask, + tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "1111111111"))) + + // Same content, same mask, same command path — only the Rust tokenizer's + // identity moved. Before this mechanism the run reused every .rs row and said + // nothing. + val ex = intercept[Mapping.TokenizerChangedException] { + runWalker(git.getRepository, dstDir, db, v1, mask, + tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "2222222222"))) + } + ex.getMessage should include("--retokenize=rs") + + val dst = openBare(dstDir) + try fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + finally dst.close() + } + + test("--retokenize on one extension re-tokenizes that extension's files and reuses the rest") { + val w = freshWorkDir("tokid-retokenize") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val memo = w.resolve("memo") + Files.createDirectories(memo) + val mask = """\.(c|rs)$""" + + // ONE command path whose behaviour changes between the two runs, because that + // is exactly the pipeline's situation: is the constant path + // tokenizeByBlobId/tokenBySha.pl, which dispatches by extension to a tokenizer + // that can be rebuilt underneath it. Two different script paths would trip the + // existing `command` meta check and prove nothing — the defect is precisely + // that the command string does NOT change when the tokenizer does. + val cmd = w.resolve("tokenize.sh") + def writeTokenizer(rustBody: String): String = { + Files.writeString(cmd, + s"""#!/bin/sh + |case "$$BFG_FILENAME" in + | *.rs) $rustBody ;; + | *) tr a-z A-Z ;; + |esac + |""".stripMargin) + cmd.toFile.setExecutable(true) + cmd.toAbsolutePath.toString + } + // v1 emits the `line:col` prefix the real defect emitted; v2 does not. + val v1 = writeTokenizer("""sed 's/^/1:1\t/'""") + + val git = initSrc(srcDir) + writeAndCommit(git, Map( + "a.c" -> "int a;\n", + "b.rs" -> "fn b() {}\n", + "c.rs" -> "fn c() {}\n", + "README.md" -> "leave-me-alone\n" + ), "only commit") + + val idV1 = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "1111111111")) + val idV2 = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "2222222222")) + + val first = runWalker(git.getRepository, dstDir, db, v1, mask, tokenizerIdentity = idV1) + first.blobCommandExecutions shouldEqual 3L + locally { + val dst = openBare(dstDir) + try { + // The defect's signature: a line:col prefix the rest of the pipeline does + // not expect, shifting every later field by one. + fileAtHead(dst, "master", "b.rs") shouldBe Some("1:1\tfn b() {}\n") + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + } finally dst.close() + } + + // Plant the memo entries the real tokenizeByBlobId/tokenBySha.pl would have + // written, so the purge has something to purge and the test can assert the + // .c entry survives. + val memoEntries = Map( + "fn b() {}\n" -> "1:1\tfn b() {}\n", + "fn c() {}\n" -> "1:1\tfn c() {}\n", + "int a;\n" -> "INT A;\n" + ).map { case (content, tokens) => + val p = TokenizerMemo.entryFor(memo, TokenizerMemo.sha1Hex(content.getBytes(UTF_8))) + Files.createDirectories(p.getParent) + Files.writeString(p, tokens) + content -> p + } + + // The tokenizer is corrected in place. Same path, same mask, same everything + // the old checks looked at. + val v2 = writeTokenizer("cat") + v2 shouldEqual v1 + + val second = runWalker(git.getRepository, dstDir, db, v2, mask, + tokenizerIdentity = idV2, retokenizeExtensions = Set("rs"), memoDir = Some(memo)) + + // Two invocations, not three: the .rs files were re-tokenized and a.c was not. + // That ratio IS the requirement — a flag that re-tokenized a.c as well would + // cost the 88% this exists to avoid. + second.blobCommandExecutions shouldEqual 2L + second.blobsCacheHit shouldEqual 1 + + val dst = openBare(dstDir) + try { + // The assertion the whole task turns on: the corrected tokenizer's output, + // not the cached wrong one. + fileAtHead(dst, "master", "b.rs") shouldBe Some("fn b() {}\n") + fileAtHead(dst, "master", "c.rs") shouldBe Some("fn c() {}\n") + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") + } finally dst.close() + + // The memo, the second cache layer. Without purging it the walker would have + // re-run the command and tokenBySha.pl would have answered from sha1(contents) + // — the same stale tokens, through a different door. + Files.exists(memoEntries("fn b() {}\n")) shouldBe false + Files.exists(memoEntries("fn c() {}\n")) shouldBe false + Files.exists(memoEntries("int a;\n")) shouldBe true + } + private def deleteRecursive(p: Path): Unit = { if (Files.isDirectory(p)) { val s = Files.list(p) diff --git a/devenv.nix b/devenv.nix index 1c48d0229..f9608171f 100644 --- a/devenv.nix +++ b/devenv.nix @@ -47,7 +47,6 @@ in pkgs.gcc pkgs.sqlite pkgs.git-filter-repo - pkgs.shellcheck srcml pkgs.universal-ctags @@ -66,14 +65,15 @@ in languages.rust.enable = true; env.LEGACY_JAVA_HOME = "${legacyJdk}"; + env.SRCML_PREFIX = "${srcml}"; cachix.pull = [ "ccslcodevcregit" ]; - # `devenv test` gates every tracked shell script. The threshold is `error` - # because the two scripts already carry 31 lower findings; raise it to - # `warning` once those are fixed. + # `devenv test` gates every tracked shell script at `error`. shellcheck is not + # in `packages`: that would change the shell env in srcml2token's RUNPATH, and + # so the tokenizer identity of every C, C++, Java and Rust blob. enterTest = '' - git ls-files -z '*.sh' | xargs -0 --no-run-if-empty shellcheck -S error + git ls-files -z '*.sh' | xargs -0 --no-run-if-empty ${pkgs.shellcheck}/bin/shellcheck -S error echo "shellcheck: no errors" pytest -q prove -r $(git ls-files '*/t/*.t' | xargs -n1 dirname | sort -u) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 25df827c0..621c777a3 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -4,19 +4,19 @@ `generate_dataset.py` produces a unified Parquet dataset containing every token from a tokenized git repository, annotated with commit metadata, authorship information, -person identity, and the commit's git trailers. It is the final +person identity, firm attribution, and the commit's git trailers. It is the final output of the CreGit pipeline (Step 10). -**The schema is 67 columns**, in this order: +**The schema is 70 columns**, in this order: | Block | Columns | # | |-------|---------|---| | [Token data](#token-data) | `repo_name`, then `file_path` … `is_structural` | 1 + 8 | | [Per-project provenance](#per-project-provenance) | `clone_url` … `file_mask` | 29 | | [Git commit metadata](#git-commit-metadata) | `cregit_commit_sha` … `commit_summary` | 9 | -| [Person identity](#person-identity) | `personid` … `repo_tag` | 5 | +| [Person identity](#person-identity) | `personid` … `repo_tag`, including the three [firm](#firm-attribution) columns | 8 | | [Commit trailers](#commit-trailers-footers) | `footer_*` | 15 | -| | **total** | **67** | +| | **total** | **70** | `repo_name` is column 1 and the 29 provenance columns are 2-30, so `file_path` is column 31. @@ -114,8 +114,28 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or | `person_name` | `TEXT` | persons | Canonical display name for this person. Derived via `coalesce(p.personname, e.personid)`. If neither is available, this is `NULL`. | | `person_email` | `TEXT` | emails | Email address that matched this commit's author name/email pair. | | `person_domain` | `TEXT` | emails | Domain part of the email address. | +| `firm_raw` | `TEXT` | `--firm-map` | The map's `company` string for `person_domain`, exactly as the map gives it. `''` when the domain is not in the map. | +| `firm` | `TEXT` | `--firm-canonical` | The canonical firm name. Equals `firm_raw` unless the reviewed canonical table renames it. Never overwrites `firm_raw`. | +| `firm_source` | `TEXT` | `--firm-map` | The map's `source` for that domain: `patch`, `gitdm`, `rich`, `builtin`, `correction` (all hand-curated), `cncf-gitdm`, `cncf-gitdm-single` (a single-person inference, 2,771 of 4,049 rows), `spinellis[-sec]`. `''` means the domain is not in the map, so this is the column to filter on for "attributed at all". | | `repo_tag` | `TEXT` | commitmap | Repository tag indicating the origin repository. Values: `'p'` (pre-history), `'b'` (BitKeeper), `'l'` (Linux), or `''` (unknown/single repo). | +### Firm attribution + +The 29 provenance columns are per-project constants. Firm is resolved **per row** +from `person_domain` by a LEFT JOIN against `--firm-map` +(`cregit-token-pipeline/data/affiliation.merged.csv`), and `firm` comes from the +reviewed canonical-name table `--firm-canonical`. `firm_raw` is never overwritten. +Both files must have a unique key: a repeated `domain` or `firm_raw` would +multiply token rows, so the generator refuses such a file before Phase 1. + +```sql +-- Firm-level token counts, excluding single-person inferences +SELECT firm, COUNT(*) AS tokens +FROM 'dataset.parquet' +WHERE is_structural = 0 AND firm <> '' AND firm_source <> 'cncf-gitdm-single' +GROUP BY firm ORDER BY tokens DESC; +``` + ### Commit trailers (footers) Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with @@ -300,18 +320,15 @@ produces no blame and no dataset row rather than rows of unparsed text: | Mechanism | Recorded as | Effect here | |---|---|---| -| the **blob denylist**, `blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list ships in the jar; nothing at run time can extend or override it | +| the **blob denylist**, `blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list ships in the jar; nothing at run time can extend or override it | | **oversized** blobs (at or above JGit's stream-file threshold) | `blobsOversized`, plus one `EXCLUDED oversized blob` line each | no rows for those blobs | -| a blob the tokenizer **timed out** on | `blobsTimedOut`, exit 4 | **no Parquet at all**: steps 3-10 never run, so this generator is never reached and the project cannot publish while a timeout is unexplained | +| a blob the tokenizer **timed out** or **crashed** on | `blobsTimedOut` or `blobsParserCrashed`, plus one `EXCLUDED failed blob` line each with `reason=timeout` or `reason=parser-crash` | no rows for those blobs. The run still exits 0 | -One example, so the shape is clear: [`2ee2673a`](https://github.com/tencent/tencentkona-21/blob/2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65/test/langtools/tools/javac/annotations/typeAnnotations/newlocations/TestNewCastArray.java) -— srcML 1.1.0's Java parser does not terminate on type-annotated array types -(upstream [`srcML/srcML#2361`](https://github.com/srcML/srcML/issues/2361), closed by the -revision `nix/srcml.nix` pins). -The list is keyed by content, not path: this file moved in a repository -reorganisation, so its four revisions at two paths are four entries. Failure is -non-termination, not slowness — 0 bytes out, one core at 100%, at every budget -from 5 s to 600 s. +The denylist holds 209 blob ids, keyed by content, in three srcML 1.1.0 defects: +8 Java blobs that do not terminate (upstream +[`srcML/srcML#2361`](https://github.com/srcML/srcML/issues/2361)), 4 C blobs that do +not terminate (no upstream issue), and 197 C and C++ blobs on which srcML crashes +under `--position`. The header of the file carries the reproducers and citations. Also outside the dataset, by mask rather than by exclusion: M4 (`.am`, `.ac`), whose tokenizer's lexer is not fit for real autotools input, and `.ixx`, `.inl`, @@ -355,9 +372,14 @@ persons.db ─────────────┤ For each `.blame` file: 1. Parse each line as `commit_sha;token_content` -2. Walk through the original source file character-by-character to match tokens -3. Classify each token (structural vs content, type, value) -4. Insert into SQLite `token_map` +2. Repair the token line: remove the Rust tokenizer's `line:col` prefix, and, + in a strict UTF-8 file that is not `.rs` and has no byte-order mark, undo the + srcML 1.1.0 Latin-1 misreading (`ö` becomes `ö`). The source is read as + `utf-8-sig`, so a byte-order mark does not shift the walk. `main` prints the + counts on a `Repaired:` line. +3. Walk through the original source file character-by-character to match tokens +4. Classify each token (structural vs content, type, value) +5. Insert into SQLite `token_map` ### Phase 2: DuckDB JOIN → Parquet @@ -365,6 +387,7 @@ Join `token_map` with the three SQLite databases: - `commits` — commit metadata (author, committer, dates, summary) - `commitmap` — cregit → original commit SHA mapping - `emails` / `persons` — person identity resolution +- `--firm-map` / `--firm-canonical` — firm attribution, when given Output is written as ZSTD-compressed Parquet. @@ -393,6 +416,8 @@ uv run python generate_dataset/generate_dataset.py \ | `--repo-name` | no | Repository name. Default: inferred from output filename. | | `--project-meta` | no | JSON sidecar of per-project provenance, from `project_meta.py`. Omit and those columns are written empty. | | `--project-key` | no | Which key of the sidecar this project is. Default: `--repo-name`. A key the sidecar does not hold is an error, not a blank row. | +| `--firm-map` | no | CSV of `domain,company,kind,source`, joined per row against `person_domain`. Omit and `firm_raw`, `firm` and `firm_source` are written empty. A repeated `domain` is an error. | +| `--firm-canonical` | no | CSV of `firm_raw,firm,…`, the reviewed canonical-name table that fills `firm`. Needs `--firm-map`. Omit and `firm` repeats `firm_raw`, so split spellings stay split. | | `--verbose` | no | Enable info-level logging to stderr. | ## Cross-reference: Perl ↔ Python diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index c4f4a1da6..e7d3af175 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -25,12 +25,14 @@ """ import argparse +import io import logging import os import re import sqlite3 import sys import tempfile +from collections import Counter from pathlib import Path logger = logging.getLogger(__name__) @@ -289,14 +291,93 @@ def parse_blame_line(line: str) -> tuple[str, str] | None: return commit_sha, token_content +# The Rust tokenizer puts its position and a TAB before every token line: +# `5:7identifier|x`, `N:-` on a DECL line, `-:-` on a unit marker. The +# pipeline never asks for positions, and classify_and_skip splits on the first +# pipe, so the prefix went into token_type and the walk skipped its length as +# well. Nothing else writes a line that starts with digits, a colon and a TAB. +_POSITION_PREFIX = re.compile(r"^(?:\d+|-):(?:\d+|-)\t") + +# The prebuilt srcML 1.1.0 binary reads UTF-8 source as Latin-1: `ö` (C3 B6) +# comes out as the two characters `ö`. A UTF-8 lead byte read as Latin-1 lies +# in U+00C2..U+00F4, and a continuation byte in U+0080..U+00BF. Text the build +# wrote correctly almost never holds that pair, and undo_mojibake also needs +# the whole token to decode as UTF-8 before it changes anything. +_MOJIBAKE = re.compile("[Â-ô][\u0080-¿]") + +_UTF8_BOM = b"\xef\xbb\xbf" + + +def undo_mojibake(text: str) -> str: + """Return the UTF-8 text that srcML misread as Latin-1, or `text` as is.""" + try: + return text.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + return text + + +def read_source(source_path: Path) -> tuple[str, bool, bool]: + """Read a source file for the walk. Return its text, whether it started + with a byte-order mark, and whether the mojibake repair can apply to its + tokens. + + The text loses a leading byte-order mark. The tokenizers drop it, so a + BOM kept as a character puts the walk one character ahead to the end of + the file. + + The repair applies only when the file is strict UTF-8, has no BOM and is + not Rust. srcML decodes a BOM file correctly, and the Rust tokenizer does + not use srcML. In those files, and in a file that is not UTF-8, a token + that matches _MOJIBAKE is the true text. + """ + raw = source_path.read_bytes() + has_bom = raw.startswith(_UTF8_BOM) + try: + raw.decode("utf-8") + strict = True + except UnicodeDecodeError: + strict = False + # TextIOWrapper, not raw.decode: it keeps the newline translation of the + # text-mode open() that this replaces, so all other files read the same. + with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-8-sig", + errors="replace") as f: + text = f.read() + repairable = strict and not has_bom and not str(source_path).endswith(".rs") + return text, has_bom, repairable + + +def repair_token_line(token_content: str, repairable: bool, + stats: Counter) -> str: + """Return the token line with the Rust position prefix removed and, when + `repairable`, the srcML mojibake undone. Count each repair in `stats`. + + This must run before classify_and_skip: the walk consumes one source + character per token character, so a wrong token text also moves every + later position in the file. + """ + prefix = _POSITION_PREFIX.match(token_content) + if prefix: + stats["position_prefix"] += 1 + token_content = token_content[prefix.end():] + if repairable and _MOJIBAKE.search(token_content): + fixed = undo_mojibake(token_content) + if fixed != token_content: + stats["mojibake_tokens"] += 1 + token_content = fixed + return token_content + + def process_blame_file( - blame_path: Path, source_path: Path, rel_path: str, db_cursor + blame_path: Path, source_path: Path, rel_path: str, db_cursor, + stats: Counter | None = None, ) -> int: with open(blame_path, encoding="utf-8", errors="replace") as f: blame_lines = f.readlines() - with open(source_path, encoding="utf-8", errors="replace") as f: - source_text = f.read() + source_text, has_bom, repairable = read_source(source_path) + if stats is None: + stats = Counter() + stats["bom_files"] += has_bom reader = SourceReader(source_text) counted = [0] @@ -310,6 +391,7 @@ def rows(): continue commit_sha, token_content = parsed + token_content = repair_token_line(token_content, repairable, stats) info = classify_and_skip(token_content, reader) counted[0] += 1 yield ( @@ -400,6 +482,103 @@ def sql_literal(value) -> str: return "'" + str(value).replace("'", "''") + "'" +# The three firm columns, in dataset order. They sit immediately after +# person_domain and before repo_tag, because firm is RESOLVED FROM person_domain: +# the key and the three values it produces belong together, and a reader who +# filters on person_domain finds the answer in the next three columns. +# +# Unlike the 29 metadata columns these are NOT per-project constants. They are +# per row, so they come from a real join against an external, auditable map +# (--firm-map, this repository's cregit-token-pipeline/data/affiliation.merged.csv) +# rather than from a SQL literal. The map stays a file on disk on purpose: baking +# 4,049 rows into a query would make the attribution unreviewable. +# +# firm_raw the map's `company` string, exactly as the map gives it +# firm the canonical name after --firm-canonical is applied +# firm_source the map's `source`: patch | gitdm | rich | builtin | correction | +# cncf-gitdm | cncf-gitdm-single | spinellis[-sec] | … +# +# An empty firm_source means "this person_domain is not in the map", so it is the +# column to filter on for "attributed at all". A reader who distrusts +# single-person inferences filters firm_source <> 'cncf-gitdm-single', which is +# 2,771 of the map's 4,049 rows. +FIRM_FIELDS = ("firm_raw", "firm", "firm_source") + + +def read_csv_column(path, column): + """One column of a CSV, as a list. Used only for the duplicate checks below. + + csv, not duckdb: these checks run before Phase 1 and must not depend on the + query engine being reachable. + """ + import csv + + with open(path, newline="") as fh: + return [(row.get(column) or "") for row in csv.DictReader(fh)] + + +def check_key_is_unique(path, column, what) -> int: + """Refuse a lookup table with a repeated key. Returns the row count. + + This is the one failure mode of the firm join that would be invisible. Both + tables are joined LEFT against token_map, so a domain appearing twice + DUPLICATES every token row of every person on that domain — the file still + validates, the schema still matches, and only the row count betrays it. + Failing here costs a second; finding it later costs the corpus. + """ + keys = [k.strip().lower() for k in read_csv_column(path, column)] + if len(keys) != len(set(keys)): + seen, dupes = set(), [] + for k in keys: + if k in seen and k not in dupes: + dupes.append(k) + seen.add(k) + raise SystemExit( + f"{path}: {what} repeats {len(dupes)} key(s) in `{column}`: " + f"{', '.join(dupes[:5])}{' …' if len(dupes) > 5 else ''}. " + "A repeated key multiplies token rows through the LEFT JOIN.") + return len(keys) + + +def firm_sql(firm_map, firm_canonical) -> tuple[str, str]: + """(SELECT lines, JOIN lines) for the three firm columns. + + With no --firm-map the columns are three empty strings and no join is added, + so a caller that predates the flag still writes a schema-valid file — the + same bargain --project-meta makes. An empty firm_source then reads as "no + attribution", which is exactly true. + """ + if not firm_map: + return ("".join(f" '' AS {f},\n" for f in FIRM_FIELDS), "") + # No canonical table means `firm` repeats `firm_raw`: the column still exists + # and still carries a name, it is just the unnormalised one. + firm_expr = ("coalesce(fc.firm, fm.company, '')" if firm_canonical + else "coalesce(fm.company, '')") + select = ( + " coalesce(fm.company, '') AS firm_raw,\n" + f" {firm_expr} AS firm,\n" + " coalesce(fm.source, '') AS firm_source,\n" + ) + # lower() on both sides: the map is written lower-cased by build_domain_map, + # but persons.db's domain column is whatever the commit's e-mail carried. + # all_varchar=true so a company spelled like a number ('1&1', '360') cannot + # be sniffed into another type and change the Parquet's schema. + join = ( + " LEFT JOIN (SELECT lower(domain) AS domain, company, source\n" + f" FROM read_csv_auto({sql_literal(firm_map)},\n" + " header=true, all_varchar=true)) fm\n" + " ON fm.domain = lower(e.domain)\n" + ) + if firm_canonical: + join += ( + " LEFT JOIN (SELECT firm_raw, firm\n" + f" FROM read_csv_auto({sql_literal(firm_canonical)},\n" + " header=true, all_varchar=true)) fc\n" + " ON fc.firm_raw = fm.company\n" + ) + return (select, join) + + def load_project_meta(meta_path, project_key) -> dict: """The project's constants, or empty strings when no sidecar was given. @@ -471,6 +650,23 @@ def main(): metavar="NAME", help="key into --project-meta (the manifest name). Defaults to --repo-name.", ) + parser.add_argument( + "--firm-map", + default="", + metavar="PATH", + help="CSV of domain,company,kind,source (cregit-token-pipeline/" + "data/affiliation.merged.csv). Joined per row against person_domain to " + "fill firm_raw and firm_source. Empty means emit the three firm columns " + "as empty strings, so an older caller still produces a schema-valid file.", + ) + parser.add_argument( + "--firm-canonical", + default="", + metavar="PATH", + help="CSV of firm_raw,firm,... — the REVIEWED canonical-name table that " + "fills the `firm` column. Needs --firm-map. Without it `firm` repeats " + "`firm_raw`, so the split spellings stay split.", + ) parser.add_argument( "--memory-limit", default=DEFAULT_MEMORY_LIMIT, @@ -529,6 +725,27 @@ def main(): project_meta = load_project_meta( args.project_meta, args.project_key or args.repo_name) + # Same rule for the firm map, and one extra reason: a repeated key in either + # lookup table multiplies token rows silently, so both are checked here + # rather than after the join has already written the Parquet. + if args.firm_canonical and not args.firm_map: + parser.error("--firm-canonical needs --firm-map: there is no firm_raw to " + "canonicalise without a map to read it from") + if args.firm_map: + if not Path(args.firm_map).is_file(): + print(f"ERROR: firm-map not found: {args.firm_map}", file=sys.stderr) + sys.exit(1) + n = check_key_is_unique(args.firm_map, "domain", "the firm map") + print(f"Firm map: {args.firm_map} ({n} domains)") + if args.firm_canonical: + if not Path(args.firm_canonical).is_file(): + print(f"ERROR: firm-canonical not found: {args.firm_canonical}", + file=sys.stderr) + sys.exit(1) + n = check_key_is_unique(args.firm_canonical, "firm_raw", + "the canonical-name table") + print(f"Firm canon: {args.firm_canonical} ({n} names)") + blame_files = sorted(blame_root.rglob("*.blame")) if not blame_files: print("WARNING: no .blame files found", file=sys.stderr) @@ -577,6 +794,7 @@ def main(): cursor = sync_conn.cursor() total_tokens = 0 files_processed = 0 + repairs = Counter() for bf in blame_files: rel = bf.relative_to(blame_root) @@ -591,7 +809,7 @@ def main(): if args.verbose: print(f" {bf.name} -> {rel_str}") - count = process_blame_file(bf, source_path, rel_str, cursor) + count = process_blame_file(bf, source_path, rel_str, cursor, repairs) total_tokens += count files_processed += 1 @@ -608,6 +826,9 @@ def main(): sync_conn.close() print(f"Synced {files_processed} files, {total_tokens} tokens") + print(f"Repaired: {repairs['mojibake_tokens']} mojibake tokens, " + f"{repairs['position_prefix']} position prefixes, " + f"{repairs['bom_files']} files with a byte-order mark") if total_tokens == 0: print("ERROR: no tokens processed, nothing to output", file=sys.stderr) @@ -646,6 +867,7 @@ def main(): con.execute(f"CALL sqlite_attach('{args.persons_db}')") meta_sql = project_meta_sql(project_meta) + firm_select, firm_join = firm_sql(args.firm_map, args.firm_canonical) query = f""" COPY ( @@ -676,6 +898,7 @@ def main(): e.emailaddr AS person_email, e.domain AS person_domain, +{firm_select} coalesce(m.repo, '') AS repo_tag, coalesce(ftr.footer_signed_off_by, []) AS footer_signed_off_by, @@ -700,7 +923,7 @@ def main(): LEFT JOIN emails e ON (c.autname = e.emailname AND c.autemail = e.emailaddr) LEFT JOIN persons p ON e.personid = p.personid - LEFT JOIN ( +{firm_join} LEFT JOIN ( SELECT f.cid, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'signed-off-by') AS footer_signed_off_by, diff --git a/nix/srcml.nix b/nix/srcml.nix index 93f91a247..62250b5dc 100644 --- a/nix/srcml.nix +++ b/nix/srcml.nix @@ -1,6 +1,6 @@ -# srcML at a pinned source revision. The 1.1.0 release binary segfaults under -# `--position` (srcML/srcML#2395, fixed upstream by cebd3ab, still unreleased), -# and the pipeline always passes `--position`. +# srcML 1.1.0 built from its release tag, for the libsrcml headers. Later +# revisions change the tokens of files that 1.1.0 parses (srcML/srcML#2325 marks +# up some macro calls as expressions; UTF-8 input is no longer read as Latin-1). { lib , stdenv , fetchFromGitHub @@ -13,7 +13,6 @@ , libarchive , curl , libiconv -, libxcb }: let @@ -36,33 +35,25 @@ let hash = "sha256-O9l7k1/2fVfEcyfJmWzXmju6TRQHK+NZFoaYmdd5KRg="; }; - clip = fetchFromGitHub { - owner = "dacap"; - repo = "clip"; - rev = "f2bd226fd96f3431f4675d9c13aa7334dd06ddda"; # tag v1.15 - hash = "sha256-qviH7XwNOTA3sMnfwaJL+GqXHxeG6giIZneuTeB+Lm8="; - }; - cli11 = fetchurl { - url = "https://github.com/CLIUtils/CLI11/releases/download/v2.7.0/CLI11.hpp"; - hash = "sha256-WqjWN4eSFj0eDfZC7uMTrO7KaKU0ZFZq5yBNWcL14Wg="; + url = "https://github.com/CLIUtils/CLI11/releases/download/v2.5.0/CLI11.hpp"; + hash = "sha256-S/CpSQqnIJF2zNpwVE+VQT5ZTSIHzKM8nNGN7RiaY6Y="; }; in stdenv.mkDerivation { pname = "srcml"; - version = "1.1.0-unstable-2026-09-20"; + version = "1.1.0"; src = fetchFromGitHub { owner = "srcML"; repo = "srcML"; - rev = "8a3a629d84ac0e0228a5757e6b9876742f88ed25"; - hash = "sha256-stn1VP+D9Awd9B+r3UWIjHJUkoNU52SOJndX9hLEnrY="; + rev = "af1efb78ddd6fc4a8fc75bc6dc3c2e22c5fd812d"; # tag v1.1.0 + hash = "sha256-AdPVoUKI4/EigJz34BuGIbbN0jUGZTIxOQbmXr8IImM="; }; nativeBuildInputs = [ cmake ninja jdk_headless ]; buildInputs = [ libxml2 libxslt libarchive curl ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ libxcb ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ]; # Without this the manpage step downloads a prebuilt page from a bare IP over @@ -80,7 +71,6 @@ stdenv.mkDerivation { tar xf ${antlr} -C "$deps" cp -r --no-preserve=mode,ownership ${tinysha1} "$deps/tinysha1" cp -r --no-preserve=mode,ownership ${ctpl} "$deps/ctpl" - cp -r --no-preserve=mode,ownership ${clip} "$deps/clip" cp --no-preserve=mode,ownership ${cli11} "$deps/cli11/CLI11.hpp" cp --no-preserve=mode,ownership \ ${lib.getDev libarchive}/include/archive.h \ @@ -91,7 +81,6 @@ stdenv.mkDerivation { "-DFETCHCONTENT_SOURCE_DIR_ANTLRSRC=$deps/antlr-2.7.7" "-DFETCHCONTENT_SOURCE_DIR_TINYSHA1=$deps/tinysha1" "-DFETCHCONTENT_SOURCE_DIR_CTPL_STL_SRC=$deps/ctpl" - "-DFETCHCONTENT_SOURCE_DIR_CLIP=$deps/clip" "-DFETCHCONTENT_SOURCE_DIR_CLI11=$deps/cli11" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE1=$deps/libarchive-include" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE=$deps/libarchive-include" diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index b7ed4ed09..79c3c4cf8 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -12,6 +12,14 @@ artifacts (jars, tokenizers) are built automatically first — run inside Build: --build-only build all pipeline artifacts and exit, running nothing + --ensure-artifacts + run the pre-run build guard and exit, building nothing that + is already current. That guard builds any MISSING artifact, + and additionally rebuilds the Rust tokenizer when its + sources are newer than the binary — it is the pipeline's only + compiled tokenizer, and a stale one silently shifts every + Rust token column by a field. The four jars and srcml2token + are still only checked for existence. Target repository: --repo-url URL git URL (or local path) of the repository to process (REQUIRED) @@ -22,7 +30,17 @@ Target repository: which is correct for GitHub/GitLab-style hosts) --mask REGEX regex selecting the files to tokenize; quote it (default: `perl tokenize/fileMask.pl`, every extension - cregit can parse). A different mask forces a full rebuild. + cregit can parse). A different mask forces a full rebuild, + unless --mask-widened applies. + --mask-widened resume across a wider mask, keeping blob_map's + tokenizations. Needs FROM_STEP>=2, not with --mode sharded. + blobExec verifies the rows and refuses with exit 3 if the + new mask drops a tokenized path or a new_blob is missing. + --retokenize EXTS comma-separated extensions (as in CregitLanguages.pm, e.g. + rs or c,h) whose cached tokenizations and memo entries are + redone, after their tokenizer identity changed. Needs + FROM_STEP=2 exactly, not with --mode sharded. blobExec + refuses (exit 7) a --retokenize that would change nothing. --work DIR working/output directory (default: ../cregit-files). NOTE: a full run (FROM_STEP=1) starts by deleting this directory; use one directory per target repository. @@ -47,9 +65,9 @@ Target repository: or more (MEMO_KEEP_THRESHOLD, overridable as CREGIT_MEMO_KEEP_THRESHOLD) is refused; see --force-clean. --blob-timeout N wall-clock budget in seconds for one blob's tokenizer - (blobExec default: 600). A child that exceeds it is killed, - that blob is left untokenized, step 2 stops with exit 4 and - a step-2 resume retries exactly those blobs. Also settable + (blobExec default: 600). A child that exceeds it is killed + and that blob is excluded, as an EXCLUDED failed blob line + says. Also settable as CREGIT_BLOB_TIMEOUT in the environment, which is how to reach it through ctp.py. --stall-timeout N watchdog window in seconds (blobExec default: 1800). If no @@ -72,6 +90,9 @@ Output: should skip them. NOTE: the HTML views are the fallback output when python3+duckdb is missing, so --skip-html without duckdb leaves the run with no final artifact. + --reblame re-blame every file in step 7, replacing existing .blame + output. Pass it when the blame itself changed; a resume + without it skips every file already blamed. --gc MODE how to pack the generated cregit repo after tokenizing (default: plain) none do not pack at all. Fastest, but every later @@ -101,6 +122,11 @@ Output: --project-key NAME which key of the sidecar holds this project. Omit to use --repo-name. A key the sidecar does not hold fails the run. + --firm-map PATH the domain->firm CSV joined per row against person_domain + to fill firm_raw and firm_source. Omit and they are empty. + --firm-canonical PATH + the reviewed canonical-name table that fills `firm`. Needs + --firm-map. Omit and `firm` repeats `firm_raw`. Tokenizer: --mode MODE tokenizer walk mode (default: pipeline) @@ -111,6 +137,11 @@ Tokenizer: for repos too large to tokenize in one process; delegates to blobExec/shard_build.sh --shards N shard count for --mode sharded (default: 4) + --tokenizer-worker + send each blob to a pool of persistent tokenizer processes + (tokenizeByBlobId/tokenWorker.pl) instead of starting the + tokenizer once per blob. Same output. Needs --mode pipeline or + pipeline-trees (default: off) --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -175,6 +206,8 @@ build_dataset_argv() { [ -n "$DUCKDB_THREADS" ] && DATASET_ARGV+=(--duckdb-threads "$DUCKDB_THREADS") [ -n "$PROJECT_META" ] && DATASET_ARGV+=(--project-meta "$PROJECT_META") [ -n "$PROJECT_KEY" ] && DATASET_ARGV+=(--project-key "$PROJECT_KEY") + [ -n "$FIRM_MAP" ] && DATASET_ARGV+=(--firm-map "$FIRM_MAP") + [ -n "$FIRM_CANONICAL" ] && DATASET_ARGV+=(--firm-canonical "$FIRM_CANONICAL") return 0 # a false test above must not fail the function under `set -e` } @@ -189,6 +222,10 @@ DEFAULT_JOBS=4 JOBS=${CREGIT_JOBS:-$DEFAULT_JOBS} FROM_STEP=1 BUILD_ONLY=0 +# 1 means "run ensure_artifacts and exit". Distinct from --build-only, which +# rebuilds all six unconditionally: this one runs exactly the guard a normal run +# runs, so it is also how that guard is tested. +ENSURE_ARTIFACTS_ONLY=0 REPO_GIT_URL="" REPO_NAME="" REPO_COMMIT_URL="" @@ -197,6 +234,7 @@ WORK="../cregit-files" # Empty means "/memo"; resolved after parsing because it depends on --work. MEMO_DIR="" SKIP_HTML=0 +REBLAME=0 GC_MODE="plain" MEMORY_LIMIT="" DUCKDB_THREADS="" @@ -205,11 +243,16 @@ DUCKDB_THREADS="" # on whether the caller knows about the sidecar. PROJECT_META="" PROJECT_KEY="" +FIRM_MAP="" +FIRM_CANONICAL="" FORCE_CLEAN=0 +MASK_WIDENED=0 +RETOKENIZE="" # Empty means "do not pass the flag". The CREGIT_* fallbacks are how ctp.py, # which has no passthrough of its own, reaches these values. BLOB_TIMEOUT="${CREGIT_BLOB_TIMEOUT:-}" STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}" +TOKENIZER_WORKER=0 KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES" @@ -339,6 +382,9 @@ tokenize_gate() { marker="TOKENIZE-PARSER-CRASHES" summary="$stage hit a parser crash: deterministic, so --blob-timeout will not help. Fix srcML, or denylist the diagnosed blob." ;; + 7) die "$stage refused --retokenize, which would have invalidated nothing (exit 7). + Check the extension spelling (as in tokenize/CregitLanguages.pm) and that + --memo-dir names this project's memo ($MEMO_DIR). Nothing in $WORK changed." ;; *) die "$stage failed (exit $status)" ;; esac @@ -356,6 +402,7 @@ need_val() { while [ $# -gt 0 ]; do case "$1" in --build-only) BUILD_ONLY=1; shift ;; + --ensure-artifacts) ENSURE_ARTIFACTS_ONLY=1; shift ;; --repo-url) need_val "$@"; REPO_GIT_URL="$2"; shift 2 ;; --repo-name) need_val "$@"; REPO_NAME="$2"; shift 2 ;; --commit-url) need_val "$@"; REPO_COMMIT_URL="$2"; shift 2 ;; @@ -363,14 +410,20 @@ while [ $# -gt 0 ]; do --work) need_val "$@"; WORK="$2"; shift 2 ;; --memo-dir) need_val "$@"; MEMO_DIR="$2"; shift 2 ;; --skip-html) SKIP_HTML=1; shift ;; + --reblame) REBLAME=1; shift ;; --force-clean) FORCE_CLEAN=1; shift ;; + --mask-widened) MASK_WIDENED=1; shift ;; + --retokenize) need_val "$@"; RETOKENIZE="$2"; shift 2 ;; --blob-timeout) need_val "$@"; BLOB_TIMEOUT="$2"; shift 2 ;; + --tokenizer-worker) TOKENIZER_WORKER=1; shift ;; --stall-timeout) need_val "$@"; STALL_TIMEOUT="$2"; shift 2 ;; --gc) need_val "$@"; GC_MODE="$2"; shift 2 ;; --memory-limit) need_val "$@"; MEMORY_LIMIT="$2"; shift 2 ;; --duckdb-threads) need_val "$@"; DUCKDB_THREADS="$2"; shift 2 ;; --project-meta) need_val "$@"; PROJECT_META="$2"; shift 2 ;; --project-key) need_val "$@"; PROJECT_KEY="$2"; shift 2 ;; + --firm-map) need_val "$@"; FIRM_MAP="$2"; shift 2 ;; + --firm-canonical) need_val "$@"; FIRM_CANONICAL="$2"; shift 2 ;; --mode) need_val "$@"; MODE="$2"; shift 2 ;; --shards) need_val "$@"; SHARDS="$2"; shift 2 ;; --jobs) need_val "$@"; JOBS="$2"; shift 2 ;; @@ -393,6 +446,75 @@ if [ -z "$MASK" ]; then [ -n "$MASK" ] || { echo "${SELF_DIR}/tokenize/fileMask.pl printed nothing" >&2; exit 2; } fi +if [ "$MASK_WIDENED" = 1 ] && [ "$FROM_STEP" = "1" ]; then + echo "--mask-widened needs FROM_STEP>=2. A step-1 run starts by deleting $WORK, so the + blob map it would reuse and the cregit.git its new_blob ids live in are both + gone before blobExec starts, and the flag would preserve nothing. + Resume instead: + runner: $0 --repo-url --work $WORK --mask-widened [same flags] 2 + ctp.py: python3 ./ctp.py run [same flags] --mask-widened --from-step 2" >&2 + exit 2 +fi + +if [ "$MASK_WIDENED" = 1 ] && [ "$MODE" = "sharded" ]; then + echo "--mask-widened is not available with --mode sharded: every shard builds a fresh + blob map, so no recorded mask exists to widen. Reuse a prior run's + tokenizations with shard_build.sh --warm-db instead." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$FROM_STEP" != "2" ]; then + echo "--retokenize needs FROM_STEP=2 exactly (got $FROM_STEP). + Step 1 deletes $WORK, so the blob map and memo whose poisoned entries this flag + removes are gone before blobExec starts — and a from-scratch run re-tokenizes + everything anyway, with the current tokenizer, which is the same outcome at + full cost. + Step 3 or later never reaches the invalidation at all: it lives in step 2, so + the flag would be skipped, steps 3-10 would run over the poisoned tokens, and + the run would exit 0. + Resume at step 2: + runner: $0 --repo-url --work $WORK --retokenize $RETOKENIZE [same flags] 2 + ctp.py: python3 ./ctp.py run [same flags] --from-step 2" >&2 + exit 2 +fi + +if [ "$REBLAME" = 1 ] && [ "$FROM_STEP" -gt 7 ]; then + echo "--reblame needs FROM_STEP<=7 (got $FROM_STEP). The re-blame happens inside + step 7. From step 8 onward the flag is skipped, step 10 rebuilds the Parquet from + the blame files already on disk, and the run exits 0 having changed nothing. + Resume at step 7: + runner: $0 --repo-url --work $WORK --reblame [same flags] 7 + ctp.py: python3 ./ctp.py run [same flags] --reblame --from-step 7" >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$MODE" = "sharded" ]; then + echo "--retokenize is not available with --mode sharded: every shard builds a fresh + blob map, so there are no cached tokenizations for it to invalidate. Retokenize + the merged result in a non-sharded step-2 resume, or drop the shards' warm db." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$MASK_WIDENED" = 1 ]; then + echo "--mask-widened and --retokenize cannot be used in the same run. Each verifies its + own precondition against the blob map's rows, and together each would verify + against a state the other is about to change. + Do them one at a time: widen first, then resume again with --retokenize." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ]; then + for _ext in ${RETOKENIZE//,/ }; do + case "$_ext" in + ''|*[!a-z0-9+]*) + echo "invalid --retokenize: '$_ext' is not an extension. Want lowercase names + without a leading dot, comma-separated, as spelled in tokenize/CregitLanguages.pm: + --retokenize rs --retokenize c,h" >&2 + exit 2 ;; + esac + done +fi + # Validate early: pack_cregit_repo runs only once tokenizing has finished. case "$GC_MODE" in none|plain|aggressive) ;; @@ -430,6 +552,19 @@ if [ -n "$PROJECT_KEY" ] && [ -z "$PROJECT_META" ]; then exit 2 fi +if [ -n "$FIRM_MAP" ] && [ ! -f "$FIRM_MAP" ]; then + echo "invalid --firm-map: '$FIRM_MAP' is not a file (build it with build_domain_map.py)" >&2 + exit 2 +fi +if [ -n "$FIRM_CANONICAL" ] && [ ! -f "$FIRM_CANONICAL" ]; then + echo "invalid --firm-canonical: '$FIRM_CANONICAL' is not a file" >&2 + exit 2 +fi +if [ -n "$FIRM_CANONICAL" ] && [ -z "$FIRM_MAP" ]; then + echo "--firm-canonical without --firm-map has no firm_raw to canonicalise" >&2 + exit 2 +fi + # Validate early: step 2 runs for hours. blobExec enforces the relationship # between the two values; this only rejects non-positive integers. for _tv in "BLOB_TIMEOUT:$BLOB_TIMEOUT:--blob-timeout" "STALL_TIMEOUT:$STALL_TIMEOUT:--stall-timeout"; do @@ -486,6 +621,9 @@ case "$MODE" in serial|pipeline|pipeline-trees|sharded) ;; *) echo "invalid --mode: $MODE (serial|pipeline|pipeline-trees|sharded)" >&2; exit 2 ;; esac +if [ "$TOKENIZER_WORKER" = 1 ] && [ "$MODE" != pipeline ] && [ "$MODE" != pipeline-trees ]; then + echo "--tokenizer-worker needs --mode pipeline or pipeline-trees" >&2; exit 2 +fi if [ "$MODE" = "sharded" ]; then [ "$SHARDS" -ge 1 ] 2>/dev/null || { echo "--shards must be a positive integer" >&2; exit 2; } fi @@ -561,30 +699,50 @@ build_all() { log "build complete" } -# needs_build ...: true when the artifact is missing, or -# when any source is newer than it. Staleness has to count, not just absence: a -# jar left behind by another branch's checkout is reused otherwise, and the -# pipeline then runs that branch's code without saying so. +# Only the Rust tokenizer is checked for staleness: it is the one compiled +# tokenizer. The jars and srcml2token are only checked for existence. +RUST_TOKENIZER_SOURCES=( + "$CREGIT/tokenize/rustTokenizer/Makefile" + "$CREGIT/tokenize/rustTokenizer/Cargo.toml" + "$CREGIT/tokenize/rustTokenizer/Cargo.lock" + "$CREGIT/tokenize/rustTokenizer/src" +) + +NEEDS_BUILD_REASON="" needs_build() { - artifact=$1 - shift - [ -e "$artifact" ] || return 0 - [ -n "$(find "$@" -type f -newer "$artifact" -print -quit 2>/dev/null)" ] + local artifact=$1; shift + NEEDS_BUILD_REASON="" + [ $# -gt 0 ] || die "needs_build: no sources declared for $artifact (a source list is missing)" + local p + for p in "$@"; do + [ -e "$p" ] || die \ +"declared source path does not exist: $p + (it guards $artifact). Either the checkout is incomplete or the source list + in this script is wrong. Refusing to guess: an unreadable source list + silently stops guarding that artifact, which is how a 16-day-old + rustTokenizer binary shipped a corpus of shifted token columns." + done + if [ ! -e "$artifact" ]; then + NEEDS_BUILD_REASON="missing" + return 0 + fi + local newer + newer=$(find "$@" -newer "$artifact" -print -quit 2>/dev/null) || true + [ -n "$newer" ] || return 1 + NEEDS_BUILD_REASON="$newer" + return 0 } -# Build only what is missing or out of date, so an up-to-date checkout starts -# instantly. Each list names sources only, never a target/ directory: build -# output is newer than the artifact by definition and would always look stale. ensure_artifacts() { - S="$CREGIT/tokenize/srcMLtoken" - R="$CREGIT/tokenize/rustTokenizer" - needs_build "$SRCML2TOKEN" "$S"/*.cpp "$S"/*.hpp "$S/Makefile" && build_srcml2token - needs_build "$RUST_TOKENIZER" "$R/src" "$R/Cargo.toml" "$R/Cargo.lock" && build_rust_tokenizer - needs_build "$BFG" "$CREGIT/blobExec/src" "$CREGIT/blobExec/build.sbt" "$CREGIT/blobExec/project" && build_blobexec - needs_build "$SLICKGITLOG_JAR" "$CREGIT/slickGitLog/src" "$CREGIT/slickGitLog/build.sbt" && build_legacy_jar slickGitLog - needs_build "$PERSONS_JAR" "$CREGIT/persons/src" "$CREGIT/persons/build.sbt" && build_legacy_jar persons - needs_build "$REMAPCOMMITS_JAR" "$CREGIT/remapCommits/src" "$CREGIT/remapCommits/build.sbt" && build_legacy_jar remapCommits - return 0 # a false needs_build above must not fail the function under `set -e` + [ -x "$SRCML2TOKEN" ] || build_srcml2token + if needs_build "$RUST_TOKENIZER" "${RUST_TOKENIZER_SOURCES[@]}"; then + log "artifact out of date ($NEEDS_BUILD_REASON): $RUST_TOKENIZER" + build_rust_tokenizer + fi + [ -f "$BFG" ] || build_blobexec + [ -f "$SLICKGITLOG_JAR" ] || build_legacy_jar slickGitLog + [ -f "$PERSONS_JAR" ] || build_legacy_jar persons + [ -f "$REMAPCOMMITS_JAR" ] || build_legacy_jar remapCommits } if [ "$BUILD_ONLY" = 1 ]; then @@ -592,10 +750,15 @@ if [ "$BUILD_ONLY" = 1 ]; then exit 0 fi -# Auto-build any missing artifact BEFORE the work directory is touched, so a -# build failure never disturbs the outputs of a previous run. +# Auto-build any missing or out-of-date artifact BEFORE the work directory is +# touched, so a build failure never disturbs the outputs of a previous run. ensure_artifacts +if [ "$ENSURE_ARTIFACTS_ONLY" = 1 ]; then + log "artifacts up to date" + exit 0 +fi + REPO_PATH_ORIGINAL="${WORK}/${REPO_NAME}-original" REPO_PATH_CREGIT="${WORK}/${REPO_NAME}-cregit" REPO_PATH_ORIGINAL_BARE="${REPO_PATH_ORIGINAL}.git" @@ -695,6 +858,77 @@ echo " Log: $LOG_FILE" echo "████████████████████████████████████████████████████████████████████████" echo "" +# --------------------------------------------------------------------------- +# --mask-widened / --retokenize: drop what a re-fold invalidates, keep what it +# reuses +# --------------------------------------------------------------------------- +# +# Both flags re-fold the whole history. A widening because every tree gains +# entries; an invalidation because a re-tokenized blob changes the tree above it, +# and blobExec empties tree_map and commit_map for exactly that reason. Either way +# every rewritten commit gets a new sha. Steps 3-10 are all derived from those shas, +# and two of them do NOT rebuild themselves, which makes this mandatory rather +# than tidy: +# +# step 6 `git clone` into a directory that already exists is `fatal: +# destination path already exists and is not an empty directory`. +# Verified. Any resume over a project that once completed dies here. +# step 7 blameRepoFiles.pl skips a file whose .blame output already exists +# (--overwrite is off by default). Those files name cregit commit shas +# from BEFORE the re-fold, so keeping them means step 10 joins blame +# against commits that no longer exist. That failure is quiet, which +# makes it worse than step 6's. +# +# Steps 3, 4 and 5 rebuild cleanly on their own (slickGitLog drops and recreates +# its schema), but their outputs are deleted anyway: "everything derived from the +# tokenized repository" is a rule someone can check, and "these three are +# self-cleaning and those two are not" is a rule that rots. +# +# KEPT, deliberately and by name: the original bare clone (step 2's input), the +# cregit bare repo (where blob_map's new_blob ids live), the blob map itself, and +# the memo. Those four are the entire point of the flag. +# +# Named paths only. No globs, and never $WORK itself — the wipe of $WORK is the +# expensive mistake this whole script is defended against. +drop_refold_derived_artifacts() { # $1 = which flag is asking, for the log + log "$1: dropping the artifacts derived from the tokenized repo," + log " because the re-fold gives every cregit commit a new sha." + log " KEEPING: $REPO_PATH_ORIGINAL_BARE, $REPO_PATH_CREGIT_BARE," + log " $DB_PATH_BLOBMAP, $MEMO_DIR" + local _stale + for _stale in \ + "$WORK/blame" \ + "$WORK/html" \ + "$REPO_PATH_ORIGINAL" \ + "$REPO_PATH_CREGIT" \ + "$DB_PATH_ORIGINAL" \ + "$DB_PATH_CREGIT" \ + "$DB_PATH_PERSONS" \ + "$XLS_PATH_PERSONS" \ + "$DATASET_PATH" \ + "${WORK}/${REPO_NAME}.validated" + do + if [ -e "$_stale" ]; then + log " dropping $_stale" + rm -rf -- "$_stale" + fi + done + mkdir -p "$WORK/blame" + [ "$SKIP_HTML" = 1 ] || mkdir -p "$WORK/html" +} + +if [ "$MASK_WIDENED" = 1 ]; then + drop_refold_derived_artifacts "--mask-widened" +fi + +# --retokenize does the same drop, but AFTER step 2 has actually invalidated +# something — see the end of step 2. blobExec refuses a --retokenize that would +# invalidate nothing, and doing the drop up front would mean a refused request had +# already deleted the blame output. Blame is this pipeline's bottleneck (measured: +# 8 files a minute, 5.6 days for the largest project), so destroying it to answer +# a request that was then rejected is the most expensive possible way to handle a +# typo in an extension name. + # --------------------------------------------------------------------------- # Step 1 — clone bare original repo # --------------------------------------------------------------------------- @@ -717,17 +951,30 @@ if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then export BFG_MEMO_DIR="$MEMO_DIR" # Route through the tokenize.pl dispatcher (not tokenizeSrcMl.pl directly) so it can -# fan out by language: srcML for .c/.h, rustTokenizer for .rs, etc. The --srcml* / -# --ctags paths are forwarded only to the srcML parser. Behavior-preserving for C. +# fan out by language: srcML for .c/.h, rustTokenizer for .rs, etc. The +# --srcml2token / --ctags paths are forwarded only to the srcML parser. export BFG_TOKENIZE_CMD="${CREGIT}/tokenize/tokenize.pl \ --srcml2token=${SRCML2TOKEN} \ - --srcml=$(which srcml) \ --ctags=$(which ctags)" +# Lets blobExec refuse a cache built by a different tokenizer; fatal when empty, +# because an empty identity turns the check off. +TOKENIZER_IDENTITY=$(perl "${CREGIT}/tokenize/tokenizerIdentity.pl" \ + --srcml2token="${SRCML2TOKEN}" \ + --ctags="$(which ctags)") \ + || die "cannot compute the tokenizer identity (tokenize/tokenizerIdentity.pl). + blobExec needs it to tell a cache built by this tokenizer from one built by a + different one. Build the artifacts first: $0 --ensure-artifacts" +[ -n "$TOKENIZER_IDENTITY" ] \ + || die "tokenize/tokenizerIdentity.pl printed nothing; refusing to run with no tokenizer identity" +log "tokenizer identity: $TOKENIZER_IDENTITY" + TOKENIZE_RC=0 if [ "$MODE" = "sharded" ]; then # Memory-bounded path: N tree-only shards in parallel, then merge + serial # re-fold into $SHARD_OUT/final/{dst.git,blobmap.db} (byte-identical to serial). + # shard_build.sh propagates blobExec's 4 and 5 instead of collapsing them, so + # this branch gets the same marker and the same guard as the serial one. "${CREGIT}/blobExec/shard_build.sh" \ --src "$REPO_PATH_ORIGINAL_BARE" \ --out "$SHARD_OUT" \ @@ -739,11 +986,32 @@ if [ "$MODE" = "sharded" ]; then ${BLOB_TIMEOUT:+--blob-timeout "$BLOB_TIMEOUT"} \ ${STALL_TIMEOUT:+--stall-timeout "$STALL_TIMEOUT"} || TOKENIZE_RC=$? tokenize_gate "$TOKENIZE_RC" "sharded tokenize (shard_build.sh)" + # Said out loud rather than left to be discovered. The tokenizer identity is not + # plumbed through shard_build.sh / shard_merge.py, so the merged blobmap.db a + # sharded build produces carries no tokenizer_id.* rows. A later non-sharded + # resume over that database records the identity of whatever tokenizer is then + # current, without being able to check it against the one that built it. + log "note: --mode sharded does not record a tokenizer identity in the merged blob map." + log " A later resume cannot detect a tokenizer change against it. Non-sharded" + log " modes record and check it; see --retokenize." else MODE_FLAG="" [ "$MODE" = "pipeline" ] && MODE_FLAG="--pipeline" [ "$MODE" = "pipeline-trees" ] && MODE_FLAG="--pipeline-trees" - java -jar "$BFG" $MODE_FLAG \ + WIDENED_FLAG="" + [ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened" + WORKER_FLAG="" + [ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl" + # --retokenize carries --memo-dir with it, always, and blobExec refuses one + # without the other: the blob map and the memo are two caches of the same + # answer, and invalidating either alone invalidates nothing. + RETOKENIZE_FLAGS=() + if [ -n "$RETOKENIZE" ]; then + RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") + fi + java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} \ + "--tokenizer-identity=$TOKENIZER_IDENTITY" \ + ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ ${STALL_TIMEOUT:+--stall-timeout=$STALL_TIMEOUT} \ "$REPO_PATH_ORIGINAL_BARE" \ @@ -765,6 +1033,12 @@ for _m in $KEEP_MARKERS; do fi done +# Only after blobExec accepted the invalidation: a refused one (exit 7) must not +# have cost the blame output. +if [ -n "$RETOKENIZE" ]; then + drop_refold_derived_artifacts "--retokenize $RETOKENIZE" +fi + pack_cregit_repo fi end_step @@ -821,9 +1095,14 @@ end_step step "blame" if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then [ -d "$REPO_PATH_CREGIT" ] || die "step 6 did not produce $REPO_PATH_CREGIT" +# Without --overwrite, a re-blame skips every file that has .blame output. +REBLAME_OPTS=() +[ "$REBLAME" = 1 ] && REBLAME_OPTS+=(--overwrite) +[ "$REBLAME" = 1 ] && log "--reblame: replacing every .blame file, not resuming" perl $CREGIT/blameRepo/blameRepoFiles.pl \ --jobs="$JOBS" \ --formatBlame=$CREGIT/blameRepo/formatBlame.pl \ + ${REBLAME_OPTS[@]+"${REBLAME_OPTS[@]}"} \ $REPO_PATH_CREGIT $WORK/blame "$MASK" fi end_step diff --git a/tests/ensure_artifacts.sh b/tests/ensure_artifacts.sh new file mode 100755 index 000000000..164d22762 --- /dev/null +++ b/tests/ensure_artifacts.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# Tests for ensure_artifacts(): the guard that decides whether a build artifact +# has to be rebuilt before a run. +# +# Why this exists. The guard used to be existence-only — +# +# [ -x "$RUST_TOKENIZER" ] || build_rust_tokenizer +# +# — so an artifact that EXISTED but was older than its sources was never +# rebuilt. Measured on this machine on 2026-09-20: +# tokenize/rustTokenizer/target/release/rust_tokenizer was dated 2026-09-04 +# 18:57 while tokenize/rustTokenizer/src/main.rs was dated 2026-09-20 21:30 — +# a 16-day-old binary running against source that had been fixed. The fix +# (729643e) removed a `line:col` prefix from every Rust token, and because +# the stale binary kept emitting it, every downstream consumer split the token +# on the wrong field and `token_type`, `token_value`, `source_text`, +# `source_line`, `source_col` and `is_structural` were all shifted by one. Not +# one step of the pipeline errored. +# +# So the guard has to compare mtimes for that binary. SCOPE, decided +# deliberately: the Rust tokenizer and nothing else. It is the pipeline's only +# COMPILED tokenizer — CregitLanguages.pm:109-115 routes C, C++ and Java to +# tokenizeSrcMl.pl and M4 to m4Tokenizer/m4.py, all interpreted and therefore +# re-read from disk every invocation. The four sbt jars and srcml2token keep +# their existence-only guards, and the last two cases in this file PIN that, so +# the gap is a recorded decision rather than something a later reader has to +# rediscover. +# +# The other half of the requirement is the fast path: an up-to-date checkout +# must still start a run without rebuilding anything, so every case below also +# asserts what was NOT built. +# +# No real building happens: `make` and `sbt` are stubbed on PATH and record +# their argv, so a case is milliseconds and nothing depends on cargo, g++, sbt +# or a JDK. The runner is driven through --ensure-artifacts, which runs exactly +# this guard and exits. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# --------------------------------------------------------------------------- +# A fixture CREGIT tree: every source path the guard consults and every +# artifact it guards, and nothing else. The runner takes CREGIT from $(pwd), +# so the fixture is a directory holding a copy of the runner. +# +# Two mtime generations are used throughout: sources are stamped OLD and +# artifacts NEW, which is what an up-to-date checkout looks like. A case makes +# one source NEW to make one artifact stale. +# --------------------------------------------------------------------------- +OLD_STAMP=202601010000 +NEW_STAMP=202602010000 + +SOURCES=( + tokenize/srcMLtoken/Makefile + tokenize/srcMLtoken/srcml2token.cpp + tokenize/srcMLtoken/srcml2token.hpp + tokenize/srcMLtoken/srcml2tokenHandlers.cpp + tokenize/srcMLtoken/srcml2tokenHandlers.hpp + tokenize/rustTokenizer/Makefile + tokenize/rustTokenizer/Cargo.toml + tokenize/rustTokenizer/Cargo.lock + tokenize/rustTokenizer/src/main.rs + blobExec/build.sbt + blobExec/project/build.properties + blobExec/project/plugins.sbt + blobExec/src/main/scala/cregit/blobexec/Main.scala + slickGitLog/build.sbt + slickGitLog/project/build.properties + slickGitLog/project/plugins.sbt + slickGitLog/src/main/scala/gitLogToDb.scala + persons/build.sbt + persons/project/build.properties + persons/project/plugins.sbt + persons/src/main/scala/unifyPersons.scala + remapCommits/build.sbt + remapCommits/project/build.properties + remapCommits/project/plugins.sbt + remapCommits/src/main/scala/remapCommits.scala +) + +ARTIFACTS=( + tokenize/srcMLtoken/srcml2token + tokenize/rustTokenizer/target/release/rust_tokenizer + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar +) + +fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/ensureart-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + for p in "${SOURCES[@]}"; do + mkdir -p "$f/$(dirname "$p")" + echo "source" > "$f/$p" + touch -t "$OLD_STAMP" "$f/$p" + done + # Directories are sources too — the guard passes whole source trees to find, + # and a directory's own mtime is what changes when a file is added or + # removed. Stamp them OLD as well, or a freshly created fixture looks like a + # checkout whose every source tree was just edited. + find "$f" -type d -exec touch -t "$OLD_STAMP" {} + + for p in "${ARTIFACTS[@]}"; do + mkdir -p "$f/$(dirname "$p")" + echo "artifact" > "$f/$p" + chmod +x "$f/$p" + touch -t "$NEW_STAMP" "$f/$p" + done + printf '%s' "$f" +} + +# `make` and `sbt` that record their argv and do nothing else. The working +# directory goes in the record too: the three legacy jars share one sbt command +# line and are told apart only by the directory it runs in. +make_stubs() { # $1 = bin dir + local b=$1 tool + for tool in make sbt; do + cat > "$b/$tool" <> "\$STUB_ARGV" +exit 0 +STUB + chmod +x "$b/$tool" + done +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/ensurebin-XXXXXX") +make_stubs "$BIN" +trap 'rm -rf "$BIN"' EXIT + +# Run the guard in a fixture. Prints the runner's own output; the builds it +# invoked land in $ARGV. +run_guard() { # $1 = fixture dir + local f=$1 + ARGV="$f/argv.log" + : > "$ARGV" + ( cd "$f" \ + && STUB_ARGV="$ARGV" PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 \ + timeout 120 bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$f/work" \ + --mask '\.c$' \ + --ensure-artifacts 2>&1 ) +} + +built() { # $1 = fixture, $2 = grep pattern for the build command + grep -q -- "$2" "$1/argv.log" +} + +build_count() { # $1 = fixture + if [ -s "$1/argv.log" ]; then wc -l < "$1/argv.log" | tr -d ' '; else echo 0; fi +} + +# --------------------------------------------------------------------------- +echo "case 1: everything current — the fast path builds nothing" +# The whole reason the guard cannot simply always rebuild: a corpus run starts +# 188 times, and six builds each would dominate the schedule. +W=$(fixture) +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +N=$(build_count "$W"); [ "$N" -eq 0 ]; check "no build was invoked (built: $N)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: a newer Rust source — the exact defect this guard exists for" +W=$(fixture) +touch -t "$NEW_STAMP" "$W/tokenize/rustTokenizer/src/main.rs" +# The artifact and the source now share a timestamp, which is NOT stale: make +# the source strictly newer, as a real edit would. +touch "$W/tokenize/rustTokenizer/src/main.rs" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +built "$W" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? +N=$(build_count "$W"); [ "$N" -eq 1 ]; check "and nothing else was (built: $N)" $? +grep -q 'main.rs' <<<"$OUT"; check "the log names the source that triggered it" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: a missing artifact still rebuilds (the old behaviour is kept)" +W=$(fixture) +rm -f "$W/tokenize/rustTokenizer/target/release/rust_tokenizer" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +built "$W" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? +N=$(build_count "$W"); [ "$N" -eq 1 ]; check "and nothing else was (built: $N)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +# Each declared Rust source, one case each. Cargo.lock matters as much as +# main.rs: a dependency bump changes the tokenizer's behaviour without touching a +# line of this project's code. +# --------------------------------------------------------------------------- +rust_stale_case() { # $1 = source path, relative to the fixture + echo "case: a newer $1 rebuilds the Rust tokenizer" + local w + w=$(fixture) + touch -t "$NEW_STAMP" "$w/$1" + touch "$w/$1" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + built "$w" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? + local n; n=$(build_count "$w"); [ "$n" -eq 1 ]; check "nothing else was (built: $n)" $? + rm -rf "$w" +} + +rust_stale_case tokenize/rustTokenizer/Cargo.lock +rust_stale_case tokenize/rustTokenizer/Cargo.toml +rust_stale_case tokenize/rustTokenizer/Makefile + +# --------------------------------------------------------------------------- +# The accepted gap, pinned so it is a decision and not a surprise. +# +# The four sbt jars and srcml2token are still guarded on existence alone. A +# stale one WOULD be used, silently. Rust was fixed alone because it is the only +# compiled tokenizer (CregitLanguages.pm:109-115 routes the other three +# languages to interpreted scripts, which cannot go stale this way) and because +# it is the artifact that actually failed. These cases assert the current +# behaviour so that whoever changes their minds has to change a test and read +# this comment. +# --------------------------------------------------------------------------- +gap_case() { # $1 = label, $2 = source to touch, $3 = grep pattern for its build + echo "case: a newer $1 source does NOT rebuild it — known, accepted gap" + local w + w=$(fixture) + touch -t "$NEW_STAMP" "$w/$2" + touch "$w/$2" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + local n; n=$(build_count "$w"); [ "$n" -eq 0 ]; check "nothing was built (built: $n)" $? + rm -rf "$w" +} + +gap_case srcml2token tokenize/srcMLtoken/srcml2token.cpp 'srcMLtoken' +gap_case blobExec blobExec/src/main/scala/cregit/blobexec/Main.scala 'assembly' +gap_case slickGitLog slickGitLog/src/main/scala/gitLogToDb.scala 'one-jar' +gap_case persons persons/src/main/scala/unifyPersons.scala 'one-jar' +gap_case remapCommits remapCommits/src/main/scala/remapCommits.scala 'one-jar' + +# But a MISSING one is still built, for all five. That behaviour predates this +# change and must survive it. +missing_case() { # $1 = label, $2 = artifact to delete, $3 = grep pattern + echo "case: a missing $1 is still built" + local w + w=$(fixture) + rm -f "$w/$2" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + built "$w" "$3"; check "$1 was built" $? + local n; n=$(build_count "$w"); [ "$n" -eq 1 ]; check "and only it (built: $n)" $? + rm -rf "$w" +} + +missing_case srcml2token tokenize/srcMLtoken/srcml2token 'srcMLtoken' +missing_case blobExec blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar 'assembly' +missing_case slickGitLog slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar 'one-jar' + +# --------------------------------------------------------------------------- +echo "case: a Rust source path the guard names but the checkout lacks is fatal" +# A typo in the source list would silently switch the guard off for that +# artifact, which is exactly the class of failure this whole file exists to +# stop. It must be loud instead. +W=$(fixture) +rm -f "$W/tokenize/rustTokenizer/Cargo.lock" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -ne 0 ]; check "the run fails (exit $RC)" $? +grep -q 'Cargo.lock' <<<"$OUT"; check "and names the missing source path" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case: --ensure-artifacts does not run the pipeline" +W=$(fixture) +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +[ ! -d "$W/work" ]; check "the work directory was never created" $? +grep -q 'Step 1' <<<"$OUT"; [ $? -ne 0 ]; check "no pipeline step ran" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/fixtures/Tiny.java b/tests/fixtures/Tiny.java new file mode 100644 index 000000000..c9b995f9b --- /dev/null +++ b/tests/fixtures/Tiny.java @@ -0,0 +1,4 @@ +package p; +public class Tiny { + int f(int a) { return a; } +} diff --git a/tests/fixtures/Utf8.java b/tests/fixtures/Utf8.java new file mode 100644 index 000000000..f6b8783b3 --- /dev/null +++ b/tests/fixtures/Utf8.java @@ -0,0 +1,3 @@ +class Café { + int grüß(String ä) { return 1; } +} diff --git a/tests/fixtures/empty.c b/tests/fixtures/empty.c new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/latin1.c b/tests/fixtures/latin1.c new file mode 100644 index 000000000..4191f97fb --- /dev/null +++ b/tests/fixtures/latin1.c @@ -0,0 +1 @@ +int h(int bé) { return 2; } diff --git a/tests/fixtures/tiny.c b/tests/fixtures/tiny.c new file mode 100644 index 000000000..78f2de106 --- /dev/null +++ b/tests/fixtures/tiny.c @@ -0,0 +1 @@ +int main(void) { return 0; } diff --git a/tests/fixtures/tiny.cpp b/tests/fixtures/tiny.cpp new file mode 100644 index 000000000..88e4ae717 --- /dev/null +++ b/tests/fixtures/tiny.cpp @@ -0,0 +1,2 @@ +struct S { int m(int a) const { return a; } }; +template T id(T v) { return v; } diff --git a/tests/fixtures/tiny.rs b/tests/fixtures/tiny.rs new file mode 100644 index 000000000..2954eacc2 --- /dev/null +++ b/tests/fixtures/tiny.rs @@ -0,0 +1,3 @@ +fn add(a: i32, b: i32) -> i32 { + a + b // "x|y" +} diff --git a/tests/fixtures/upper.H b/tests/fixtures/upper.H new file mode 100644 index 000000000..e039d96f3 --- /dev/null +++ b/tests/fixtures/upper.H @@ -0,0 +1,6 @@ +namespace n { +class A { +public: + int x; +}; +} diff --git a/tests/fixtures/utf8.c b/tests/fixtures/utf8.c new file mode 100644 index 000000000..a5d166fda --- /dev/null +++ b/tests/fixtures/utf8.c @@ -0,0 +1,2 @@ +int café(int ä) { return 0; } +int g(int 😀x) { return 1; } diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh index d12aba9ad..11229242d 100755 --- a/tests/pipeline_workdir_guard.sh +++ b/tests/pipeline_workdir_guard.sh @@ -175,6 +175,81 @@ OUT=$(CREGIT_MEMO_KEEP_THRESHOLD=2 run_runner "$W" --memo-dir "$M"); RC=$? grep -q "$W/memo" <<<"$OUT"; check "the refusal names the memo at risk" $? rm -rf "$W" "$M" +echo "case 12: --mask-widened with FROM_STEP=1 must refuse before touching anything" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened); RC=$? +[ "$RC" -eq 2 ]; check "refuses with exit 2 (got $RC)" $? +grep -q "needs FROM_STEP>=2" <<<"$OUT"; check "says which step it needs" $? +grep -q -- "--from-step 2" <<<"$OUT"; check "gives the ctp.py resume form" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map it would have reused is untouched" $? +[ ! -f "$W/pipeline.log" ]; check "and it refused before opening the log, so nothing ran" $? +rm -rf "$W" + +echo "case 13: --mask-widened with FROM_STEP=2 reaches blobExec and is accepted there" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened 2); RC=$? +! grep -q "needs FROM_STEP>=2" <<<"$OUT"; check "no refusal from the step guard" $? +grep -q "maskWidened=true" <<<"$OUT"; check "blobExec reports the flag as set" $? +! grep -q "unknown flag" <<<"$OUT"; check "the shipped jar understands it" $? +[ -f "$W/proj-blobmap.db" ]; check "and the work dir was kept, as a resume does" $? +rm -rf "$W" + +echo "case 14: --mask-widened with --mode sharded must refuse: shards have no mask to widen" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened --mode sharded 2); RC=$? +[ "$RC" -eq 2 ]; check "refuses with exit 2 (got $RC)" $? +grep -q "not available with --mode sharded" <<<"$OUT"; check "says why" $? +grep -q -- "--warm-db" <<<"$OUT"; check "names the sharded equivalent" $? +rm -rf "$W" + +echo "case 15: --mask-widened drops what the re-fold invalidates and keeps what it reuses" +W=$(fixture none) +mkdir -p "$W/blame/src" "$W/proj-cregit" "$W/proj-original" "$W/proj-cregit.git" +echo stale > "$W/blame/src/a.c.blame" +echo stale > "$W/proj-cregit/keepout" +echo stale > "$W/proj-cregit.db" +echo stale > "$W/proj-original.db" +echo stale > "$W/proj-persons.db" +echo stale > "$W/proj-persons.xls" +echo stale > "$W/proj-dataset.parquet" +echo stale > "$W/proj.validated" +plant_memo "$W/memo" 3 +OUT=$(run_runner "$W" --mask-widened 2); RC=$? +[ ! -f "$W/blame/src/a.c.blame" ]; check "stale blame is gone (step 7 would have skipped it)" $? +[ ! -e "$W/proj-cregit/keepout" ]; check "the non-bare cregit clone is gone (step 6 clones into it)" $? +[ ! -e "$W/proj-original" ]; check "the non-bare original clone is gone (step 6 clones into it too)" $? +[ ! -f "$W/proj-cregit.db" ]; check "the cregit log DB is gone" $? +[ ! -f "$W/proj-original.db" ]; check "the original log DB is gone" $? +[ ! -f "$W/proj-persons.db" ]; check "the persons DB is gone" $? +[ ! -f "$W/proj-persons.xls" ]; check "the persons sheet is gone" $? +[ ! -f "$W/proj-dataset.parquet" ]; check "the old Parquet is gone" $? +[ ! -f "$W/proj.validated" ]; check "the stale validated stamp is gone" $? +[ -d "$W/blame" ]; check "but blame/ itself is recreated for step 7" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map is KEPT" $? +[ -d "$W/proj-cregit.git" ]; check "the cregit bare repo is KEPT (new_blob ids live there)" $? +[ -d "$W/proj-original.git" ]; check "the original bare repo is KEPT" $? +[ "$(count_memo "$W/memo")" -eq 3 ]; check "every memo entry is KEPT" $? +grep -q "KEEPING" <<<"$OUT"; check "and the log says what it kept" $? +rm -rf "$W" + +echo "case 16: without --mask-widened a resume touches none of that" +W=$(fixture none) +mkdir -p "$W/blame/src" +echo keepme > "$W/blame/src/a.c.blame" +echo keepme > "$W/proj-cregit.db" +OUT=$(run_runner "$W" 2); RC=$? +[ -f "$W/blame/src/a.c.blame" ]; check "blame is untouched" $? +[ -f "$W/proj-cregit.db" ]; check "the cregit log DB is untouched" $? +! grep -q "KEEPING" <<<"$OUT"; check "and the widening block did not run" $? +rm -rf "$W" + +echo "case 17: without the flag nothing changes, and the flag is what reaches blobExec" +grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $? +grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER" +check "and is only added when asked for" $? +grep -q 'java -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" +check "so it reaches blobExec's argv" $? + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ] diff --git a/tests/reblame_passthrough.sh b/tests/reblame_passthrough.sh new file mode 100755 index 000000000..451799eab --- /dev/null +++ b/tests/reblame_passthrough.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Tests for the runner's half of the re-blame mechanism: the --reblame +# passthrough to step 7, and the guard that stops the flag being a quiet no-op. +# +# Why this exists. blameRepoFiles.pl skips any file whose .blame output already +# exists, and step 7 never passed --overwrite. So a deliberate re-blame — change a +# git blame flag, resume at step 7 — reported every file as already done, exited 0, +# and step 10 rebuilt the Parquet from the OLD blame. Measured on a three-project +# pilot before this flag existed: 0 of 15,036,195 tokens changed author, with +# identical schema and identical row counts, which is exactly what a successful run +# looks like. The unit test for --overwrite already existed and passed; nothing +# tested that the runner passes it. That is the gap these cases close. +# +# `blameRepoFiles.pl` and `java` are stubbed on PATH / in the fixture and record +# their argv, so no blame runs and no case takes more than a second or two. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB + chmod +x "$1/java" +} + +# A throwaway CREGIT tree. The runner takes CREGIT from $(pwd), and its artifact +# staleness guard runs BEFORE any step — including a step-7 resume — so the tree +# needs tokenize/ and each module's build files plus placeholder artifacts, or the +# guard starts a real build and the run dies before step 7. The artifacts are +# placeholders: the guard digests their bytes, so any content works. +# +# On top of that, a stub blameRepoFiles.pl that appends its argv to $BLAME_ARGV. +# The real script is not used, because the assertion is about which flags arrive, +# not about what blame produces. +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/reblamerepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p/src" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + done + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + mkdir -p "$f/blameRepo" + cat > "$f/blameRepo/blameRepoFiles.pl" <<'STUB' +#!/usr/bin/env perl +open(my $fh, '>>', $ENV{BLAME_ARGV}) or exit 0; +print $fh join(' ', @ARGV), "\n"; +close $fh; +exit 0; +STUB + chmod +x "$f/blameRepo/blameRepoFiles.pl" + : > "$f/blameRepo/formatBlame.pl" + printf '%s' "$f" +} + +# A work directory that looks finished through step 6: the non-bare cregit clone +# step 7 requires, and a blame directory holding output a re-blame must replace. +fixture() { + local w + w=$(mktemp -d "${TMPDIR:-/tmp}/reblamework-XXXXXX") + mkdir -p "$w/proj-cregit" "$w/proj-cregit.git" "$w/blame" "$w/html" + echo "days of blame" > "$w/blame/one.c.blame" + echo "db" > "$w/proj-cregit.db" + printf '%s' "$w" +} + +run_from() { # $1 = work dir, $2 = FROM_STEP, rest = extra runner args + local w=$1 from=$2; shift 2 + # BLAME_ARGV must be exported, not just assigned. `VAR=x somefunc` sets the + # variable for the function without exporting it, so the runner's child shell + # and the stub perl beneath it never saw it. + ( cd "$REPO" \ + && PATH="$BIN:$PATH" BLAME_ARGV="${BLAME_ARGV:-}" timeout 120 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --gc none \ + --mask '\.c$' \ + "$@" "$from" 2>&1 ) +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/reblamebin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: without --reblame, step 7 does NOT get --overwrite" +W=$(fixture); ARGV="$W/blame-argv.log" +OUT=$(BLAME_ARGV="$ARGV" run_from "$W" 7); RC=$? +[ -f "$ARGV" ]; check "step 7 invoked blameRepoFiles.pl" $? +grep -q -- '--jobs=' "$ARGV"; check "and passed --jobs" $? +grep -q -- '--overwrite' "$ARGV"; [ $? -ne 0 ] +check "and NOT --overwrite: a resume stays cheap by default" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: with --reblame, step 7 gets --overwrite" +W=$(fixture); ARGV="$W/blame-argv.log" +OUT=$(BLAME_ARGV="$ARGV" run_from "$W" 7 --reblame); RC=$? +[ -f "$ARGV" ]; check "step 7 invoked blameRepoFiles.pl" $? +grep -q -- '--overwrite' "$ARGV"; check "--overwrite reached blameRepoFiles.pl" $? +grep -q -- '--formatBlame=' "$ARGV"; check "and the formatter is still named" $? +# Match the log sentence, not the substring "reblame". The flag name appears in the +# usage text and in every refusal message too, so a bare substring match passed even +# when step 7 never ran. +grep -q 'replacing every .blame file' <<<"$OUT" +check "and the run says it is replacing every .blame file" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: --reblame after step 7 is refused, because it would be skipped" +# The whole point of the flag is to close a silent no-op. Accepting it where it +# cannot act would reopen the same hole one step later. +for FROM in 8 9 10; do + W=$(fixture) + OUT=$(BLAME_ARGV="$W/blame-argv.log" run_from "$W" "$FROM" --reblame); RC=$? + [ "$RC" -ne 0 ]; check "refused at step $FROM (exit $RC)" $? + grep -q 'FROM_STEP<=7' <<<"$OUT"; check "and names the step it needs" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 4: --reblame is accepted at every step that still reaches step 7" +for FROM in 2 7; do + W=$(fixture); ARGV="$W/blame-argv.log" + OUT=$(BLAME_ARGV="$ARGV" run_from "$W" "$FROM" --reblame); RC=$? + grep -q 'FROM_STEP<=7' <<<"$OUT"; [ $? -ne 0 ] + check "not refused at step $FROM" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/retokenize_passthrough.sh b/tests/retokenize_passthrough.sh new file mode 100755 index 000000000..6e839127f --- /dev/null +++ b/tests/retokenize_passthrough.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Tests for the runner's half of the cache-invalidation mechanism: the tokenizer +# identity it computes and passes to blobExec on EVERY run, the --retokenize +# passthrough, the guards that stop the flag being a quiet no-op, and the +# derived-artifact drop that must happen only after the invalidation succeeded. +# +# Why each of these is here: +# +# * the identity has to arrive. If the runner does not pass +# --tokenizer-identity, blobExec records nothing, compares nothing, and the +# hole is exactly as open as it was — a corrected tokenizer reusing every +# cached row, silently, which is what put a line:col prefix into 741,869 .rs +# entries across 45 projects. +# * --retokenize has to arrive WITH --memo-dir. They are two caches of the same +# answer: the blob map, and the memo keyed on sha1 of the file's content with +# no tokenizer in the key. Invalidating either alone invalidates nothing. +# * the flag must not be reachable in a mode or a step where it would do +# nothing. FROM_STEP=1 deletes the caches first; FROM_STEP=3 skips step 2, so +# the invalidation never runs and the run exits 0 over poisoned tokens; +# sharded mode builds a fresh blob map per shard. +# * exit 7 must fail the run. blobExec returns it when --retokenize would have +# invalidated nothing. Treating that as success is the whole failure mode. +# * the derived artifacts (blame, html, the databases) must be dropped only +# AFTER step 2 succeeded. Blame is the pipeline's bottleneck — measured at 8 +# files a minute — so deleting it to serve a request blobExec then rejects is +# the most expensive way to handle a typo. +# +# `java` is stubbed on PATH and records its argv, so no tokenizing happens and no +# case takes more than a second or two. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# A `java` that exits with $STUB_RC, logs its argv, and (on 0) creates the +# destination repo its caller checks for. +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +: "${STUB_RC:=0}" +[ -n "${STUB_ARGV:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGV" +pos=(); skip=0 +for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + case "$a" in + -jar) skip=1 ;; + -*) ;; + *) pos+=("$a") ;; + esac +done +if [ "$STUB_RC" = 0 ] && [ "${#pos[@]}" -ge 2 ]; then mkdir -p "${pos[1]}"; fi +exit "$STUB_RC" +STUB + chmod +x "$1/java" +} + +# --------------------------------------------------------------------------- +# A throwaway CREGIT tree. The runner takes CREGIT from $(pwd), and it needs the +# tokenize/ scripts (the identity is computed by tokenize/tokenizerIdentity.pl) +# plus each module's declared sources and artifacts, or the staleness guard would +# start real builds. The artifacts are placeholders: the identity digests their +# BYTES, so any content works, and a case that "changes a tokenizer" just writes a +# different string into one. +# --------------------------------------------------------------------------- +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/retokrepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + mkdir -p "$f/$p/src" + done + # Placeholder artifacts, stamped newer than everything above so the staleness + # guard is satisfied and no build runs. + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + # The identity also digests the libsrcml that srcml2token names. + printf '#!/bin/sh\necho "%s/libsrcml"\n' "$f" > "$f/tokenize/srcMLtoken/srcml2token" + echo "placeholder v1" > "$f/libsrcml" + printf '%s' "$f" +} + +# A work directory that looks like a finished run resumable at step 2: the bare +# original, a blob map, a memo, and the derived artifacts a re-fold invalidates. +fixture() { + local w + w=$(mktemp -d "${TMPDIR:-/tmp}/retokwork-XXXXXX") + mkdir -p "$w/memo/ab/cd" "$w/proj-original.git" "$w/blame" "$w/html" + echo "hours of tokenizing" > "$w/proj-blobmap.db" + echo "tokens" > "$w/memo/ab/cd/abcdef" + echo "days of blame" > "$w/blame/one.c.blame" + echo "html" > "$w/html/index.html" + echo "db" > "$w/proj-original.db" + printf '%s' "$w" +} + +run_step2() { # $1 = work dir, rest = extra runner args + FROM_STEP + local w=$1; shift + ( cd "$REPO" \ + && PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --gc none \ + "$@" 2>&1 ) +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/retokbin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: every run passes a tokenizer identity, whether or not it invalidates" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" 2); RC=$? +# Step 2 is what these cases are about. The run goes on to die at step 5, because +# the stubbed java writes no databases; reaching step 3 is the signal that step 2 +# finished and the tokenize gate was satisfied. +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep -q -- '--tokenizer-identity=' "$ARGV"; check "blobExec was given --tokenizer-identity" $? +# It must be a real per-extension map, not an empty string that records nothing. +grep -qE -- '--tokenizer-identity=[a-z0-9+]+=[0-9a-f]{64}' "$ARGV" +check "and the value is ext=sha256 pairs" $? +grep -q -- '--retokenize=' "$ARGV"; [ $? -ne 0 ] +check "and NO --retokenize: nothing is invalidated by default" $? +grep -q -- '--memo-dir=' "$ARGV"; [ $? -ne 0 ] +check "and no --memo-dir either, which blobExec would refuse on its own" $? +[ -f "$W/blame/one.c.blame" ]; check "the blame output was left alone" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: --retokenize is passed through, and always with --memo-dir" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep -q -- '--retokenize=rs' "$ARGV"; check "--retokenize=rs reached blobExec" $? +grep -q -- "--memo-dir=$W/memo" "$ARGV"; check "and --memo-dir named this project's memo" $? +grep -q -- '--tokenizer-identity=' "$ARGV"; check "and the identity too, which blobExec requires" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: a successful invalidation drops the artifacts derived from the re-fold" +# Mandatory, not tidy: a re-tokenized blob changes the tree above it, so every +# cregit commit gets a new sha. Step 6's clone then fails on an existing directory +# and step 7's blame silently keeps .blame files naming commits that are gone. +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +[ ! -f "$W/blame/one.c.blame" ]; check "the stale blame output is gone" $? +[ ! -f "$W/html/index.html" ]; check "the stale html is gone" $? +[ ! -f "$W/proj-original.db" ]; check "the stale git-log db is gone" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map is KEPT — it is what was reused" $? +[ -f "$W/memo/ab/cd/abcdef" ]; check "and so is the memo" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 4: a REFUSED invalidation (exit 7) fails the run and keeps the blame output" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=7 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +[ "$RC" -ne 0 ]; check "the run fails (exit $RC), it does not exit 0" $? +grep -q 'invalidat' <<<"$OUT"; check "and says the invalidation was refused" $? +[ -f "$W/blame/one.c.blame" ]; check "days of blame output survived the refusal" $? +[ -f "$W/proj-blobmap.db" ]; check "so did the blob map" $? +[ ! -f "$W/TOKENIZE-TIMEOUTS" ]; check "no resumable-work marker was written" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 5: --retokenize at step 1 is refused, because step 1 deletes the caches" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs 1); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'FROM_STEP=2' <<<"$OUT"; check "and names the resume it wants" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 6: --retokenize at step 3 is refused, because it would be skipped entirely" +# The invalidation lives in step 2. From step 3 the flag does nothing, the poisoned +# tokens go through steps 3-10, and the run exits 0. +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs 3); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'exactly' <<<"$OUT"; check "and says the step must be exactly 2" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 7: --retokenize with --mode sharded is refused" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs --mode sharded --shards 2 2); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'sharded' <<<"$OUT"; check "and says why" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 8: a malformed extension list is refused before the clone" +for BAD in ".rs" "RS" "r s" "" "rs,.c"; do + W=$(fixture) + OUT=$(STUB_RC=0 run_step2 "$W" --retokenize "$BAD" 2); RC=$? + [ "$RC" -ne 0 ]; check "refused [$BAD] (exit $RC)" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 9: --mask-widened and --retokenize are not combined" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs --mask-widened --mask '\.c$' 2); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 10: --tokenizer-worker reaches blobExec, and only in the pipelined modes" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --tokenizer-worker 2); RC=$? +grep -q -- "--tokenizer-worker=$REPO/tokenizeByBlobId/tokenWorker.pl" "$ARGV" +check "blobExec was given the worker path" $? +OUT=$(STUB_RC=0 run_step2 "$W" --tokenizer-worker --mode serial 2); RC=$? +[ "$RC" -ne 0 ] && grep -q 'needs --mode pipeline' <<<"$OUT"; check "--mode serial is refused (exit $RC)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/t/languages.t b/tests/t/languages.t index a1c62bd62..1096a4fe9 100644 --- a/tests/t/languages.t +++ b/tests/t/languages.t @@ -1,23 +1,17 @@ #!/usr/bin/env perl -# The extension table has been stable since 2017 and needs no test. These six -# assertions cover what this PR adds: a case-insensitive mask that newly selects -# .C and .H, the override that routes those to C++, and a pipeline default -# derived from the table rather than written out a second time. +# The extension table has been stable since 2017 and needs no test. These +# assertions cover the case-insensitive mask, which selects .C and .H, and a +# pipeline default derived from the table rather than written out a second time. use strict; use warnings; -use Test::More tests => 6; +use Test::More tests => 2; use FindBin; use lib "$FindBin::Bin/../../tokenize"; use CregitLanguages; -is(CregitLanguages::language_for_ext('C'), 'C++', ".C routes to C++, per the GNU convention"); -is(CregitLanguages::language_for_ext('H'), 'C++', ".H routes to C++"); -is(CregitLanguages::language_for_ext('c'), 'C', ".c still routes to C"); -is(CregitLanguages::language_for_ext('h'), 'C', ".h still routes to C"); - my $mask = CregitLanguages::file_mask(); ok("src/Matrix.C" =~ /$mask/, "the mask selects .C, which the old \\.[ch]\$ mask did not"); diff --git a/tests/t/tokenizeSrcMl.t b/tests/t/tokenizeSrcMl.t index b29781418..6f1a3d7eb 100644 --- a/tests/t/tokenizeSrcMl.t +++ b/tests/t/tokenizeSrcMl.t @@ -13,8 +13,6 @@ my $srcml2token = "$root/tokenize/srcMLtoken/srcml2token"; my $fixtures = "$root/tokenize/srcMLtoken/tests"; my $expected = "$FindBin::Bin/expected"; -plan skip_all => "srcml not on PATH" - unless system("srcml --version >/dev/null 2>&1") == 0; plan skip_all => "ctags not on PATH" unless system("ctags --version >/dev/null 2>&1") == 0; plan skip_all => "srcml2token not built (cd tokenize/srcMLtoken && make)" @@ -79,7 +77,7 @@ sub run_tokenizer { { my $crasher = "$FindBin::Bin/fixtures/srcml-position-crash.c"; - my $rawRc = system("srcml -l C --position '$crasher' >/dev/null 2>&1"); + my $rawRc = system("'$srcml2token' -l C '$crasher' >/dev/null 2>&1"); my $rawSignal = $rawRc & 127; if (not $rawSignal and ($rawRc >> 8) > 128) { @@ -87,12 +85,12 @@ sub run_tokenizer { } SKIP: { - skip "srcml no longer dies on this fixture (rc=$rawRc): the upstream bug " + skip "srcml2token no longer dies on this fixture (rc=$rawRc): the upstream bug " . "appears fixed, so the crash-handling assertions cannot be exercised", 6 unless $rawSignal; is($rawSignal == 11 || $rawSignal == 6, 1, - "fixture kills srcml with SIGSEGV(11) or SIGABRT(6), got signal $rawSignal"); + "fixture kills srcml2token with SIGSEGV(11) or SIGABRT(6), got signal $rawSignal"); my ($status, $out, $err) = run_tokenizer("--position", "'$crasher'"); @@ -104,14 +102,13 @@ sub run_tokenizer { "no tokenization is emitted on stdout"); like($err, qr/FAILED/, "the failure is reported on stderr instead of passing silently"); - like($err, qr/killed by signal $rawSignal/, - "stderr names the signal, so srcml's death is attributed to srcml " - . "and not to srcml2token, which exits 0 on the truncated XML"); + like($err, qr/srcml2token was killed by signal $rawSignal/, + "stderr names the signal"); } } { - my $dies = "$workdir/srcml-that-dies"; + my $dies = "$workdir/srcml2token-that-dies"; open(my $fh, '>', $dies) or die $!; print $fh "#!/bin/sh\nkill -SEGV \$\$\n"; close $fh; @@ -123,10 +120,10 @@ sub run_tokenizer { close $fh; my ($status, $out, $err) = - run_tokenizer("--srcml='$dies'", "--position", "'$victim'"); + run_tokenizer("--srcml2token='$dies'", "--position", "'$victim'"); is($status >> 8, $PARSER_CRASH_EXIT, - "a srcml killed by a signal exits $PARSER_CRASH_EXIT, whatever srcml we ship"); + "a srcml2token killed by a signal exits $PARSER_CRASH_EXIT, whatever srcML we ship"); is($out, "", "and emits no tokenization"); like($err, qr/killed by signal 11/, "and names the signal on stderr"); } @@ -160,6 +157,6 @@ sub run_tokenizer { close $fh; my ($status, $out, $err) = run_tokenizer("--position", "\"$odd\""); - is($status, 0, "a path containing a quote is passed to the shell safely"); + is($status, 0, "a path containing a quote tokenizes"); like($out, qr/begin_unit/, "and really was tokenized"); } diff --git a/tests/t/tokenizerIdentity.t b/tests/t/tokenizerIdentity.t new file mode 100644 index 000000000..18fb46ddb --- /dev/null +++ b/tests/t/tokenizerIdentity.t @@ -0,0 +1,218 @@ +#!/usr/bin/env perl + +# Tests for tokenize/tokenizerIdentity.pl, the value blobExec compares to decide +# whether a cached tokenization was made by the tokenizer this run is using. +# +# What has to be true of it, and why: +# +# 1. It changes when a tokenizer changes. That is the whole point: the defect +# it exists for is that `command` (the constant path +# tokenizeByBlobId/tokenBySha.pl) and `mask` both stayed the same while the +# Rust tokenizer's output format was corrected, so every cached .rs row +# stayed a cache hit. +# 2. It does NOT change when nothing changes. An identity that moved on every +# run would refuse every resume, which is indistinguishable from having no +# cache at all. +# 3. It changes the identity of the RIGHT extensions and no others. Selectivity +# is the difference between re-tokenizing 741,869 .rs entries and +# re-tokenizing the whole corpus at 88% of total pipeline time. +# 4. A component it cannot read is fatal, never omitted. An identity computed +# without a component compares EQUAL across a change in that component, +# which is exactly the silence being removed. +# 5. blobExec can parse what it prints. + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); +use File::Path qw(make_path); +use File::Copy; + +use lib "$FindBin::Bin/../../tokenize"; +use CregitLanguages; + +my $realTokenizeDir = "$FindBin::Bin/../../tokenize"; + +# A throwaway copy of tokenize/ with stand-in "binaries", so a test can change a +# tokenizer without touching the checkout or building anything. +sub fixture { + my $dir = tempdir(CLEANUP => 1); + my $tok = "$dir/tokenize"; + make_path($tok); + for my $f (qw(tokenize.pl CregitLanguages.pm tokenizerIdentity.pl tokenizeSrcMl.pl CregitSrcMl.pm)) { + copy("$realTokenizeDir/$f", "$tok/$f") or die "copy $f: $!"; + } + chmod 0755, "$tok/tokenizerIdentity.pl", "$tok/tokenize.pl", "$tok/tokenizeSrcMl.pl"; + + # Stand-ins for the parsers and the external binaries. Content is what the + # identity digests, so "changing a tokenizer" is writing a different string. + make_path("$tok/rustTokenizer/target/release"); + write_file("$tok/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v1\n"); + make_path("$tok/m4Tokenizer"); + write_file("$tok/m4Tokenizer/m4.py", "M4 v1\n"); + write_srcml2token($dir, "v1"); + write_file("$dir/libsrcml", "LIBSRCML v1\n"); + write_file("$dir/ctags", "CTAGS v1\n"); + return $dir; +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die "write $path: $!"; + print $fh $content; + close $fh; + chmod 0755, $path; +} + +sub write_srcml2token { + my ($dir, $version) = @_; + write_file("$dir/srcml2token", "#!/bin/sh\n# SRCML2TOKEN $version\necho '$dir/libsrcml'\n"); +} + +sub identity { + my ($dir, @extra) = @_; + my @cmd = ("perl", "$dir/tokenize/tokenizerIdentity.pl", + "--srcml2token=$dir/srcml2token", + "--ctags=$dir/ctags", @extra); + open(my $ph, '-|', @cmd) or die "run: $!"; + my $out = do { local $/; <$ph> }; + close $ph; + my $status = $?; + $out = '' unless defined $out; + chomp $out; + return ($out, $status); +} + +sub as_map { + my ($spec) = @_; + return map { split(/=/, $_, 2) } split(/,/, $spec); +} + +# --------------------------------------------------------------------------- +my $dir = fixture(); +my ($base, $status) = identity($dir); +is($status, 0, "tokenizerIdentity.pl succeeds on a complete checkout"); + +my %baseMap = as_map($base); + +# 3 — coverage. Every masked extension and nothing else, because those are the +# extensions the pipeline's mask can select and therefore the ones blob_map can +# hold rows for. +is_deeply( + [sort keys %baseMap], + [CregitLanguages::masked_extensions_sorted()], + "every masked extension gets an identity, and only those"); + +# 5 — the format blobExec's TokenizerIdentity.parse accepts. +for my $ext (sort keys %baseMap) { + like($ext, qr/^[a-z0-9+]+$/, "extension [$ext] is lowercase and dotless"); + like($baseMap{$ext}, qr/^[0-9a-f]{64}$/, "identity for [$ext] is a sha256 hex digest"); +} + +# 2 — stability. Run twice, unchanged, and get the same string. +my ($again) = identity($dir); +is($again, $base, "an unchanged checkout yields a byte-identical identity"); + +# --------------------------------------------------------------------------- +# 1 and 3 — the Rust tokenizer changes, and ONLY .rs moves. This is the exact +# shape of the live defect: a corrected rustTokenizer binary, everything else +# untouched. +write_file("$dir/tokenize/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v2 fixed\n"); +my ($afterRust) = identity($dir); +my %rustMap = as_map($afterRust); + +isnt($rustMap{rs}, $baseMap{rs}, "rebuilding the Rust tokenizer moves the .rs identity"); +for my $ext (grep { $_ ne 'rs' } sort keys %baseMap) { + is($rustMap{$ext}, $baseMap{$ext}, + "and leaves [$ext] alone, so its tokenizations are not invalidated"); +} + +# --------------------------------------------------------------------------- +# 3, the other direction — the srcML chain moves the srcML-routed extensions and +# not Rust. Each of the three components counts: the token stream is the product +# of all of them. +for my $component (qw(srcml2token libsrcml ctags)) { + my $d = fixture(); + my ($before) = identity($d); + if ($component eq "srcml2token") { write_srcml2token($d, "v2") } + else { write_file("$d/$component", "$component v2\n") } + my ($after) = identity($d); + my %b = as_map($before); + my %a = as_map($after); + isnt($a{c}, $b{c}, "changing $component moves the .c identity"); + is($a{rs}, $b{rs}, "changing $component leaves .rs alone"); +} + +# tokenizeSrcMl.pl itself, and CregitSrcMl.pm, its copy in the tokenizer worker. +for my $script (qw(tokenizeSrcMl.pl CregitSrcMl.pm)) { + my $d = fixture(); + my ($before) = identity($d); + write_file("$d/tokenize/$script", "# v2\n"); + my ($after) = identity($d); + my %b = as_map($before); + my %a = as_map($after); + isnt($a{c}, $b{c}, "editing $script moves the .c identity"); + is($a{rs}, $b{rs}, "and leaves .rs alone"); +} + +# --------------------------------------------------------------------------- +# The shared files move EVERY extension, deliberately. tokenize.pl decides which +# parser runs and with which flags (--position, which the token format depends +# on) and CregitLanguages.pm decides which language an extension is, so either can +# change an extension's tokens with no parser change at all. Blunt in the safe +# direction: a refusal costs a conversation, a false "unchanged" costs a corpus. +for my $shared (qw(tokenize.pl CregitLanguages.pm)) { + my $d = fixture(); + my ($before) = identity($d); + my %b = as_map($before); + open(my $fh, '>>', "$d/tokenize/$shared") or die $!; + print $fh "\n# a change\n"; + close $fh; + my ($after) = identity($d); + my %a = as_map($after); + for my $ext (sort keys %b) { + isnt($a{$ext}, $b{$ext}, "editing $shared moves [$ext] too"); + } +} + +# --------------------------------------------------------------------------- +# 4 — a missing component is fatal, and says which file and why. +{ + my $d = fixture(); + unlink("$d/tokenize/rustTokenizer/target/release/rust_tokenizer"); + my @cmd = ("perl", "$d/tokenize/tokenizerIdentity.pl", + "--srcml2token=$d/srcml2token", "--ctags=$d/ctags"); + my $err = `@cmd 2>&1 1>/dev/null`; + isnt($?, 0, "an unbuilt Rust tokenizer is fatal, not omitted"); + like($err, qr/rust_tokenizer/, "and the message names the file"); +} + +{ + my $d = fixture(); + my @cmd = ("perl", "$d/tokenize/tokenizerIdentity.pl", "--ctags=$d/ctags"); + my $err = `@cmd 2>&1 1>/dev/null`; + isnt($?, 0, "omitting --srcml2token is fatal: .c tokens depend on it"); + like($err, qr/--srcml2token/, "and the message names the flag"); +} + +{ + my $d = fixture(); + write_file("$d/srcml2token", "#!/bin/sh\nexit 0\n"); + my ($out, $status) = identity($d); + isnt($status, 0, "an srcml2token that does not name its libsrcml is fatal"); + is($out, "", "and prints no identity"); +} + +# --------------------------------------------------------------------------- +# --all-extensions covers the routed-but-unmasked ones (.am/.ac), so the flag can +# be used the day the m4 lexer is fixed and M4 joins the mask. +{ + my ($all) = identity($dir, "--all-extensions"); + my %allMap = as_map($all); + is_deeply([sort keys %allMap], [CregitLanguages::extensions_sorted()], + "--all-extensions covers every extension in the table"); + ok(exists $allMap{am}, "including .am, which is routed to m4 but not masked"); +} + +done_testing(); diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index f29377be0..0337ed754 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -11,14 +11,17 @@ import json import sqlite3 +from collections import Counter import pytest -from generate_dataset import (DEFAULT_MEMORY_LIMIT, PROJECT_META_FIELDS, - SourceReader, is_ws, load_project_meta, - parse_memory_limit, project_meta_sql, - skip_comment, skip_literal, skip_token, - sql_literal) +from generate_dataset import (DEFAULT_MEMORY_LIMIT, FIRM_FIELDS, + PROJECT_META_FIELDS, SourceReader, + check_key_is_unique, firm_sql, is_ws, + load_project_meta, parse_memory_limit, + process_blame_file, project_meta_sql, + repair_token_line, skip_comment, skip_literal, skip_token, + sql_literal, undo_mojibake) def reader(text: str) -> SourceReader: @@ -301,11 +304,11 @@ def test_a_key_absent_from_the_sidecar_fails_loudly(tmp_path): # --------------------------------------------------------------------------- # # end to end, against DuckDB. The unit tests above prove the SQL text; only a -# written Parquet proves the column set, and 67 columns is the contract +# written Parquet proves the column set, and 70 columns is the contract # cregit-token-pipeline/validate_schema.py gates the corpus with. # --------------------------------------------------------------------------- # -TOTAL_COLUMNS = 67 # 38 token/commit columns + 29 metadata +TOTAL_COLUMNS = 70 # 38 token/commit + 29 metadata + 3 firm SHA = "a" * 40 @@ -450,45 +453,478 @@ def test_an_unknown_project_key_stops_the_run_before_phase_1(monkeypatch, tmp_pa # --------------------------------------------------------------------------- # # footer trailers resolve to personids +# --------------------------------------------------------------------------- # -def seed_trailer(value, *, email_rows=()): - """Give the tiny project one Reviewed-by trailer and the emails rows given.""" +def seed_footers(*trailers, people=()): + """Return a `seed` for generate(): trailers into footers, people into + emails/persons. `people` is (personid, emailaddr, personname).""" def seed(cregit_db, persons_db): con = sqlite3.connect(cregit_db) - con.execute("INSERT INTO footers VALUES (?,?,?,?)", - (SHA, 0, "Reviewed-by", value)) + for idx, (key, value) in enumerate(trailers): + con.execute("INSERT INTO footers VALUES (?,?,?,?)", + (SHA, idx, key, value)) con.commit() con.close() - con = sqlite3.connect(persons_db) - for personid, emailaddr, emailname in email_rows: - con.execute( - "INSERT INTO emails (personid, fullemail, emailaddr, emailname, " - "lcemail, userid, domain) VALUES (?,?,?,?,?,?,?)", - (personid, f"{emailname} <{emailaddr}>", emailaddr, emailname, - emailaddr.lower(), personid, emailaddr.partition("@")[2])) - con.execute("INSERT INTO persons VALUES (?,?)", (personid, emailname)) + for i, (pid, addr, pname) in enumerate(people, start=1): + con.execute("INSERT INTO emails (recordid, personid, fullemail, " + "emailaddr, emailname) VALUES (?,?,?,?,?)", + (i, pid, f"{pname} <{addr}>", addr, pname)) + con.execute("INSERT OR IGNORE INTO persons VALUES (?,?)", (pid, pname)) con.commit() con.close() return seed -def footer_personids(duckdb, out): - return duckdb.sql("select footer_personids from read_parquet(?)", - params=[str(out)]).fetchall()[0][0] +def footers_of(duckdb, path): + return duckdb.sql("select footer_personids, footer_person_names, " + f"footer_reviewed_by from read_parquet('{path}')").fetchone() + + +@pytest.mark.parametrize("value", ["bob@example.com", "Bob bob@example.com"]) +def test_a_bare_address_trailer_resolves_to_a_personid(monkeypatch, tmp_path, value): + """`Reviewed-by: bob@example.com`, no angle brackets. The trailer text + reached footer_reviewed_by while footer_personids stayed empty, because the + coalesce never reached the bare-address pattern and that pattern had no + capture group anyway.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", value), + people=[("bob", "bob@example.com", "Bob B")])) + + ids, names, reviewed = footers_of(duckdb, out) + assert list(reviewed) == [value] + assert list(ids) == ["bob"] + assert list(names) == ["Bob B"] + + +def test_the_angle_bracket_form_still_wins_over_a_second_address(monkeypatch, tmp_path): + """Precedence, which the nullif must not disturb: a trailer holding both + forms resolves to the bracketed one.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", "noreply@example.com writing for Bob B "), + people=[("bob", "bob@example.com", "Bob B"), + ("noreply", "noreply@example.com", "No Reply")])) + + ids, _names, _reviewed = footers_of(duckdb, out) + assert list(ids) == ["bob"] + + +def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): + """`Former-commit-id: ` carries no address, so it must contribute no + person. It used to join on '', which matches an emails row for an author + committing as `Name <>` — 27 of 199 corpus persons DBs hold one, so this + published a wrong personid on every commit carrying such a trailer.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Former-commit-id", "6a32a91a877bc2341810ef674dfdc3be21c500bc"), + people=[("ghost", "", "A Ghost")])) + + ids, names, _reviewed = footers_of(duckdb, out) + assert list(ids) == [] + assert list(names) == [] + + +# --------------------------------------------------------------------------- # +# firm attribution. The shape is different from everything above: the 29 +# metadata columns are per-project CONSTANTS injected as SQL literals, while +# firm is PER ROW and comes from a real join against an external CSV. So the +# failure modes are different too — a duplicate key in either lookup table +# multiplies token rows through the LEFT JOIN, and nothing downstream notices. +# --------------------------------------------------------------------------- # + +FIRM_MAP_HEADER = "domain,company,kind,source\n" + + +def write_map(tmp_path, *lines, name="firm.csv"): + path = tmp_path / name + path.write_text(FIRM_MAP_HEADER + "".join(f"{l}\n" for l in lines)) + return path + + +def write_canonical(tmp_path, *lines, name="canon.csv"): + path = tmp_path / name + path.write_text("firm_raw,firm,decision,note\n" + + "".join(f"{l}\n" for l in lines)) + return path + + +def generate_with_domain(monkeypatch, tmp_path, domain, argv_extra=()): + """The tiny project, plus one identified person on `domain`. + + build_tiny_project leaves emails and persons empty, so person_domain is NULL + there and every firm column is blank whatever the map says. A positive test + needs a person the commit actually joins to. + """ + import generate_dataset as gd + + blame, src, cregit_db, persons_db = build_tiny_project(tmp_path) + con = sqlite3.connect(persons_db) + con.execute( + "INSERT INTO emails (personid, fullemail, emailaddr, emailname, " + "lcemail, userid, domain) VALUES (?,?,?,?,?,?,?)", + ("p1", "A Dev ", "a@example.com", "A Dev", + "a@example.com", "a", domain)) + con.execute("INSERT INTO persons VALUES ('p1', 'A Dev')") + con.commit() + con.close() + + out = tmp_path / "out" / "proj-dataset.parquet" + monkeypatch.setattr(gd.sys, "argv", [ + "generate_dataset.py", + "--blame-dir", str(blame), + "--source-dir", str(src), + "--cregit-db", str(cregit_db), + "--persons-db", str(persons_db), + "--output", str(out), + "--repo-name", "proj", + *argv_extra, + ]) + gd.main() + return out + + +def firm_of(duckdb, path): + return duckdb.sql("select person_domain, firm_raw, firm, firm_source " + f"from read_parquet('{path}')").fetchall() -def test_a_trailer_carrying_a_bare_address_resolves_to_its_person(monkeypatch, tmp_path): +def test_the_firm_field_list_is_three_names_in_dataset_order(): + assert FIRM_FIELDS == ("firm_raw", "firm", "firm_source") + + +def test_no_firm_map_emits_no_join_and_three_empty_literals(): + """An older caller must still produce a schema-valid file, the same bargain + --project-meta makes. So the columns exist and the join does not.""" + select, join = firm_sql("", "") + assert join == "" + assert select.count("'' AS") == 3 + + +def test_a_map_without_a_canonical_table_makes_firm_repeat_firm_raw(): + """Honest rather than clever: without a reviewed table the split spellings + stay split, and `firm` says the same thing `firm_raw` does.""" + select, join = firm_sql("/m.csv", "") + assert "read_csv_auto('/m.csv'" in join + assert "fc" not in join + assert "coalesce(fm.company, '') AS firm," in select + + +def test_the_map_path_is_escaped_like_every_other_literal(): + """The query is one f-string and the path comes from the command line.""" + _select, join = firm_sql("/o'brien/m.csv", "") + assert "'/o''brien/m.csv'" in join + + +def test_the_three_firm_columns_sit_between_person_domain_and_repo_tag( + monkeypatch, tmp_path): + """Position is a claim, not a convenience: firm is resolved FROM + person_domain, so the key and its answers are adjacent.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path) + names = [n for n, _ in schema_of(duckdb, out)] + i = names.index("person_domain") + assert names[i:i + 5] == ["person_domain", "firm_raw", "firm", + "firm_source", "repo_tag"] + assert len(names) == TOTAL_COLUMNS + + +def test_a_call_with_no_firm_flags_writes_three_empty_strings( + monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path) + assert duckdb.sql("select firm_raw, firm, firm_source " + f"from read_parquet('{out}')").fetchone() == ("", "", "") + + +def test_a_domain_in_the_map_gets_its_firm_and_its_source(monkeypatch, tmp_path): + """The headline case, in miniature: the map says who, and firm_source says on + what evidence. `correction` is this repository's reviewed overlay.""" + duckdb = pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "qti.qualcomm.com,Qualcomm,company,correction") + out = generate_with_domain(monkeypatch, tmp_path, "qti.qualcomm.com", + argv_extra=("--firm-map", str(firm_map))) + assert firm_of(duckdb, out) == [ + ("qti.qualcomm.com", "Qualcomm", "Qualcomm", "correction")] + + +def test_the_canonical_table_fills_firm_and_never_touches_firm_raw( + monkeypatch, tmp_path): + """The partner's standing preference: carry more, cut at publication. The raw + string is evidence and must survive beside the canonical name.""" + duckdb = pytest.importorskip("duckdb") + firm_map = write_map( + tmp_path, "au1.ibm.com,International Business Machines,company,cncf-gitdm") + canon = write_canonical( + tmp_path, "International Business Machines,IBM,merge,one firm spelled two ways") + out = generate_with_domain(monkeypatch, tmp_path, "au1.ibm.com", + argv_extra=("--firm-map", str(firm_map), + "--firm-canonical", str(canon))) + assert firm_of(duckdb, out) == [ + ("au1.ibm.com", "International Business Machines", "IBM", "cncf-gitdm")] + + +def test_a_name_the_canonical_table_does_not_mention_passes_through( + monkeypatch, tmp_path): + """The table lists only the names that change. Everything else is already + canonical, and a missing row must not blank the column.""" + duckdb = pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") + canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") + out = generate_with_domain(monkeypatch, tmp_path, "google.com", + argv_extra=("--firm-map", str(firm_map), + "--firm-canonical", str(canon))) + assert firm_of(duckdb, out) == [ + ("google.com", "Google", "Google", "gitdm")] + + +def test_a_domain_absent_from_the_map_gets_three_empty_strings( + monkeypatch, tmp_path): + """An empty firm_source is the filter for 'not attributed at all', so it must + mean exactly that rather than 'the map had no source column'.""" + duckdb = pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") + out = generate_with_domain(monkeypatch, tmp_path, "nowhere.example", + argv_extra=("--firm-map", str(firm_map))) + assert firm_of(duckdb, out) == [("nowhere.example", "", "", "")] + + +def test_the_domain_match_ignores_case(monkeypatch, tmp_path): + """build_domain_map writes lower-cased domains, but persons.db carries + whatever the commit's e-mail header held.""" duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path, seed=seed_trailer( - "Bob bob@example.com", email_rows=[("p_bob", "bob@example.com", "Bob")])) - assert footer_personids(duckdb, out) == ["p_bob"] + firm_map = write_map(tmp_path, "redhat.com,Red Hat,company,gitdm") + out = generate_with_domain(monkeypatch, tmp_path, "RedHat.COM", + argv_extra=("--firm-map", str(firm_map))) + assert firm_of(duckdb, out) == [("RedHat.COM", "Red Hat", "Red Hat", "gitdm")] -def test_a_trailer_with_no_address_does_not_match_an_empty_emails_row(monkeypatch, tmp_path): +def test_a_company_spelled_like_a_number_stays_a_string(monkeypatch, tmp_path): + """all_varchar=true on the read. Without it DuckDB sniffs `360` as a number, + the firm columns change type, and validate_schema.py fails the whole corpus + on one project's map hit.""" duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path, seed=seed_trailer( - "Former-commit-id: 0123456789abcdef0123456789abcdef01234567", - email_rows=[("p_empty", "", "Nameless")])) - assert footer_personids(duckdb, out) == [] + firm_map = write_map(tmp_path, "360.cn,360,company,gitdm") + out = generate_with_domain(monkeypatch, tmp_path, "360.cn", + argv_extra=("--firm-map", str(firm_map))) + assert firm_of(duckdb, out) == [("360.cn", "360", "360", "gitdm")] + assert dict(schema_of(duckdb, out))["firm_raw"] == "VARCHAR" + + +def test_one_token_stays_one_row_when_the_map_matches(monkeypatch, tmp_path): + """The join must not fan out. This is the assertion that would catch a future + map keyed on something less unique than a domain.""" + duckdb = pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") + canon = write_canonical(tmp_path, "Google,Google LLC,merge,irrelevant here") + out = generate_with_domain(monkeypatch, tmp_path, "google.com", + argv_extra=("--firm-map", str(firm_map), + "--firm-canonical", str(canon))) + assert duckdb.sql(f"select count(*) from read_parquet('{out}')").fetchone() == (1,) + + +def test_a_repeated_domain_in_the_map_stops_the_run_before_phase_1( + monkeypatch, tmp_path): + """The one failure mode of this join that would be invisible. Two rows for one + domain duplicate every token row of every person on it: the file still + validates, the schema still matches, and only the row count betrays it.""" + pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "google.com,Google,company,gitdm", + "Google.com,Alphabet,company,gitdm") + with pytest.raises(SystemExit, match="multiplies token rows"): + generate_with_domain(monkeypatch, tmp_path, "google.com", + argv_extra=("--firm-map", str(firm_map))) + assert not (tmp_path / "out" / "proj-dataset.parquet").exists() + + +def test_a_repeated_name_in_the_canonical_table_stops_the_run( + monkeypatch, tmp_path): + """Two canonical names for one raw string is an unresolved review, not a + default to pick from.""" + pytest.importorskip("duckdb") + firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") + canon = write_canonical(tmp_path, "Google,Alphabet,merge,one reviewer", + "Google,Google LLC,merge,another reviewer") + with pytest.raises(SystemExit, match="multiplies token rows"): + generate_with_domain(monkeypatch, tmp_path, "google.com", + argv_extra=("--firm-map", str(firm_map), + "--firm-canonical", str(canon))) + + +def test_check_key_is_unique_returns_the_row_count(tmp_path): + path = write_map(tmp_path, "a.example,A,company,gitdm", + "b.example,B,company,gitdm") + assert check_key_is_unique(path, "domain", "the firm map") == 2 + + +def test_a_canonical_table_without_a_map_is_refused(monkeypatch, tmp_path): + """argparse exits 2. There is no firm_raw to canonicalise without a map, and + accepting the pair would write `firm` out of nothing.""" + canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, + argv_extra=("--firm-canonical", str(canon))) + + +def test_a_missing_firm_map_stops_the_run_before_phase_1(monkeypatch, tmp_path): + """A typo in the path must not produce a corpus of blank firm columns.""" + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, + argv_extra=("--firm-map", str(tmp_path / "absent.csv"))) + + +def test_a_missing_canonical_table_stops_the_run_before_phase_1( + monkeypatch, tmp_path): + firm_map = write_map(tmp_path, "a.example,A,company,gitdm") + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, + argv_extra=("--firm-map", str(firm_map), + "--firm-canonical", str(tmp_path / "absent.csv"))) + + +# --------------------------------------------------------------------------- # +# Token-line repairs before the source walk: srcML mojibake, the Rust position +# prefix, and the byte-order mark. Each one moved the walk, so each test checks +# the text and the position of the token after the defect. +# --------------------------------------------------------------------------- # + +def walk(tmp_path, name, source: bytes, *token_lines, stats=None): + """Run process_blame_file on one file and return its token_map rows.""" + (tmp_path / name).write_bytes(source) + blame = tmp_path / f"{name}.blame" + blame.write_text("".join(f"{SHA};;\t{t}\n" for t in token_lines), + encoding="utf-8") + con = sqlite3.connect(":memory:") + con.execute("""CREATE TABLE token_map (file_path TEXT, token_index INTEGER, + commit_sha CHAR(40), token_type TEXT, token_value TEXT, + source_text TEXT, source_line INTEGER, source_col INTEGER, + is_structural INTEGER, func_name TEXT)""") + process_blame_file(blame, tmp_path / name, name, con.cursor(), stats) + return con.execute("""SELECT token_type, token_value, source_text, + source_line, source_col, is_structural FROM token_map + ORDER BY token_index""").fetchall() + + +C_SOURCE = "/* Högskolan */\nint x;\n".encode() +C_MOJIBAKE = ("comment|/* Högskolan */", "keyword|int", "name|x", + "operator|;") + + +def test_undo_mojibake_restores_the_utf8_text(): + assert undo_mojibake("Högskolan") == "Högskolan" + assert undo_mojibake("â\x80\x94") == "\u2014" + + +@pytest.mark.parametrize("text", [ + "Högskolan", # correct text: o-umlaut is above the lead-byte range + "é¿", # a lead byte with too few continuation bytes + "Āö", # a character above U+00FF cannot be Latin-1 + "plain ascii", +]) +def test_undo_mojibake_leaves_other_text_alone(text): + assert undo_mojibake(text) == text + + +def test_a_mojibake_token_gets_the_true_text_and_the_walk_stays_aligned(tmp_path): + rows = walk(tmp_path, "a.c", C_SOURCE, *C_MOJIBAKE) + assert rows[0][1] == "/* Högskolan */" + assert rows[0][2].startswith("/* Högskolan */") + # The next token is where the source has it: line 2, column 1. + assert rows[1][1:5] == ("int", "int ", 2, 1) + assert rows[2][3:5] == (2, 5) + + +def test_a_correct_token_is_not_changed(tmp_path): + rows = walk(tmp_path, "a.c", C_SOURCE, "comment|/* Högskolan */", + *C_MOJIBAKE[1:]) + assert rows[0][1] == "/* Högskolan */" + assert rows[1][3:5] == (2, 1) + + +@pytest.mark.parametrize("name, source", [ + # srcML decodes a BOM file correctly, so the pair is the true text. + ("a.c", b"\xef\xbb\xbf/* H\xc3\x83\xc2\xb6 */\nint x;\n"), + # Not UTF-8: nothing tells what the true text is. + ("a.c", b"/* H\xc3\xb6 \xff */\nint x;\n"), + # The Rust tokenizer does not use srcML. + ("a.rs", "/* Hö */\nint x;\n".encode()), +]) +def test_the_gate_keeps_a_matching_token_as_it_is(tmp_path, name, source): + rows = walk(tmp_path, name, source, "comment|/* Hö */", "keyword|int") + assert rows[0][1] == "/* Hö */" + + +def test_a_rust_line_loses_its_position_prefix(tmp_path): + rows = walk(tmp_path, "m.rs", b"fn main() {}\n", + "-:-\tbegin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", + "1:1\tkeyword|fn", "1:4\tidentifier|main", "1:8\top|(", + "1:9\top|)", "1:11\top|{", "1:12\top|}", "-:-\tend_unit") + assert [r[0] for r in rows] == ["begin_unit", "keyword", "identifier", + "op", "op", "op", "op", "end_unit"] + assert rows[0][5] == 1 and rows[-1][5] == 1 + # The walk now agrees with the position the tokenizer wrote. + assert [r[3:5] for r in rows[1:7]] == [(1, 1), (1, 4), (1, 8), (1, 9), + (1, 11), (1, 12)] + assert rows[2][2] == "main" + + +def test_a_decl_line_prefix_is_removed_too(tmp_path): + rows = walk(tmp_path, "m.rs", b"fn f\n", "1:-\tDECL|fn|f", + "1:1\tkeyword|fn") + assert rows[0][0] == "DECL" + assert rows[1][3:5] == (1, 1) + + +def test_a_c_line_that_starts_with_digits_is_kept(tmp_path): + rows = walk(tmp_path, "a.c", b"1\n", "literal|1") + assert rows[0][:2] == ("literal", "1") + + +def test_a_byte_order_mark_does_not_move_the_walk(tmp_path): + rows = walk(tmp_path, "a.c", b"\xef\xbb\xbfint x;\n", "keyword|int", + "name|x", "operator|;") + assert rows[0][2:5] == ("int ", 1, 1) + assert rows[1][2:5] == ("x", 1, 5) + + +def test_windows_line_ends_still_read_as_one_newline(tmp_path): + rows = walk(tmp_path, "a.c", b"int\r\nx;\r\n", "keyword|int", "name|x") + assert rows[1][3:5] == (2, 1) + + +def test_the_repairs_are_counted(tmp_path): + stats = Counter() + walk(tmp_path, "a.c", C_SOURCE, *C_MOJIBAKE, stats=stats) + walk(tmp_path, "m.rs", b"fn\n", "1:1\tkeyword|fn", "-:-\tend_unit", + stats=stats) + walk(tmp_path, "b.c", b"\xef\xbb\xbfint\n", "keyword|int", stats=stats) + assert stats == Counter(mojibake_tokens=1, position_prefix=2, bom_files=1) + + +def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path): + # The tokenizer no longer writes the prefix without --position. Such a + # line must pass unchanged: the repair removes text, it never moves a + # position by a fixed amount. + stats = Counter() + rows = walk(tmp_path, "m.rs", b"fn main\n", + "begin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", + "keyword|fn", "identifier|main", "end_unit", stats=stats) + assert [r[0] for r in rows] == ["begin_unit", "keyword", "identifier", + "end_unit"] + assert [r[3:5] for r in rows[1:3]] == [(1, 1), (1, 4)] + assert stats["position_prefix"] == 0 + + +@pytest.mark.parametrize("line, repairable, expected", [ + ("12:5\tkeyword|fn", False, "keyword|fn"), + ("12:-\tDECL|fn|f", False, "DECL|fn|f"), + ("-:-\tend_unit", False, "end_unit"), + ("keyword|fn", False, "keyword|fn"), + ("comment|Hö", True, "comment|Hö"), + ("comment|Hö", False, "comment|Hö"), + ("12:5|keyword|fn", False, "12:5|keyword|fn"), # --position form: a pipe +]) +def test_repair_token_line(line, repairable, expected): + assert repair_token_line(line, repairable, Counter()) == expected diff --git a/tests/test_token_worker.sh b/tests/test_token_worker.sh new file mode 100755 index 000000000..d01f46397 --- /dev/null +++ b/tests/test_token_worker.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Run inside `devenv shell`: it needs perl, python3, srcml, ctags and cargo. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +SRCML2TOKEN="$ROOT/tokenize/srcMLtoken/srcml2token" +[[ -x "$SRCML2TOKEN" ]] || make -C "$ROOT/tokenize/srcMLtoken" +[[ -x "$ROOT/tokenize/rustTokenizer/target/release/rust_tokenizer" ]] || make -C "$ROOT/tokenize/rustTokenizer" + +TEMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/cregit-token-worker.XXXXXX") +trap 'rm -rf "$TEMP_ROOT"' EXIT + +FIXTURES=("$ROOT"/tests/fixtures/* "$ROOT/tests/t/fixtures/srcml-position-crash.c") +python3 "$ROOT/tests/token_worker_driver.py" \ + --worker "$ROOT/tokenizeByBlobId/tokenWorker.pl" \ + --token-by-sha "$ROOT/tokenizeByBlobId/tokenBySha.pl" \ + --tokenize-command "$ROOT/tokenize/tokenize.pl --srcml2token=$SRCML2TOKEN --ctags=$(command -v ctags)" \ + "${FIXTURES[@]/#/--fixture=}" \ + --temp-root "$TEMP_ROOT" diff --git a/tests/token_worker_driver.py b/tests/token_worker_driver.py new file mode 100755 index 000000000..a7b94f943 --- /dev/null +++ b/tests/token_worker_driver.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Checks tokenizeByBlobId/tokenWorker.pl against tokenizeByBlobId/tokenBySha.pl.""" + +import argparse +import hashlib +import os +import pathlib +import shutil +import subprocess +import time + +TIMEOUT_EXIT = 124 +PARSER_CRASH_EXIT = 33 + + +class Worker: + def __init__(self, executable, memo_dir, tokenize_command): + env = dict(os.environ, BFG_MEMO_DIR=str(memo_dir), BFG_TOKENIZE_CMD=tokenize_command) + self.process = subprocess.Popen( + [str(executable)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + ready = self.process.stdout.readline() + if ready != b"READY\n": + raise AssertionError(f"worker did not start: {ready!r}") + + def request(self, filename, contents, timeout_secs=20): + name = os.fsencode(filename) + header = f"REQ {len(contents)} {len(name)} {timeout_secs}\n".encode() + self.process.stdin.write(header + name + contents) + self.process.stdin.flush() + response = self.process.stdout.readline().decode().split() + if len(response) != 4 or response[0] != "RES": + raise AssertionError(f"malformed response header: {response!r}") + exit_code, out_len, err_len = map(int, response[1:]) + return exit_code, self.process.stdout.read(out_len), self.process.stdout.read(err_len) + + def close(self): + self.process.stdin.close() + if self.process.wait(timeout=5) != 0: + raise AssertionError(f"worker exited {self.process.returncode} on EOF") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + print(f"PASS {message}") + + +def token_by_sha(args, memo_dir, fixture): + env = dict( + os.environ, + BFG_FILENAME=fixture.name, + BFG_MEMO_DIR=str(memo_dir), + BFG_TOKENIZE_CMD=args.tokenize_command, + ) + return subprocess.run( + [str(args.token_by_sha)], input=fixture.read_bytes(), capture_output=True, env=env + ) + + +def memo_file(memo_dir, contents): + digest = hashlib.sha1(contents).hexdigest() + return memo_dir / digest[:2] / digest[2:4] / digest + + +def new_dir(args, name): + path = args.temp_root / name + path.mkdir() + return path + + +def write_script(path, body): + path.write_text("#!/usr/bin/env bash\n" + body, encoding="utf-8") + path.chmod(0o755) + return path + + +def no_process_left(marker): + time.sleep(0.2) + return subprocess.run(["pgrep", "-f", marker], capture_output=True).returncode == 1 + + +def timed_request(worker, filename, contents): + started = time.monotonic() + response = worker.request(filename, contents, timeout_secs=1) + return response, time.monotonic() - started + + +def check_same_as_token_by_sha(args): + memo = new_dir(args, "worker-memo") + worker = Worker(args.worker, memo, args.tokenize_command) + for index, fixture in enumerate(args.fixture): + expected = token_by_sha(args, new_dir(args, f"reference-memo-{index}"), fixture) + actual = worker.request(fixture.name, fixture.read_bytes()) + check(actual[0] == expected.returncode, f"{fixture.name}: exit {expected.returncode}") + if expected.returncode == 0: + check(actual[1] == expected.stdout, f"{fixture.name}: same tokens") + check( + memo_file(memo, fixture.read_bytes()).read_bytes() == expected.stdout, + f"{fixture.name}: same memo file", + ) + + first = args.fixture[0] + expected = memo_file(memo, first.read_bytes()).read_bytes() + check(worker.request(first.name, first.read_bytes()) == (0, expected, b""), "a memo hit gives the same bytes") + unknown = worker.request("unknown.xyzzy", b"x") + check(unknown[0] == 255 and b"unknown file extension" in unknown[2], "an unknown extension fails") + check(worker.request(first.name, first.read_bytes())[0] == 0, "the worker serves the next request") + worker.close() + + +def check_timeout(args, marker): + memo = new_dir(args, "timeout-memo") + script = write_script(args.temp_root / "hang.sh", f"exec perl -e 'sleep 100' {marker}\n") + worker = Worker(args.worker, memo, str(script)) + (exit_code, out, _), elapsed = timed_request(worker, "hang.c", b"int x;\n") + check(exit_code == TIMEOUT_EXIT and out == b"" and elapsed < 3, f"a hung tokenizer exits {TIMEOUT_EXIT}") + check(no_process_left(marker), "the timeout kills the tokenizer's process group") + check(not any(memo.rglob("*")), "a timeout memoizes nothing") + worker.close() + + +def in_process_command(args, marker): + options = args.tokenize_command.split() + real = next(o.split("=", 1)[1] for o in options if o.startswith("--srcml2token=")) + srcml2token = write_script( + args.temp_root / "srcml2token.sh", + 'src="${@: -1}"\n' + f'grep -qs HANG_MARKER "$src" && exec perl -e "sleep 100" {marker}\n' + 'grep -qs ABORT_MARKER "$src" && kill -ABRT $$\n' + f'exec {real} "$@"\n', + ) + ctags = write_script( + args.temp_root / "ctags.sh", f'echo CTAGS_STDERR >&2\nexec {shutil.which("ctags")} "$@"\n' + ) + options = [o for o in options if not o.startswith(("--srcml2token=", "--ctags="))] + return " ".join(options + [f"--srcml2token={srcml2token}", f"--ctags={ctags}"]) + + +def check_in_process_failures(args, marker): + memo = new_dir(args, "in-process-memo") + worker = Worker(args.worker, memo, in_process_command(args, marker)) + (exit_code, _, _), elapsed = timed_request(worker, "hang.c", b"int HANG_MARKER;\n") + check(exit_code == TIMEOUT_EXIT and elapsed < 3, f"in process, a hung srcml2token exits {TIMEOUT_EXIT}") + check(no_process_left(marker), "and is killed") + + exit_code, out, err = worker.request("abort.c", b"int ABORT_MARKER;\n") + check(exit_code == PARSER_CRASH_EXIT and out == b"", f"in process, a srcml2token signal death exits {PARSER_CRASH_EXIT}") + check(b"killed by signal 6" in err and b"CTAGS_STDERR" in err, "with the srcml2token and ctags stderr") + check(not any(memo.rglob("*")), "and nothing is memoized") + + fixture = args.fixture[0] + expected = token_by_sha(args, new_dir(args, "in-process-reference"), fixture).stdout + check(worker.request(fixture.name, fixture.read_bytes())[1] == expected, "the worker recovers") + worker.close() + + +def check_stale_srcml2token(args): + stale = write_script(args.temp_root / "srcml2token", "exit 0\n") + command = f"{args.tokenize_command} --srcml2token={stale}" + env = dict(os.environ, BFG_MEMO_DIR=str(new_dir(args, "stale-memo")), BFG_TOKENIZE_CMD=command) + started = subprocess.run([str(args.worker)], input=b"", capture_output=True, env=env) + check(started.returncode != 0 and b"READY" not in started.stdout, "a srcml2token without --libsrcml-path stops the start") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--worker", required=True, type=pathlib.Path) + parser.add_argument("--token-by-sha", required=True, type=pathlib.Path) + parser.add_argument("--tokenize-command", required=True) + parser.add_argument("--fixture", action="append", required=True, type=pathlib.Path) + parser.add_argument("--temp-root", required=True, type=pathlib.Path) + args = parser.parse_args() + + marker = f"cregit-token-worker-{os.getpid()}" + check_same_as_token_by_sha(args) + check_timeout(args, marker) + check_in_process_failures(args, marker) + check_stale_srcml2token(args) + print("ALL TOKEN WORKER TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/tests/tokenize_gate.sh b/tests/tokenize_gate.sh index d6ab714fa..2a61ec119 100755 --- a/tests/tokenize_gate.sh +++ b/tests/tokenize_gate.sh @@ -203,11 +203,12 @@ OUT=$(STUB_RC=0 run_step2 "$W" --stall-timeout abc); RC=$? rm -rf "$W" echo "case 10: the runner exits with blobExec's own status, not a flat 1" -for rc in 4 5; do +for rc in 4 5 6; do W=$(fixture) STUB_RC=$rc run_step2 "$W" > /dev/null 2>&1 GOT=$? [ "$GOT" -eq "$rc" ]; check "blobExec $rc makes the runner exit $rc (got $GOT)" $? + [ -n "$(ls "$W"/TOKENIZE-* 2>/dev/null)" ]; check "and the marker is still written" $? rm -rf "$W" done W=$(fixture) diff --git a/tokenize/CregitLanguages.pm b/tokenize/CregitLanguages.pm index b142e5cd8..6d85fd04e 100644 --- a/tokenize/CregitLanguages.pm +++ b/tokenize/CregitLanguages.pm @@ -13,16 +13,45 @@ package CregitLanguages; # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# The only extension table in cregit. tokenize.pl, tokenizeSrcMl.pl, -# tokenizeByBlobId/tokenBySha.pl and fileMask.pl all derive from it. +# THE single source of truth for "which files can cregit tokenize". +# +# Before this module the same knowledge was written out three times and the +# copies did not agree: +# +# tokenize/tokenize.pl %extensions — the dispatcher's gate +# tokenize/tokenizeSrcMl.pl %extensions — the srcML parser's autodetect +# tokenizeByBlobId/tokenBySha.pl %mapLang — the per-blob gate bfg calls +# +# tokenBySha.pl mapped `go`, `md` and `yaml`; tokenize.pl had no parser for any +# of them. So such a blob passed the first gate and died at the second with +# "Unknown parser for extension" — a whole project's run lost to an +# inconsistency between two tables that describe the same capability. They are +# now one table, and t/languages.t holds them to it. +# +# The file mask the pipeline selects blobs with is DERIVED from this table +# (file_mask below), never typed out again: a mask naming an extension the table +# does not know is a run that dies mid-way, and an extension in the table that +# no mask names is source silently left untokenized. use strict; use warnings; -# Lowercase extension, no leading dot -> language, spelled as --language takes it. +# Extension (lowercase, no leading dot) -> language name. The language names are +# the ones %LANG_PARSER_REL keys on and the ones passed as --language. +# +# Verified against the pinned srcML (1.1.0) on 2026-09-19 by tokenizing the same +# C++ fixture under each extension and comparing the token stream to .cpp's: +# .hxx and .tcc are byte-identical to .cpp. `.ixx .inl .cppm .cxxm .ipp` were +# candidates and are NOT here: srcml 1.1.0 does not know those extensions and +# then ignores `-l C++`, emitting an XML declaration with no at all. It +# exits 0 doing so, so the chain produces an EMPTY token file and reports +# success. That is why t/languages.t probes srcML per extension instead of +# trusting a reference page. our %EXT_LANG = ( + # C 'c' => 'C', 'h' => 'C', + # C++ 'c++' => 'C++', 'cc' => 'C++', 'cp' => 'C++', @@ -33,13 +62,41 @@ our %EXT_LANG = ( 'hpp' => 'C++', 'hxx' => 'C++', 'tcc' => 'C++', + # Java 'java' => 'Java', + # M4 (autoconf/automake input). Routed, but NOT in the mask: see + # %MASKED_LANGUAGES for the measurement that keeps it out. 'ac' => 'M4', 'am' => 'M4', + # Rust 'rs' => 'Rust', ); -# M4 is missing on purpose: m4.py raises StopIteration on real autotools quoting. +# Languages the pipeline's file mask is allowed to select, i.e. the ones whose +# parser is known to produce correct tokens for real-world input. +# +# M4 is deliberately ABSENT, on measurement. m4Tokenizer/m4.py was Python 2 and +# did not run at all under the pinned Python 3 (SyntaxError on `print`), so the +# "the parser exists and is executable" check that cleared M4 for inclusion was +# testing the wrong thing. It has since been ported, and it still must not be in +# the mask, because its lexer is wrong on real autotools input: +# +# Lexer::__init__ sets end_quote to a BACKTICK, not the apostrophe that m4 +# uses and that this file's own changequote() defaults to. So `` `x' `` runs +# the string on to the next backtick and swallows whatever is between -- +# measured on a two-line fixture, an AC_MSG_WARN macro call disappeared inside +# a string token. Setting end_quote to an apostrophe is NOT the fix: shell +# backticks in Makefile.am then never close, and the sample below goes from +# 36/38 files parsed to 12/38. +# +# Measured 2026-09-19 over 38 real configure.ac/configure.in/Makefile.am files +# on this machine: 36 parse, 2 die with "generator raised StopIteration" (an +# unterminated string at EOF; under Python 2 the same condition silently +# truncated the token stream and exited 0). +# +# Selecting .am/.ac would therefore fail whole projects at step 2, for 1.4 MB of +# source in 22 projects. Fixing the m4 lexer is its own task with its own +# fixtures; when it is done, add 'M4' here and the mask widens by itself. our %MASKED_LANGUAGES = ( 'C' => 1, 'C++' => 1, @@ -47,6 +104,8 @@ our %MASKED_LANGUAGES = ( 'Rust' => 1, ); +# Language -> parser, relative to the tokenize/ directory. Every value must +# exist and be executable in a built checkout; t/languages.t asserts it. our %LANG_PARSER_REL = ( 'C' => 'tokenizeSrcMl.pl', 'C++' => 'tokenizeSrcMl.pl', @@ -55,40 +114,51 @@ our %LANG_PARSER_REL = ( 'Rust' => 'rustTokenizer/target/release/rust_tokenizer', ); +# The parser every srcML-routed language shares. Named so callers can ask "is +# this extension srcML's?" without string-matching a path. our $SRCML_PARSER_REL = 'tokenizeSrcMl.pl'; -# An uppercase .C or .H is C++ by the GNU convention. Every other extension is -# case-insensitive. -our %CASE_SENSITIVE_EXT = ( 'C' => 'C++', 'H' => 'C++' ); - -# $ext arrives without its dot and with its case intact. -sub language_for_ext { - my ($ext) = @_; - return undef unless defined $ext && $ext ne ''; - return $CASE_SENSITIVE_EXT{$ext} if exists $CASE_SENSITIVE_EXT{$ext}; - return $EXT_LANG{ lc($ext) }; +# Extensions keyed WITH the leading dot, as tokenize.pl and tokenizeSrcMl.pl +# look them up (both lowercase the extension first). +sub extensions_by_dot { + return map { ( ".$_" => $EXT_LANG{$_} ) } keys %EXT_LANG; } +# The same, restricted to the languages srcML parses. tokenizeSrcMl.pl must not +# autodetect a .rs file as Rust: it would hand `-l Rust` to srcml. sub srcml_extensions_by_dot { - return map { ( ".$_" => $EXT_LANG{$_} ) } + return map { ( ".$_" => $EXT_LANG{$_} ) } grep { $LANG_PARSER_REL{ $EXT_LANG{$_} } eq $SRCML_PARSER_REL } keys %EXT_LANG; } +# Language -> absolute parser path, given the tokenize/ directory. sub parsers { my ($basedir) = @_; - die "parsers() needs the tokenize/ directory" unless defined $basedir and $basedir ne ''; + die "parsers() needs the tokenize/ directory" unless defined $basedir and $basedir ne ""; return map { ( $_ => "$basedir/$LANG_PARSER_REL{$_}" ) } keys %LANG_PARSER_REL; } -sub extensions_sorted { return sort keys %EXT_LANG } +# Extensions in a stable order, so the derived mask is byte-stable: it is stored +# in blobExec's meta table and compared string-for-string on every resume. +sub extensions_sorted { + return sort keys %EXT_LANG; +} +# The extensions the mask may name: every extension of every masked language. sub masked_extensions_sorted { return grep { $MASKED_LANGUAGES{ $EXT_LANG{$_} } } extensions_sorted(); } -# Sorted, because blobExec stores this mask and compares it on resume. Anchored -# at the end only: every consumer searches a path, not a bare basename. +# The universal file mask: every extension of every masked language, and nothing +# else. +# +# Case-insensitive, because `.C` and `.H` are real files and the per-language +# masks this replaces missed them. Anchored at the end only: every consumer +# matches it against a repository-relative path, not just a basename, and all +# four regex engines it reaches (Scala/Java in blobExec, Perl in blameRepoFiles +# and prettyPrintFiles, and whatever later reads the recorded string) search +# rather than full-match. sub file_mask { return '(?i)\.(' . join('|', map { quotemeta } masked_extensions_sorted()) . ')$'; } diff --git a/tokenize/CregitSrcMl.pm b/tokenize/CregitSrcMl.pm new file mode 100644 index 000000000..0b61382d7 --- /dev/null +++ b/tokenize/CregitSrcMl.pm @@ -0,0 +1,225 @@ +package CregitSrcMl; + +# tokenizeSrcMl.pl in a persistent process, with one ctags per language. Its +# output must stay byte-identical to tokenizeSrcMl.pl (tests/test_token_worker.sh). + +use strict; +use warnings; +use bytes; +use File::Temp (); +use POSIX qw(_exit); + +our $PARSER_CRASH_EXIT = 33; + +my $EMPTY_STREAM = "the token stream was empty. A successful srcML parse always emits at " + . "least the begin_unit/end_unit wrapper, even for an empty file, so an " + . "empty stream is a failed parse and never a legitimately empty result."; + +sub new { + my ($class, %options) = @_; + my $self = bless { ctags => "ctags-universal", position => 0, %options, + ctags_procs => {}, retired_errors => [] }, $class; + open(my $probe, '-|', $self->{srcml2token}, "--libsrcml-path") + or die "unable to execute srcml2token [$self->{srcml2token}]: $!\n"; + my $library = <$probe> // ""; + close($probe); + die "[$self->{srcml2token}] does not answer --libsrcml-path; rebuild it: make -C tokenize/srcMLtoken\n" + if $? != 0 or $library eq ""; + return $self; +} + +sub tokenize { + my ($self, $language, $inputName, $workDir, $timeoutSecs) = @_; + my $timedOut = 0; + local $SIG{ALRM} = sub { $timedOut = 1; $self->kill_all() }; + alarm($timeoutSecs); + my @result = eval { $self->run($language, $inputName, $workDir) }; + my $error = $@; + alarm(0); + my $helperErr = $self->take_helper_stderr(); + + if ($timedOut) { + $self->shutdown(); + return (124, "", $helperErr . "tokenize command timed out after [$timeoutSecs] seconds\n"); + } + return (255, "", $helperErr . $error) if not @result; + $result[2] = $helperErr . $result[2]; + return @result; +} + +sub run { + my ($self, $language, $inputName, $workDir) = @_; + my $declarations = $self->read_declarations($language, $inputName, "$workDir/$inputName"); + my ($status, $tokens, $stderr) = $self->run_srcml2token($language, $inputName, $workDir); + return parser_crash($inputName, $stderr, srcml2token_failure($status)) if $status != 0; + return parser_crash($inputName, $stderr, $EMPTY_STREAM) if not @$tokens; + return (0, merge($tokens, $declarations, $self->{position}), $stderr); +} + +sub srcml2token_failure { + my ($status) = @_; + my $signal = $status & 127; + return "srcml2token exited " . ($status >> 8) . "." if not $signal; + return "srcml2token was killed by signal $signal. srcML 1.1.0, which it links, " + . "dies on a signal on some C/C++ inputs under --position."; +} + +sub parser_crash { + my ($filename, $stderr, $why) = @_; + return ($PARSER_CRASH_EXIT, "", $stderr + . "cregit: tokenization of [$filename] FAILED: $why\n" + . "cregit: refusing to emit a tokenization for [$filename]; " + . "exiting $PARSER_CRASH_EXIT so the caller can count this blob.\n"); +} + +sub merge { + my ($tokens, $declarations, $position) = @_; + my $lastLine = -1; + my $output = ""; + for my $tokenLine (@$tokens) { + die "unable to parse srcml line [$tokenLine]\n" unless $tokenLine =~ /^([0-9]+|-):([0-9]+|-)\s+(.+)$/; + my ($line, $col, $token) = ($1, $2, $3); + my $lineNumber = $line eq '-' ? 0 : $line; + $output .= declaration_lines($declarations->{$line}, $line, $position) if $lineNumber != $lastLine; + $lastLine = $lineNumber; + $output .= "$line:$col|" if $position; + $output .= "$token\n"; + $output .= ($position ? "-:-|" : "") . "\n" if $token =~ /^end_/; + } + return $output; +} + +# Like tokenizeSrcMl.pl, a name that is twice on one line prints the type of its +# last declaration twice. +sub declaration_lines { + my ($declarations, $line, $position) = @_; + return "" if not $declarations; + return join "", map { ($position ? "$line:-|" : "") . "DECL|$declarations->{types}{$_}|$_\n" } + @{ $declarations->{names} }; +} + +sub read_declarations { + my ($self, $language, $inputName, $path) = @_; + open(my $in, '<:raw', $path) or die "unable to read [$path]: $!\n"; + my $source = do { local $/; <$in> } // ""; + my $ctags = $self->ctags_for($language); + print {$ctags->{in}} qq({"command":"generate-tags","filename":"$inputName","size":) + . length($source) . "}\n", $source; + $ctags->{in}->flush; + + my %declarations; + while (defined(my $tag = readline($ctags->{out}))) { + return \%declarations if $tag =~ /\A\{"_type": "completed"/; + chomp $tag; + if ($tag !~ /^([0-9]+) (.+) @@@ (.*) @@@ (.*)$/) { + $self->retire(delete $self->{ctags_procs}{$language}); + die "unable to parse output [$tag]\n"; + } + my ($line, $name, $type, $signature) = ($1, $2, $3, $4); + $name .= " $signature" if $signature ne "-"; + $declarations{$line}{types}{$name} = $type; + push @{ $declarations{$line}{names} }, $name; + } + $self->retire(delete $self->{ctags_procs}{$language}); + return \%declarations; +} + +# The same -x format as tokenizeSrcMl.pl, so that the names are byte-identical: +# ctags --_interactive drops a JSON field that is not valid UTF-8. +sub ctags_for { + my ($self, $language) = @_; + return $self->{ctags_procs}{$language} //= do { + my $proc = spawn_pipe($self->{ctags}, "--_interactive", "--language-force=$language", "-x", + "--sort=n", "--_xformat=%n %N @@@ %K @@@ %S"); + my $hello = readline($proc->{out}) // "EOF"; + die "ctags did not start: $hello\n" unless $hello =~ /"_type": "program"/; + $proc; + }; +} + +sub run_srcml2token { + my ($self, $language, $inputName, $workDir) = @_; + my $out = File::Temp->new(TEMPLATE => "cregit-srcml2token-out-XXXXX", TMPDIR => 1); + my $err = File::Temp->new(TEMPLATE => "cregit-srcml2token-err-XXXXX", TMPDIR => 1); + my $pid = fork() // die "unable to fork srcml2token: $!\n"; + if ($pid == 0) { + setpgrp(0, 0); + chdir($workDir) and open(STDIN, '<', '/dev/null') and open(STDOUT, '>', $out->filename) + and open(STDERR, '>', $err->filename) or _exit(255); + { no warnings 'exec'; exec $self->{srcml2token}, "-l", $language, $inputName; } + print STDERR "unable to execute srcml2token [$self->{srcml2token}]: $!\n"; + _exit(255); + } + $self->{parser_pid} = $pid; + waitpid($pid, 0); + my $status = $?; + delete $self->{parser_pid}; + seek($out, 0, 0); + chomp(my @tokens = <$out>); + return ($status, \@tokens, read_all($err)); +} + +# A helper writes its stderr to a file: a full stderr pipe would block it while +# this process waits for its stdout. +sub spawn_pipe { + my (@command) = @_; + pipe(my $childIn, my $parentOut) or die "unable to create pipe: $!\n"; + pipe(my $parentIn, my $childOut) or die "unable to create pipe: $!\n"; + my $err = File::Temp->new(TEMPLATE => "cregit-helper-err-XXXXX", TMPDIR => 1); + my $pid = fork() // die "unable to fork [$command[0]]: $!\n"; + if ($pid == 0) { + open(STDIN, '<&', $childIn) and open(STDOUT, '>&', $childOut) + and open(STDERR, '>>', $err->filename) or _exit(255); + { no warnings 'exec'; exec @command; } + print STDERR "unable to execute [$command[0]]: $!\n"; + _exit(255); + } + close($childIn); + close($childOut); + binmode $parentOut; + binmode $parentIn; + return { pid => $pid, in => $parentOut, out => $parentIn, err => $err }; +} + +sub take_helper_stderr { + my ($self) = @_; + my @live = grep { $_ } values(%{ $self->{ctags_procs} }); + my $text = join "", map { read_all($_) } @{ $self->{retired_errors} }, map { $_->{err} } @live; + $self->{retired_errors} = []; + truncate($_->{err}, 0) for @live; + return $text; +} + +sub read_all { + my ($fh) = @_; + seek($fh, 0, 0); + local $/; + return <$fh> // ""; +} + +sub retire { + my ($self, $proc) = @_; + return if not $proc; + push @{ $self->{retired_errors} }, $proc->{err}; + close($proc->{in}); + close($proc->{out}); + kill 'KILL', $proc->{pid}; + waitpid($proc->{pid}, 0); +} + +sub kill_all { + my ($self) = @_; + if (my $pid = $self->{parser_pid}) { + kill 'TERM', -$pid; + select(undef, undef, undef, 0.1); + kill 'KILL', -$pid; + } + kill 'KILL', map { $_->{pid} } grep { $_ } values(%{ $self->{ctags_procs} }); +} + +sub shutdown { + my ($self) = @_; + $self->retire(delete $self->{ctags_procs}{$_}) for keys %{ $self->{ctags_procs} }; +} + +1; diff --git a/tokenize/rustTokenizer/src/main.rs b/tokenize/rustTokenizer/src/main.rs index 67ee08f46..20f5c1246 100644 --- a/tokenize/rustTokenizer/src/main.rs +++ b/tokenize/rustTokenizer/src/main.rs @@ -21,7 +21,7 @@ static KEYWORDS: phf::Set<&'static str> = phf::phf_set! { }; fn main() { - let (path, emit_positions) = parse_args(); + let (path, position) = parse_args(); let src = match fs::read_to_string(&path) { Ok(s) => s, Err(e) => { @@ -29,15 +29,18 @@ fn main() { exit(1); } }; - print_pipe_separated_tokens(&src, emit_positions); + tokenize_source(&src, position); } fn parse_args() -> (String, bool) { let mut path: Option = None; - let mut emit_positions = false; + // Positions are OFF unless asked for, exactly as tokenizeSrcMl.pl gates them. This + // flag used to be swallowed and positions emitted unconditionally; see + // tokenize_source for why that corrupted every downstream row. + let mut position = false; for arg in env::args().skip(1) { if arg == "--position" { - emit_positions = true; + position = true; continue; } if arg.starts_with("--language=") || arg == "--verbose" { @@ -58,19 +61,38 @@ fn parse_args() -> (String, bool) { eprintln!("Usage: rust_tokenizer [--language=Rust] [--position] "); exit(2); }); - (path, emit_positions) + (path, position) } -fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { - let print_positionless_line = |body: &str| { - if emit_positions { +// Emits the cregit FINAL token format. That format is defined by tokenizeSrcMl.pl -- the +// only other tokenizer the pipeline actually routes to (CregitLanguages.pm +// %LANG_PARSER_REL: C/C++/Java -> tokenizeSrcMl.pl, Rust -> here; Go is unrouted and M4 +// is excluded from the file mask) -- and it is PIPE-separated with NO position prefix +// unless --position is given: +// +// tokenize/t/expected/main.c.nopos.token begin_unit|revision:... comment|/* ... */ +// tokenize/t/expected/main.c.token -:-|begin_unit|... 1:1|comment|/* ... */ +// +// Do NOT copy the TAB form in tokenize/srcMLtoken/tests/expected/*.token: that is +// srcml2token's INTERMEDIATE output, which tokenizeSrcMl.pl:120 consumes with +// /^([0-9]+|-):([0-9]+|-)\s+(.+)$/ and re-emits with `|`. It never reaches a consumer. +// +// The consumers both split on `|`: +// generate_dataset/generate_dataset.py:243 re.match(r"^(.+?)\|(.+)$", token_content) +// prettyPrint/prettyPrint-author.pl:976 split('\|', $value) +// The first group is NON-GREEDY, so any extra leading `line:col` field lands whole +// in token_type and shifts token_value, source_text and is_structural by one. +fn tokenize_source(src: &str, position: bool) { + // A line with no source position of its own: `-:-|` under --position, bare otherwise. + let marker = |body: &str| { + if position { println!("-:-|{}", body); } else { println!("{}", body); } }; - print_positionless_line(&format!( + marker(&format!( "begin_unit|revision:{};language:Rust;cregit-version:{}", REVISION, CREGIT_VERSION )); @@ -84,7 +106,7 @@ fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { let slice = &src[byte..end]; if let Some(out) = classify(&tok.kind, slice) { - if emit_positions { + if position { println!("{}:{}|{}", line, col, out); } else { println!("{}", out); @@ -103,8 +125,11 @@ fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { byte = end; } - print_positionless_line("end_unit"); - print_positionless_line(""); + marker("end_unit"); + // tokenizeSrcMl.pl:143 prints a bare marker line after every `end_*` token, so the + // stream ends `end_unit` + blank (or `-:-|end_unit` + `-:-|`). generate_dataset.py + // classifies that blank as token_type `blank`, is_structural 1. + marker(""); } // None skips output (whitespace / Eof); the caller still advances the cursor. Every diff --git a/tokenize/srcMLtoken/Makefile b/tokenize/srcMLtoken/Makefile index 4458f33c0..9f34887bb 100644 --- a/tokenize/srcMLtoken/Makefile +++ b/tokenize/srcMLtoken/Makefile @@ -3,30 +3,30 @@ XERCES_CFLAGS ?= $(shell pkg-config --cflags xerces-c 2>/dev/null) XERCES_LIBS ?= $(shell pkg-config --libs xerces-c 2>/dev/null || echo -lxerces-c-3.2) +ifndef SRCML_PREFIX +$(error SRCML_PREFIX is not set: build inside `devenv shell`, or set it to the prefix of a srcML build with headers) +endif +SRCML_CFLAGS ?= -I$(SRCML_PREFIX)/include +SRCML_LIBS ?= -L$(SRCML_PREFIX)/lib -Wl,-rpath,$(SRCML_PREFIX)/lib -lsrcml + +FIXTURES = StringUtil.java:Java audit.c:C ceiling.cpp:C++ main.c:C srcml2tokenHandlers.cpp:C++ + default: srcml2token srcml2token: srcml2token.cpp srcml2tokenHandlers.cpp srcml2tokenHandlers.hpp srcml2token.hpp - g++ -g -std=c++11 -O -Wall -pedantic $(XERCES_CFLAGS) -o srcml2token srcml2token.cpp srcml2tokenHandlers.cpp $(XERCES_LIBS) + g++ -g -std=c++11 -O -Wall -pedantic $(XERCES_CFLAGS) $(SRCML_CFLAGS) -o srcml2token srcml2token.cpp srcml2tokenHandlers.cpp $(XERCES_LIBS) $(SRCML_LIBS) test: srcml2token mkdir -p tests/out - srcml --position --language Java tests/StringUtil.java | ./srcml2token > tests/out/StringUtil.java.token - diff tests/expected/StringUtil.java.token tests/out/StringUtil.java.token - srcml --position --language C tests/audit.c | ./srcml2token > tests/out/audit.c.token - diff tests/expected/audit.c.token tests/out/audit.c.token - srcml --position --language C++ tests/ceiling.cpp | ./srcml2token > tests/out/ceiling.cpp.token - diff tests/expected/ceiling.cpp.token tests/out/ceiling.cpp.token - srcml --position --language C tests/main.c | ./srcml2token > tests/out/main.c.token - diff tests/expected/main.c.token tests/out/main.c.token - srcml --position --language 'C++' tests/srcml2tokenHandlers.cpp | ./srcml2token > tests/out/srcml2tokenHandlers.cpp.token - diff tests/expected/srcml2tokenHandlers.cpp.token tests/out/srcml2tokenHandlers.cpp.token - + for f in $(FIXTURES); do \ + ./srcml2token -l "$${f#*:}" "tests/$${f%%:*}" > "tests/out/$${f%%:*}.token" \ + && diff "tests/expected/$${f%%:*}.token" "tests/out/$${f%%:*}.token" || exit 1; \ + done + test -f "$$(./srcml2token --libsrcml-path)" + printf 'int x;\n' > tests/out/unknown.xyz + ! ./srcml2token -l C tests/out/unknown.xyz 2> tests/out/unknown.err + grep -q 'does not know the extension' tests/out/unknown.err expected: srcml2token mkdir -p tests/expected - srcml --position --language 'C++' tests/srcml2tokenHandlers.cpp | ./srcml2token > tests/expected/srcml2tokenHandlers.cpp.token - srcml --position --language C tests/main.c | ./srcml2token > tests/expected/main.c.token - srcml --position --language 'C++' tests/ceiling.cpp | ./srcml2token > tests/expected/ceiling.cpp.token - srcml --position --language C tests/audit.c | ./srcml2token > tests/expected/audit.c.token - srcml --position --language Java tests/StringUtil.java | ./srcml2token > tests/expected/StringUtil.java.token - + for f in $(FIXTURES); do ./srcml2token -l "$${f#*:}" "tests/$${f%%:*}" > "tests/expected/$${f%%:*}.token"; done diff --git a/tokenize/srcMLtoken/README.md b/tokenize/srcMLtoken/README.md index 7146f42dc..1ea038fc6 100644 --- a/tokenize/srcMLtoken/README.md +++ b/tokenize/srcMLtoken/README.md @@ -1,31 +1,28 @@ # srcMLtoken -`srcMLtoken` transcodes the output of srcML ([www.srcml.org](www.srcml.org)) to a tokenized view that can be used by Cregit. - -## Requirements - -`srcml` must be installed and in the PATH ([www.srcml.org](www.srcml.org)) +`srcml2token` parses a C, C++ or Java file with libsrcml +([www.srcml.org](https://www.srcml.org)) and prints a tokenized view of its +srcML that Cregit uses. ## How to use -The input of `srml2token` is the output of `srcml`. `srml2token` takes no parameters, and it reads its input from _stdin_ and writes to _stdout_. - ```sh -srcml -l C --position | srcml2token +srcml2token -l C ``` +It parses the file as `srcml -l C --position ` does, and writes the +tokens to _stdout_. `srcml2token --libsrcml-path` prints the libsrcml that it +loads; `tokenize/tokenizerIdentity.pl` digests that file. + ## How to build -It requires `xerces` to be installed. Simply run: +It needs `xerces-c` and the libsrcml headers and library. Inside `devenv shell`, +`SRCML_PREFIX` names the srcML build of `nix/srcml.nix`. Run: ```sh -make +make && make test ``` -## TODO - -- Create a `cmake` configuration file. - ## License The code is mostly derived from the examples of Xerces, hence it is under the @@ -33,3 +30,4 @@ Apache-2.0 Dependencies: - `xerces`: Apache-2.0 + - `libsrcml`: GPL-3.0, so the linked `srcml2token` binary is under GPL-3.0 diff --git a/tokenize/srcMLtoken/srcml2token.cpp b/tokenize/srcMLtoken/srcml2token.cpp index a91f110ef..ca9d2951d 100644 --- a/tokenize/srcMLtoken/srcml2token.cpp +++ b/tokenize/srcMLtoken/srcml2token.cpp @@ -15,96 +15,106 @@ * limitations under the License. */ -// --------------------------------------------------------------------------- -// Includes -// --------------------------------------------------------------------------- -#include -#include -#include "srcml2token.hpp" +#include +#include +#include #include +#include "srcml2token.hpp" -// --------------------------------------------------------------------------- -// Local helper methods -// --------------------------------------------------------------------------- -void usage() +static int usage() { - XERCES_STD_QUALIFIER cout << "\nUsage:\n" - " srcml2token \n\n" - "This program converts the output of srcML into a simplified tokenized version\n" - << XERCES_STD_QUALIFIER endl; + XERCES_STD_QUALIFIER cerr << "Usage: srcml2token -l \n" + " srcml2token --libsrcml-path\n" + "Parses the file with libsrcml and prints its tokens.\n"; + return 2; } +static int fail(const std::string& why) +{ + XERCES_STD_QUALIFIER cerr << "srcml2token: " << why << XERCES_STD_QUALIFIER endl; + return 3; +} -// --------------------------------------------------------------------------- -// Program entry point -// --------------------------------------------------------------------------- -int main(int argC, char* argV[]) +// The tokenizer identity digests the library that is loaded, not the one on disk. +static int print_libsrcml_path() { - // Initialize the XML4C system - try - { - XMLPlatformUtils::Initialize(); - } + Dl_info info; + if (!dladdr(reinterpret_cast(&srcml_version_string), &info) || !info.dli_fname) + return fail("cannot find the loaded libsrcml"); + XERCES_STD_QUALIFIER cout << info.dli_fname << XERCES_STD_QUALIFIER endl; + return 0; +} - catch (const XMLException& toCatch) - { - XERCES_STD_QUALIFIER cerr << "Error during initialization! Message:\n" - << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; - return 1; - } +// The same calls as `srcml -l --position `. Like that command, +// refuse an extension that srcML does not know: it would ignore the language. +static int parse_source(const char* language, const char* filename, std::string& xml) +{ + srcml_archive* archive = srcml_archive_create(); + char* buffer = nullptr; + size_t size = 0; + int status = srcml_archive_write_open_memory(archive, &buffer, &size); + if (status == SRCML_STATUS_OK) status = srcml_archive_enable_option(archive, SRCML_OPTION_POSITION); + if (status == SRCML_STATUS_OK) status = srcml_archive_set_language(archive, language); + if (status == SRCML_STATUS_OK) status = srcml_archive_set_tabstop(archive, 8); + if (status == SRCML_STATUS_OK) status = srcml_archive_enable_solitary_unit(archive); + if (status == SRCML_STATUS_OK) status = srcml_archive_disable_hash(archive); + if (status == SRCML_STATUS_OK && !srcml_archive_check_extension(archive, filename)) + status = SRCML_STATUS_UNSET_LANGUAGE; + srcml_unit* unit = status == SRCML_STATUS_OK ? srcml_unit_create(archive) : nullptr; + if (unit) { + status = srcml_unit_set_language(unit, language); + if (status == SRCML_STATUS_OK) status = srcml_unit_set_filename(unit, filename); + if (status == SRCML_STATUS_OK) status = srcml_unit_parse_filename(unit, filename); + if (status == SRCML_STATUS_OK) status = srcml_archive_write_unit(archive, unit); + srcml_unit_free(unit); + } + srcml_archive_close(archive); + srcml_archive_free(archive); + if (status == SRCML_STATUS_OK && buffer) + xml.assign(buffer, size); + srcml_memory_free(buffer); + return status; +} +static int tokenize(const std::string& xml) +{ + XMLPlatformUtils::Initialize(); SAX2XMLReader* parser = XMLReaderFactory::createXMLReader(); - srcml2tokenHandlers handler; parser->setContentHandler(&handler); parser->setErrorHandler(&handler); - - int errorCount = 0; - // create a faux scope so that 'src' destructor is called before - // XMLPlatformUtils::Terminate - { - // - // Kick off the parse and catch any exceptions. Create a standard - // input input source and tell the parser to parse from that. - // - try - { - if (argC < 2) { - StdInInputSource src; - parser->parse(src); - } else { - parser->parse(argV[1]); - } - } - catch (const OutOfMemoryException&) - { - XERCES_STD_QUALIFIER cerr << "OutOfMemoryException" << XERCES_STD_QUALIFIER endl; - errorCount = 2; - return 4; - } - catch (const XMLException& e) - { - XERCES_STD_QUALIFIER cerr << "\nError during parsing: \n" - << StrX(e.getMessage()) - << "\n" << XERCES_STD_QUALIFIER endl; - errorCount = 1; - return 4; - } - + int exitCode = 0; + try { + MemBufInputSource source(reinterpret_cast(xml.data()), xml.size(), "srcml"); + parser->parse(source); + } + catch (const OutOfMemoryException&) { + XERCES_STD_QUALIFIER cerr << "OutOfMemoryException" << XERCES_STD_QUALIFIER endl; + exitCode = 4; + } + catch (const XMLException& e) { + XERCES_STD_QUALIFIER cerr << "\nError during parsing: \n" << StrX(e.getMessage()) << "\n" << XERCES_STD_QUALIFIER endl; + exitCode = 4; } - - // - // Delete the parser itself. Must be done prior to calling Terminate, below. - // delete parser; - XMLPlatformUtils::Terminate(); - - if (errorCount > 0) - return 4; - else - return 0; + return exitCode; } +int main(int argC, char* argV[]) +{ + if (argC == 2 && std::string(argV[1]) == "--libsrcml-path") + return print_libsrcml_path(); + if (argC != 4 || std::string(argV[1]) != "-l") + return usage(); + std::string xml; + int status = parse_source(argV[2], argV[3], xml); + if (status == SRCML_STATUS_UNSET_LANGUAGE) + return fail("srcML does not know the extension of [" + std::string(argV[3]) + "]"); + if (status != SRCML_STATUS_OK) + return fail("libsrcml could not parse [" + std::string(argV[3]) + "] as " + argV[2] + + " (status " + std::to_string(status) + ")"); + return tokenize(xml); +} diff --git a/tokenize/tokenize.pl b/tokenize/tokenize.pl old mode 100644 new mode 100755 index 6538d03e4..ea4737b48 --- a/tokenize/tokenize.pl +++ b/tokenize/tokenize.pl @@ -17,11 +17,16 @@ use File::Basename; use FindBin; use lib $FindBin::Bin; +# The extension table and the language->parser table both come from here, and so +# does tokenizeByBlobId/tokenBySha.pl's. They used to be separate literals that +# disagreed; see CregitLanguages.pm. use CregitLanguages; my %declarations; my %listDeclarations; +my %extensions = CregitLanguages::extensions_by_dot(); + my $basedir = dirname($0); $basedir = "." if ($basedir eq ""); @@ -75,7 +80,7 @@ # autodetect Usage("File has no extension. You must provide one [$filename]") unless $filename =~ /(\.[a-z0-9\+]+)$/i; my $ext = lc($1); - $language = CregitLanguages::language_for_ext( substr($1, 1) ); + $language = $extensions{$ext}; Usage("Unknown extension [$ext] in file [$filename]. You must provide language using --language option") unless defined $language and $language ne ""; Usage("Unknown parser for extension [$ext] in file [$filename]. You must provide language using --language option") unless defined($parsers{$language}); } else { @@ -147,6 +152,11 @@ sub Tokenize { my $status = execute_command(@command); return if $status == 0; + # Propagate the parser's own exit status instead of die-ing, which would + # collapse every failure onto 255. tokenizeSrcMl.pl signals "srcML died / the + # tokenization is unusable" with a specific status, and blobExec counts that + # status separately from an ordinary error -- so flattening it here would put + # the crash back into the untraceable bucket it came from. if ($status == -1) { die "Unable to execute command @command: $!"; } diff --git a/tokenize/tokenizeSrcMl.pl b/tokenize/tokenizeSrcMl.pl index d7227a8e9..101e9552c 100755 --- a/tokenize/tokenizeSrcMl.pl +++ b/tokenize/tokenizeSrcMl.pl @@ -15,11 +15,18 @@ use strict; use File::Basename; -use File::Temp qw(tempfile); use FindBin; use lib $FindBin::Bin; use CregitLanguages; +# Exit status meaning "this parse did not produce a usable tokenization": +# srcml2token died on a signal or exited non-zero, or the token stream came back +# empty (which a healthy srcML parse never is -- see Verify_Parse). +# +# Picked to be distinguishable from every other outcome in the chain: below 128, +# so it can never be confused with a shell's 128+signal encoding; clear of GNU +# timeout's 124 and 137, which blobExec already reads as "we killed it"; and clear +# of blobExec's own 2 (abort), 3 (mask changed) and 4 (timed out). our $PARSER_CRASH_EXIT = 33; my %declarations; @@ -43,7 +50,6 @@ Options: --srcml2token= - --srcml= --language= --ctags=- --position @@ -52,15 +58,13 @@ my $basedir = dirname($0); $basedir = "." if ($basedir eq ""); -my $srcml = "srcml"; my $srcml2token = "$basedir/srcMLtoken/srcml2token"; my $ctags = "ctags-universal"; my $language = ""; my $verbose; my $position = 0; -GetOptions ("srcml=s" => \$srcml, - "srcml2token=s" => \$srcml2token, +GetOptions ("srcml2token=s" => \$srcml2token, "language=s" => \$language, "ctags=s" => \$ctags, "position" => \$position, @@ -84,8 +88,7 @@ # autodetect Usage("File has no extension. You must provide one [$filename]") unless $filename =~ /(\.[a-z\+]+)$/i; my $ext = lc($1); - $language = CregitLanguages::language_for_ext( substr($1, 1) ); - $language = undef unless defined $language and exists $extensions{ lc($1) }; + $language = $extensions{$ext}; Usage("Unknown extension [$ext] in file [$filename]. You must provide language using --language option") unless defined $language and $language ne ""; } @@ -106,27 +109,14 @@ sub Tokenize { - my $saveDir = `pwd`; - chomp $saveDir; my ($filename) = @_; - - my ($statusFh, $statusFile) = - tempfile("cregit-pipestatus-XXXXXX", TMPDIR => 1, UNLINK => 1); - close $statusFh; - - my $pipeline = sprintf( - '%s -l %s --position %s | %s; printf %%s\\ %%s "${PIPESTATUS[0]}" "${PIPESTATUS[1]}" > %s', - Shell_Quote($srcml), Shell_Quote($language), - Shell_Quote($filename), Shell_Quote($srcml2token), - Shell_Quote($statusFile)); - - open(parser, "-|", "bash", "-c", $pipeline) - or die "Unable to execute srcml pipeline on file [$filename]: $!"; + open(my $parser, "-|", $srcml2token, "-l", $language, $filename) + or die "Unable to execute srcml2token on file [$filename]: $!"; my $lastLine = -1; my $tokensRead = 0; - while () { + while (<$parser>) { $tokensRead++; # print STDERR; chomp; @@ -159,68 +149,29 @@ sub Tokenize } } - my $closed = close parser; - my $closeStatus = $?; - chdir($saveDir); - - Verify_Parse($filename, $tokensRead, $statusFile, $closed, $closeStatus); -} - -sub Shell_Quote -{ - my ($s) = @_; - $s = '' unless defined $s; - $s =~ s/'/'\\''/g; - return "'$s'"; + close $parser; + Verify_Parse($filename, $tokensRead, $?); } +# Fail closed. Anything other than "srcml2token exited 0 with a non-empty token +# stream" is a defect, and must be reported with a status the caller can count +# rather than silently becoming a 0-byte tokenization. sub Verify_Parse { - my ($filename, $tokensRead, $statusFile, $closed, $closeStatus) = @_; - - my ($srcmlStatus, $tokenStatus) = Read_Pipe_Status($statusFile); - - if (defined $srcmlStatus and $srcmlStatus > 128) { - my $signal = $srcmlStatus - 128; - Parser_Crash($filename, - "srcml was killed by signal $signal (shell status $srcmlStatus)."); - } - if (defined $srcmlStatus and $srcmlStatus != 0) { - Parser_Crash($filename, "srcml exited $srcmlStatus."); - } - if (defined $tokenStatus and $tokenStatus > 128) { - my $signal = $tokenStatus - 128; - Parser_Crash($filename, - "srcml2token was killed by signal $signal (shell status $tokenStatus)."); - } - if (defined $tokenStatus and $tokenStatus != 0) { - Parser_Crash($filename, "srcml2token exited $tokenStatus."); - } - - if (not defined $srcmlStatus or not defined $tokenStatus) { - if (not $closed or $closeStatus != 0) { - Parser_Crash_Unknown($filename, $closeStatus); - } - } - - if ($tokensRead == 0) { - Parser_Crash($filename, - "the token stream was empty. A successful srcML parse always emits at " - . "least the begin_unit/end_unit wrapper, even for an empty file, so an " - . "empty stream is a failed parse and never a legitimately empty result."); - } -} - -sub Read_Pipe_Status -{ - my ($statusFile) = @_; - open(my $fh, '<', $statusFile) or return (undef, undef); - my $line = <$fh>; - close $fh; - return (undef, undef) unless defined $line; - chomp $line; - return (undef, undef) unless $line =~ /^([0-9]+)\s+([0-9]+)$/; - return ($1, $2); + my ($filename, $tokensRead, $status) = @_; + my $signal = $status & 127; + Parser_Crash($filename, + "srcml2token was killed by signal $signal. srcML 1.1.0, which it links, " + . "dies on a signal on some C/C++ inputs under --position.") if $signal; + Parser_Crash($filename, "srcml2token exited " . ($status >> 8) . ".") if $status != 0; + + # Even a zero-byte source file yields the begin_unit/end_unit wrapper, so zero + # tokens always means the parse failed. + Parser_Crash($filename, + "the token stream was empty. A successful srcML parse always emits at " + . "least the begin_unit/end_unit wrapper, even for an empty file, so an " + . "empty stream is a failed parse and never a legitimately empty result.") + if $tokensRead == 0; } sub Parser_Crash @@ -232,17 +183,6 @@ sub Parser_Crash exit($PARSER_CRASH_EXIT); } -sub Parser_Crash_Unknown -{ - my ($filename, $closeStatus) = @_; - print STDERR "cregit: tokenization of [$filename] FAILED: the srcml pipeline " - . "reported a non-zero close status ($closeStatus) and bash did not report " - . "PIPESTATUS, so the failing stage is unknown.\n"; - exit($PARSER_CRASH_EXIT); -} - - - sub Declarations_Test { foreach my $line (sort {$a <=> $b} keys %declarations) { diff --git a/tokenize/tokenizerIdentity.pl b/tokenize/tokenizerIdentity.pl new file mode 100755 index 000000000..62898711d --- /dev/null +++ b/tokenize/tokenizerIdentity.pl @@ -0,0 +1,173 @@ +#!/usr/bin/env perl + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Prints blobExec's --tokenizer-identity argument: one opaque value per file +# extension, changing whenever the thing that turns that extension's bytes into +# tokens changes. +# +# ext=,ext=,... +# +# Why this exists. blobExec decided whether to reuse a cached tokenization on +# `command` and `mask`. `command` is the fixed path +# tokenizeByBlobId/tokenBySha.pl and `mask` says WHICH files to tokenize, never +# HOW. So when the Rust tokenizer's output format was corrected in 729643e, +# neither value moved: every cached .rs row stayed a cache hit and a run with +# the corrected binary reproduced the broken tokens, in 45 projects, with no +# error anywhere. An identity that tracks the TOOLCHAIN is the missing key. +# +# What goes into an extension's identity, and why each part: +# +# * its language's parser (CregitLanguages::LANG_PARSER_REL) — the tokenizer +# itself. For Rust that is the compiled cargo artifact, which is exactly the +# file that went stale. +# * srcml2token, the libsrcml it loads, ctags and CregitSrcMl.pm, for the +# srcML-routed languages only. srcml2token parses with libsrcml, and +# tokenizeSrcMl.pl (or CregitSrcMl.pm in the tokenizer worker) merges ctags +# output into its tokens. A change in any of them changes the tokens. +# * tokenize.pl and CregitLanguages.pm, for every extension. tokenize.pl is +# the dispatcher that decides which parser runs and with which flags +# (--position among them, which the token format depends on), and +# CregitLanguages.pm decides which language an extension is. Either one can +# change an extension's tokens without the parser changing at all. +# +# Including the shared files makes the identity CONSERVATIVE: editing +# tokenize.pl moves every extension's identity, so every project with cached +# rows refuses until an operator decides what to do. That is the right direction +# to be blunt in. A refusal costs a conversation; a false "unchanged" costs a +# republished corpus. +# +# The output is sorted and every value is a sha256 hex digest of a digest list, +# so the string is byte-stable for a given checkout: it is stored in the blob +# map's meta table and compared string-for-string on every resume. + +use strict; +use warnings; + +use Digest::SHA qw(sha256_hex); +use File::Basename; +use FindBin; +use lib $FindBin::Bin; +use CregitLanguages; +use Getopt::Long; + +my $usage = " +Usage: $0 [options] + +Prints blobExec's --tokenizer-identity value for this checkout: + ext=,ext=,... + +Options: + --srcml2token= the built srcML tokenizer (required for C/C++/Java) + --ctags= the ctags binary (required for C/C++/Java) + --all-extensions cover every extension in the table, not only the + masked ones (.am/.ac are routed but not masked) +"; + +my $srcml2tokenPath = ""; +my $ctagsPath = ""; +my $allExtensions = 0; + +GetOptions( + "srcml2token=s" => \$srcml2tokenPath, + "ctags=s" => \$ctagsPath, + "all-extensions" => \$allExtensions, +) or die($usage); + +my $basedir = $FindBin::Bin; + +# Every extension's identity includes these, because either one can change what +# an extension's tokens are without any parser changing. +my @shared = ("$basedir/tokenize.pl", "$basedir/CregitLanguages.pm"); + +my %parsers = CregitLanguages::parsers($basedir); + +my @extensions = $allExtensions + ? CregitLanguages::extensions_sorted() + : CregitLanguages::masked_extensions_sorted(); + +# A file that is named but absent is fatal. Emitting an identity that silently +# omitted a component would be worse than useless: it would compare EQUAL across +# a change in that component, which is the exact failure this program exists to +# make impossible. The Rust parser is a build artifact, so this is also where an +# unbuilt checkout is caught. +sub digest_of { + my ($path) = @_; + open(my $fh, '<', $path) + or die "tokenizerIdentity: cannot read [$path]: $!\n" + . " It is part of a tokenizer's identity, and an identity computed without it\n" + . " would compare equal across a change in it. Build the checkout first\n" + . " (./run_pipeline_process.sh --ensure-artifacts) and try again.\n"; + binmode $fh; + my $sha = Digest::SHA->new(256); + $sha->addfile($fh); + close $fh; + return $sha->hexdigest; +} + +sub libsrcml_of { + my ($srcml2token) = @_; + open(my $ph, '-|', $srcml2token, "--libsrcml-path") + or die "tokenizerIdentity: cannot run [$srcml2token]: $!\n"; + my $path = <$ph> // ""; + close $ph; + chomp $path; + die "tokenizerIdentity: [$srcml2token] does not answer --libsrcml-path, so it is\n" + . " not the srcml2token that parses with libsrcml. Rebuild it: make -C tokenize/srcMLtoken\n" + if $? != 0 or $path eq ""; + return $path; +} + +my %digestCache; +sub cached_digest { + my ($path) = @_; + $digestCache{$path} //= digest_of($path); + return $digestCache{$path}; +} + +my @pairs; +my $libsrcml; +for my $ext (@extensions) { + my $language = $CregitLanguages::EXT_LANG{$ext}; + die "tokenizerIdentity: no language for extension [$ext]\n" unless defined $language; + my $parser = $parsers{$language}; + die "tokenizerIdentity: no parser for language [$language]\n" unless defined $parser; + + my @components = (@shared, $parser); + + # Required rather than optional: an identity for .c computed without + # srcml2token would not move when srcml2token does. + if ($CregitLanguages::LANG_PARSER_REL{$language} eq $CregitLanguages::SRCML_PARSER_REL) { + for my $pair (["--srcml2token", $srcml2tokenPath], + ["--ctags", $ctagsPath]) { + my ($flag, $path) = @$pair; + die "tokenizerIdentity: $flag is required, because extension [$ext] is parsed by\n" + . " $CregitLanguages::SRCML_PARSER_REL and its tokens depend on that binary.\n$usage" + if $path eq ""; + push @components, $path; + } + $libsrcml //= libsrcml_of($srcml2tokenPath); + push @components, $libsrcml, "$basedir/CregitSrcMl.pm"; + } + + # Digest of the component digests, not of the concatenated bytes: the list is + # what identifies the toolchain, and a per-file digest makes a failure + # readable with --verbose-style debugging if it is ever needed. + my $combined = sha256_hex(join("\n", map { cached_digest($_) } @components)); + push @pairs, "$ext=$combined"; +} + +die "tokenizerIdentity: no extensions to report\n" unless @pairs; + +print join(",", @pairs), "\n"; diff --git a/tokenizeByBlobId/WORKER_PROTOCOL.md b/tokenizeByBlobId/WORKER_PROTOCOL.md new file mode 100644 index 000000000..d416f29dd --- /dev/null +++ b/tokenizeByBlobId/WORKER_PROTOCOL.md @@ -0,0 +1,43 @@ +# Tokenizer worker protocol + +`tokenWorker.pl` is a persistent `tokenBySha.pl`. With +`--tokenizer-worker=`, blobExec starts one worker per processor and +sends each blob to an idle worker. A worker handles one request at a time. + +The worker reads `BFG_MEMO_DIR` and `BFG_TOKENIZE_CMD` from its environment, +as `tokenBySha.pl` does. Its tokens, its memo files and its exit statuses are +the same as those of `tokenBySha.pl`. + +For C, C++ and Java, the worker does the work of `tokenizeSrcMl.pl` in its own +process (`tokenize/CregitSrcMl.pm`) if `BFG_TOKENIZE_CMD` is `tokenize.pl` with +only the options `--srcml2token=`, `--ctags=` and `--position`. It keeps one +`ctags --_interactive` for each language alive, and starts only `srcml2token` +for each file. For all other languages and commands, it starts +`BFG_TOKENIZE_CMD` for each blob. The worker does not start if `srcml2token` +does not answer `--libsrcml-path`. + +## Messages + +At start, the worker writes `READY\n` to stdout. Then, for each request: + +``` +JVM -> worker: REQ \n +worker -> JVM: RES \n +``` + +- `name` is the file name of the blob. Its extension selects the language. +- Lengths are byte counts. The strings are raw bytes. +- `exit` is the status that `tokenBySha.pl` gives: `0` (the tokens are in + `out`), `33` (parser crash) or another status (tokenizer error, `out` is + empty). `124` means that the tokenizer did not finish in `timeoutSeconds`. +- The worker memoizes only a successful tokenization. + +## Failures + +- The worker runs the tokenizer (or `srcml2token`) in its own process group. At the + timeout it kills the group and the helpers, and answers `124`. +- If no answer comes in `timeoutSeconds + 5` seconds, blobExec kills the + worker and its children, starts a new worker, and reports a timeout. +- If the worker stops in a request, blobExec reports a tokenizer error for + that blob and starts a new worker. +- At EOF on stdin, the worker exits 0. A malformed request makes it exit 2. diff --git a/tokenizeByBlobId/tokenBySha.pl b/tokenizeByBlobId/tokenBySha.pl old mode 100644 new mode 100755 index 7099e9a0f..93d91c7d0 --- a/tokenizeByBlobId/tokenBySha.pl +++ b/tokenizeByBlobId/tokenBySha.pl @@ -44,6 +44,8 @@ make_path($buildDir) if not -d $buildDir; +my %mapLang = %CregitLanguages::EXT_LANG; + if (not defined($ENV{BFG_MEMO_DIR}) || $ENV{BFG_MEMO_DIR} eq "") { die "You must define the environment variable BFG_MEMO_DIR equal to the directory where to memoize" @@ -84,14 +86,12 @@ die "BFG_FILENAME environment variable not set " if $blobFN eq ""; my $fileExt; -my $blobLang; if ($blobFN =~ /\.([^.]+)$/) { $fileExt = lc($1); - $blobLang = CregitLanguages::language_for_ext($1); } -if (not defined($blobLang)) { +if (not defined($mapLang{$fileExt})) { die "unknown file extension [$fileExt]"; } @@ -117,7 +117,7 @@ my ($fout, $outfile) = mkstemp( "$buildDir/tmpfile-out-XXXXX" ); - my $langOp = "--language=" . $blobLang; + my $langOp = "--language=" . $mapLang{$fileExt}; open(PROC, "cd \"$wd\" && $tokenizeCmd $langOp \"$inName\" |") or die "unable to execute $tokenizeCmd (verify variable BFG_TOKENIZE_CMD) [$tokenizeCmd]"; diff --git a/tokenizeByBlobId/tokenWorker.pl b/tokenizeByBlobId/tokenWorker.pl new file mode 100755 index 000000000..f44c708e8 --- /dev/null +++ b/tokenizeByBlobId/tokenWorker.pl @@ -0,0 +1,160 @@ +#!/usr/bin/env perl + +# A persistent tokenBySha.pl with the same output and memo. The protocol is in +# WORKER_PROTOCOL.md. C, C++ and Java go through CregitSrcMl in this process. + +use strict; +use warnings; +use bytes; +use Digest::SHA qw(sha1_hex); +use File::Basename qw(basename dirname); +use File::Copy qw(move); +use File::Path qw(make_path); +use File::Temp qw(tempfile); +use FindBin qw($RealBin); +use POSIX qw(_exit); +use lib "$RealBin/../tokenize"; +use CregitLanguages; +use CregitSrcMl; + +binmode STDIN; +binmode STDOUT; +$| = 1; + +my $memoDir = $ENV{BFG_MEMO_DIR} // ""; +die "You must define BFG_MEMO_DIR\n" if $memoDir eq ""; +die "Sha dir [$memoDir] does not exist\n" if not -d $memoDir; +my @tokenizeCommand = split ' ', $ENV{BFG_TOKENIZE_CMD} // ""; +die "Tokenize command not defined. Use BFG_TOKENIZE_CMD\n" if not @tokenizeCommand; +my $buildDir = "$RealBin/build"; +make_path($buildDir); +my $inProcess = in_process_tokenizer(@tokenizeCommand); + +print "READY\n"; +while (defined(my $header = )) { + my ($bodyLength, $nameLength, $timeoutSecs) = $header =~ /\AREQ (\d+) (\d+) (\d+)\n\z/ + or protocol_error("malformed request header [$header]"); + my $filename = read_exact($nameLength); + my $contents = read_exact($bodyLength); + my ($exit, $out, $err) = eval { tokenize_blob($filename, $contents, $timeoutSecs) }; + ($exit, $out, $err) = (255, "", $@) if not defined $exit; + print "RES $exit ", length($out), " ", length($err), "\n", $out, $err; +} +exit 0; + +sub read_exact { + my ($length) = @_; + my $value = ""; + while (length($value) < $length) { + my $read = read(STDIN, $value, $length - length($value), length($value)); + protocol_error(defined $read ? "the request ended early" : "cannot read the request: $!") + if not $read; + } + return $value; +} + +sub protocol_error { + print STDERR "tokenWorker.pl: @_\n"; + exit 2; +} + +sub tokenize_blob { + my ($filename, $contents, $timeoutSecs) = @_; + my ($ext) = $filename =~ /\.([^.]+)$/; + $ext = lc($ext // ""); + my $language = $CregitLanguages::EXT_LANG{$ext}; + return (255, "", "unknown file extension [$ext]\n") if not defined $language; + + my $sha1 = sha1_hex($contents); + my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . "/$sha1"; + return (0, read_file($memoFile), "") if -f $memoFile; + + my $workDir = File::Temp->newdir("tokdir-XXXXX", DIR => $buildDir); + write_file("$workDir/input.$ext", $contents); + my ($exit, $out, $err) = tokenize_file($language, "input.$ext", "$workDir", $timeoutSecs); + return ($exit, "", $err) if $exit != 0; + write_memo($memoFile, $out); + return (0, $out, $err); +} + +sub tokenize_file { + my ($language, $inputName, $workDir, $timeoutSecs) = @_; + return $inProcess->tokenize($language, $inputName, $workDir, $timeoutSecs) + if $inProcess and $CregitLanguages::LANG_PARSER_REL{$language} eq $CregitLanguages::SRCML_PARSER_REL; + return run_tokenizer($workDir, $timeoutSecs, @tokenizeCommand, "--language=$language", $inputName); +} + +# Only a tokenize.pl command with the options that it gives to tokenizeSrcMl.pl +# can run in this process. Any other command starts once per blob. +sub in_process_tokenizer { + my ($program, @options) = @_; + return undef if basename($program) ne "tokenize.pl"; + my %config = (srcml2token => dirname($program) . "/srcMLtoken/srcml2token"); + for (@options) { + if (/\A--(srcml2token|ctags)=(.+)\z/) { $config{$1} = $2 } + elsif ($_ eq "--position") { $config{position} = 1 } + else { return undef } + } + return CregitSrcMl->new(%config); +} + +sub run_tokenizer { + my ($workDir, $timeoutSecs, @command) = @_; + my $pid = fork() // die "cannot fork the tokenizer: $!\n"; + exec_in_group($workDir, @command) if $pid == 0; + + my $timedOut = 0; + local $SIG{ALRM} = sub { $timedOut = 1; kill_group($pid) }; + alarm($timeoutSecs); + waitpid($pid, 0); + my $status = $?; + alarm(0); + + my $err = read_file("$workDir/.stderr"); + return (124, "", $err . "tokenize command timed out after [$timeoutSecs] seconds\n") if $timedOut; + return (255, "", $err . "tokenize command was killed by signal " . ($status & 127) . "\n") if $status & 127; + return ($status >> 8, read_file("$workDir/.stdout"), $err); +} + +# The child's own process group lets a timeout kill srcml and everything else +# the tokenizer started. +sub exec_in_group { + my ($workDir, @command) = @_; + setpgrp(0, 0); + chdir($workDir) and open(STDIN, '<', '/dev/null') and open(STDOUT, '>', '.stdout') + and open(STDERR, '>', '.stderr') or _exit(255); + { no warnings 'exec'; exec { $command[0] } @command; } + print STDERR "cannot execute the tokenizer [$command[0]]: $!\n"; + _exit(255); +} + +sub kill_group { + my ($pid) = @_; + kill 'TERM', -$pid; + select(undef, undef, undef, 0.1); + kill 'KILL', -$pid; +} + +sub read_file { + my ($path) = @_; + open(my $in, '<:raw', $path) or die "cannot read [$path]: $!\n"; + local $/; + return <$in> // ""; +} + +sub write_file { + my ($path, $contents) = @_; + open(my $out, '>:raw', $path) or die "cannot write [$path]: $!\n"; + print {$out} $contents; + close($out) or die "cannot close [$path]: $!\n"; +} + +sub write_memo { + my ($memoFile, $output) = @_; + make_path(dirname($memoFile)); + my ($out, $temp) = tempfile("tmpfile-out-XXXXX", DIR => $buildDir); + binmode $out; + print {$out} $output; + close($out) or die "cannot close [$temp]: $!\n"; + move($temp, $memoFile) or die "cannot move the tokenization to [$memoFile]: $!\n"; +} From 4a56800c06addaa7e687b6a0115eff3f47704a8c Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:39:21 +0000 Subject: [PATCH 02/15] docs(blobexec): cut comments the code already says Review of #102 asked for much shorter comments in blobExec. Delete comments that restate the code, narrate incidents and measurements, or describe the next lines; keep a non-obvious why, trap or reference in at most 3 lines. The denylist TSV header goes from 287 comment lines to 8. Data rows are byte-identical; the shipped list still parses to 209 entries. Comments only: sbt test passes with the same 177 tests. --- .../cregit/blobexec/blob-denylist.tsv | 291 +--------------- .../scala/cregit/blobexec/BlobDenylist.scala | 43 +-- .../main/scala/cregit/blobexec/BlobExec.scala | 131 ++----- .../src/main/scala/cregit/blobexec/Main.scala | 52 +-- .../main/scala/cregit/blobexec/Mapping.scala | 320 +++--------------- .../cregit/blobexec/TokenizerIdentity.scala | 46 +-- .../scala/cregit/blobexec/TokenizerMemo.scala | 45 +-- .../main/scala/cregit/blobexec/Walker.scala | 181 ++-------- .../cregit/blobexec/BlobDenylistSpec.scala | 87 +---- .../scala/cregit/blobexec/BlobExecSpec.scala | 30 +- .../cregit/blobexec/FailedBlobSpec.scala | 5 - .../cregit/blobexec/MainOptionsSpec.scala | 15 +- .../scala/cregit/blobexec/MappingSpec.scala | 70 +--- .../cregit/blobexec/RetokenizeSpec.scala | 55 +-- .../blobexec/TokenizerIdentitySpec.scala | 7 +- .../cregit/blobexec/TokenizerMemoSpec.scala | 14 +- .../blobexec/WalkerIntegrationSpec.scala | 123 ++----- 17 files changed, 202 insertions(+), 1313 deletions(-) diff --git a/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv b/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv index 64e65c6ad..5a757a58d 100644 --- a/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv +++ b/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv @@ -1,288 +1,9 @@ -# blobExec blob denylist — blobs excluded from the rewrite by content id. +# Blobs blobExec never tokenizes, keyed by git blob sha so one entry covers every +# path and copy of the content. A listed blob is dropped from rewritten trees, +# counted as blobsDenylisted, and does not change the exit status. # -# A denylisted blob is never handed to the tokenizer. It is dropped from every -# rewritten tree, exactly like an oversized blob (Walker.readBlob), so the file is -# ABSENT from the tokenized repository rather than present as raw source: it -# produces no blame and no dataset row. It is counted separately from -# blobsOversized and blobsTimedOut, logged once with its sha, path and reason, and -# it does NOT affect the exit status. That last point is the difference between -# this list and a timeout: an entry here is a diagnosed, deterministic, cited -# exclusion, while a timeout is a hang nobody has explained yet and must keep -# blocking publication. -# -# This is a DATA file, not a constant in the code, so a paper can cite the list -# itself. It is loaded from the classpath (BlobDenylist.ResourcePath) and shipped -# inside the assembly jar, so adding an entry means editing this file and running -# `sbt assembly`. Nothing at run time can extend or override it: an undocumented -# exclusion is the one thing that would make the dataset's omissions unauditable. -# -# FORMAT: three TAB-separated fields. Blank lines and lines starting with # are -# ignored. Leading and trailing whitespace on a field is stripped. -# -# -# -# The key is the GIT BLOB id, which is what the walker has in hand -# (Walker.BlobMissTask.origId) and what blobExec's own timeout line prints. It is -# not the memo's key: tokenizeByBlobId/tokenBySha.pl hashes the file contents -# alone, with no git object header. -# -# The key is deliberately NOT the path. The entries below are the historical -# versions of 36 source files, one of them at two paths after an OpenJDK -# repository reorganisation and one of them a single blob vendored into TWO -# DIFFERENT PROJECTS (zlib's `deflate.c`, byte-identical in ossec__ossec-hids and -# azerothcore__azerothcore-wotlk — one entry covers both). A path-keyed list would -# have to be updated every time a project moves a file, would need a duplicate -# entry per vendored copy, and would not distinguish the versions of a file that -# fail from the versions that parse — which is a 4/4 split on one file in class 2 -# and, in class 3, splits as narrow as 3 of 423. Content is the stable identity. -# The paths are recorded for a reader rather than matched on. -# -# =========================================================================== -# THREE DISTINCT DEFECTS, and the third is NOT LIKE THE OTHER TWO. -# -# classes 1 and 2 are srcML NON-TERMINATION: srcml spins forever, and the -# pipeline only ever sees a timeout. -# class 3 is a srcML CRASH: srcml DIES ON A SIGNAL (SIGSEGV or SIGABRT) in -# milliseconds. It is not a hang, it must not be described as one, and it -# reaches the pipeline through a different code path (exit 33 from -# tokenizeSrcMl.pl, `blobsParserCrashed` and exit 6 from blobExec). -# -# Only class 1 has an upstream issue. Do not cite #2361 for class 2 or class 3. -# =========================================================================== -# -# --------------------------------------------------------------------------- -# CLASS 1 — the JAVA parser, on type-annotated array types. -# Upstream: srcML/srcML#2361, open, no patch. -# 8 blobs, TWO files, both in tencent__tencentkona-21 (OpenJDK-derived, -# 50,663 .java files at HEAD, 147,265 blobs recorded). Both are OpenJDK -# langtools regression tests whose stated purpose is to stress exactly the -# grammar that breaks the parser: not production source, and carrying no -# contributor-behaviour signal. That is what makes excluding them cheap. -# --------------------------------------------------------------------------- -# -# TestNewCastArray.java — @bug 8005681 "Repeated annotations on new,array,cast" -# test/langtools/tools/javac/annotations/typeAnnotations/classfile/ -# 2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65 14,503 B -# langtools/test/tools/javac/annotations/typeAnnotations/classfile/ -# 303c1a109f1fdc4b9d84e15111c56931202844ac 14,506 B -# 504aaf9109fc01d6d62d99ac529ded99a8dfa8ca 14,451 B -# c91924d1e87cb82d372a73b24579ccc34609efaa 14,450 B -# -# CheckErrorsForSource7.java — @run main CheckErrorsForSource7 -# langtools/test/tools/javac/annotations/typeAnnotations/failures/ -# 56c8adf588e697a3e42b600b2e5ec54710592c61 8,388 B trigger at line 178 -# 5e9c9138ed9c43c20591d1814cb72ad014d1a9e5 8,750 B trigger at line 181 -# 6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7 8,645 B trigger at line 179 -# test/langtools/tools/javac/annotations/typeAnnotations/failures/ -# afbd81e5789ba060147b0de3be4f01759963299d 8,617 B trigger at line 178 -# (5e9c9138… lives at both paths; all four versions of the file hang) -# -# TestNewCastArray's trigger, minimised in srcml-nontermination-analysis.md, is a -# declaration whose declared type is a multi-dimensional array carrying a type -# annotation on its first dimension WHEN the initializer is an array-creation -# expression that also carries one: -# -# String @A [] [] s = new String @A [2] [2]; -# -# CheckErrorsForSource7 is a DIFFERENT FILE with a SECOND, DISTINCT sufficient -# construct in the same #2361 family, minimised here on 2026-09-20. It needs no -# initializer at all: -# -# Object [] @TA [] arr @TA[]; // the real line -# class C { int @A [] a @A []; } // minimal reproducer, 31 B -# -# BOTH an annotated dimension in the declared type AND an annotated trailing -# C-style array dimension on the declarator are required; either one alone parses -# in under 90 ms, and it makes no difference whether the declaration is a field or -# a local, or whether an initializer is present. The neighbouring line 179, -# ` @TA int @TA[] ret(Object obj) @TA[]`, was measured and is NOT a trigger: on -# its own it parses in 8 ms. Offer this reproducer upstream on #2361 alongside the -# first one — it is a second sufficient condition the issue does not record. -# -# --------------------------------------------------------------------------- -# CLASS 2 — the C parser, on C++ inside a `.h`. -# NO UPSTREAM ISSUE EXISTS. This is NOT #2361: different language, different -# construct, separately minimised. The citation is the analysis document. -# 4 blobs, ONE file, in facebook__sapling. Unlike class 1 this is ordinary -# production source, so the exclusion does cost real signal — 4 blobs of a -# 1.9-2.6 kB header, bounded below. -# --------------------------------------------------------------------------- -# -# eden/fs/utils/StatTimes.h (historical only; NOT present at HEAD) -# 5460d43f1884e3be230e02215151b9b963844a48 1,861 B -# 720914b27e170b73b38ef662fb04239273dd0a34 2,580 B -# 99c191e9a208547f5c9b1224c290f106a640de59 2,443 B -# dbb885e0bf046755c4fcfbddfc2454f90c550c62 2,442 B -# -# CregitLanguages.pm maps the `.h` extension to language C, so tokenBySha.pl runs -# `srcml -l C` over a header whose contents are C++. The C parser then does not -# terminate. Minimised on 2026-09-20 from 1,861 bytes to 41: -# -# namespace n{struct T&f(const struct S);} -# -# Necessary and sufficient, each condition measured. Remove any one and the input -# parses in under 12 ms: -# -# - an enclosing NAMED `namespace X {` or `class X {` block. An anonymous -# `namespace {` does NOT trigger it. -# - a return type that is a REFERENCE to an ELABORATED struct type, `struct T&`. -# `struct T*` parses; `T&` without the `struct` keyword parses. -# - a parameter of CV-QUALIFIED ELABORATED struct type, `const struct S` -# (`volatile` does it too). The parameter need not be a reference and need not -# be named. `const S` without `struct`, and `struct S` without the qualifier, -# both parse. -# -# THE THREE BACKSLASH-CONTINUED `#elif defined(…) || \` CONDITIONALS AT LINES 24, -# 37 AND 50 ARE NOT THE TRIGGER. They are the file's most conspicuous feature and -# the obvious suspect, and they are innocent: extracted on their own they parse in -# 8 ms, and — the decisive evidence — this file has EIGHT historical versions, ALL -# EIGHT carry three or four such continued `#elif` conditionals, and only FOUR -# hang. The split is exactly the reference return type: the four that hang all -# declare `inline const struct timespec& stAtime(const struct stat& st)`, and the -# four that parse in milliseconds all return `struct timespec` BY VALUE. A natural -# experiment in the corpus's own history, 4/4, agreeing with the minimisation. -# -# The same 1,861 bytes parse as C++ in 9 ms and produce 14,125 bytes of srcML, so -# a `.hpp` name would never have hit this. That narrowness — C++ declared in a -# `.h`, named namespace, reference to an elaborated struct, cv-qualified -# elaborated struct parameter — is why 188 C and C++ projects produced ONE -# instance of this class rather than thousands. -# -# Blast radius, measured rather than assumed: a bounded probe of ALL 29,100 -# mask-selected C/C++/Java blobs in facebook__sapling's whole history found these -# 4 and no others. 4 of 29,100 is 0.014%. -# -# --------------------------------------------------------------------------- -# WHY THIS IS NON-TERMINATION AND NOT SLOWNESS, for every CLASS 1 AND CLASS 2 -# entry. Class 3 is not a hang at all; its evidence is further down. -# --------------------------------------------------------------------------- -# Zero bytes of output, one core at 100%, and resident memory FLAT, at every -# budget tried. Flat memory with a saturated core rules out backtracking that -# would eventually finish and rules out memory exhaustion. Measured 2026-09-20 -# against the pinned binary, reading the counters from the srcml CHILD and not -# from the `timeout` supervisor, which blocks in wait4 at 0% CPU and would look -# idle: -# -# StatTimes.h as C 5 s / 15 s / 60 s / 300 s 0 B out, 100%, RSS 9.2-9.4 MB -# the other 3 .h as C 15 s / 60 s 0 B out, 100%, RSS 9.3-9.4 MB -# CheckErrors… as Java 15 s / 60 s 0 B out, 100%, RSS 10.1 MB -# all 6 newly listed here also killed at 600 s in the -# pipeline, status 124 from -# `timeout` — not 137, not OOM -# -# srcML v1.1.0 (2025-08-14) is the latest release, so there is nothing to upgrade -# to. A working patch for class 1 was built and validated at 12,016/12,016 -# byte-identical blobs and deliberately NOT adopted, on provenance grounds. Both -# write-ups, and the minimisation behind class 2, live in -# cregit-token-pipeline/.superpowers/sdd/peppy-pondering-music/ — -# srcml-nontermination-analysis.md, srcml-upstream-fix-adjudication.md and -# task-8c-finish-report.md. -# -# --------------------------------------------------------------------------- -# THIS LIST IS NOT THE COMPLETE SET OF NON-TERMINATING BLOBS IN THE CORPUS -# --------------------------------------------------------------------------- -# A FURTHER non-termination class exists and is deliberately NOT listed here -# (it is not class 3 below, which is a crash, not a hang), because excluding it -# is a decision about the dataset's content rather than about a stress test: the -# C++ parser also fails to terminate on `src/hotspot/share/runtime/ -# continuationFreezeThaw.cpp` in tencent__tencentkona-21 — ~105 kB of ORDINARY -# PRODUCTION SOURCE in the JVM runtime, 0 bytes out at 15 s and 120 s with the -# same flat-memory, one-core signature. Denylisting production source removes real -# contributor-behaviour signal, so it is escalated in task-8c-finish-report.md -# rather than decided in this file. Until that decision is taken, -# tencent__tencentkona-21 cannot complete step 2, and the timeout — which DOES -# block publication — is the correct thing to be stopping it. -# -# -# --------------------------------------------------------------------------- -# CLASS 3 — the C and C++ parsers, CRASHING ON A SIGNAL under `--position`. -# NO UPSTREAM ISSUE EXISTS. This is NOT #2361 and NOT class 2: those are -# non-termination, this is a SEGFAULT OR ABORT in milliseconds. 197 blobs, -# 36 files, 18 projects (19 corpus projects are affected — see the shared -# `deflate.c` note above). Cited to silent-empty-sweep.md. -# Ordinary production source, not stress tests, so this exclusion does cost -# real signal — bounded and enumerated below. -# --------------------------------------------------------------------------- -# -# WHY THIS CLASS HAD TO BE ADDED AT ALL. Before the tokenizer was made to fail -# closed, this defect was INVISIBLE. The shell pipeline in tokenizeSrcMl.pl, -# -# srcml -l C --position FILE | srcml2token -# -# returned PIPESTATUS 139,0: srcml died on SIGSEGV, and srcml2token exited 0 -# anyway because it exits 0 even on truncated XML. The caller therefore wrote a -# 0-byte tokenization and REPORTED SUCCESS, and the file silently vanished from -# the dataset with no counter recording it. 36 files in 19 of 186 published -# projects were lost that way. tokenizeSrcMl.pl now reads both PIPESTATUS stages -# and exits 33, and blobExec counts `blobsParserCrashed` and exits 6, so the loss -# is loud. These entries are what lets those 19 projects finish: a denylisted -# blob never reaches the tokenizer at all (Walker.readBlob checks the list before -# opening the blob), so it cannot trip the new gate, and the omission is counted -# as `blobsDenylisted` and named with this reason instead of disappearing. -# -# HOW THE 197 WERE FOUND, and why it is not 36. The sweep measured HEAD only, but -# the pipeline walks WHOLE HISTORIES, so a HEAD-only list would not let a single -# affected project complete. Every historical version of all 36 paths was -# enumerated — `git log --all --full-history -m --raw --no-abbrev`, which is the -# invocation that catches blobs introduced by merge commits and blobs hidden by -# history simplification; a plain `git log --raw` misses both, and did miss one -# project's HEAD blob — giving 904 distinct blobs. Every one of the 904 was then -# run through the tokenizer's real chain: -# -# 197 crash (153 SIGSEGV, 44 SIGABRT) -> listed here -# 707 parse cleanly, exit 0, non-empty -> NOT listed -# 0 time out -> class 3 is disjoint from 1 and 2 -# -# Every blob here was verified individually; none is listed on the strength of -# the sweep alone. The 707 that parse are the control: this is a content-triggered -# crash, not a property of the file name, and the splits are narrow — 3 of 423 -# revisions of apache's `server/util.c`, 51 of 178 of one googleapis test, but -# 24 of 24 of oceanbase's `ob_ctype_uca.cc` and 16 of 16 of kamailio's -# `duktape.c`. -# -# `--position` IS THE TRIGGER FOR MOST OF THESE, BUT NOT FOR ALL, AND THE SWEEP -# OVERSTATES IT. Measured per blob, with and without the flag: -# -# 132 --position is the trigger: exit 0 without it, SIGSEGV with it -# 49 NOT the trigger: srcML dies on a signal EITHER WAY; the flag only -# changes WHICH signal (typically SIGSEGV without, SIGABRT with) -# 16 dropping the flag would not help: srcML exits 1 with a parse -# diagnostic without it, and still yields no tokenization -# -# silent-empty-sweep.md concludes "`--position` is the trigger ... Dropping -# `--position` makes these same files parse cleanly." That is true for 132 of the -# 197, and FALSE for 65. It is false even for 7 of the 33 distinct blobs the sweep -# itself measured, including ALL FOUR of its SIGABRT cases: the sweep's control -# was `ext/liblzma/check/crc32_small.c`, which happens to be in the -# position-only group. Recorded here rather than quietly inherited, because the -# difference matters: dropping `--position` is not a fix, and for 65 of these -# blobs it would not even be a workaround. It cannot be dropped in any case — -# the token format carries `line:col` positions. -# -# So each entry's reason states which of the three it is, measured, rather than -# repeating a single claim across all 197. -# -# NOT SLOWNESS AND NOT A HANG, for every class 3 entry: srcml is dead within -# milliseconds with a core dumped, 0 bytes on stdout, wait status 139 (SIGSEGV) -# or 134 (SIGABRT) reported through bash PIPESTATUS. No budget is involved, which -# is exactly why the blob denylist and not `--blob-timeout` is the right -# instrument, and why these do not and must not block publication. -# -# Language: 119 C, 78 C++. Java is untouched — 273,410 Java files in the corpus -# go through the same tokenizeSrcMl.pl with the same `--position` and produce not -# one crash, so this is srcML's C/C++ position tracking specifically. Rust never -# reaches srcml at all (%LANG_PARSER_REL routes it to rustTokenizer). Size is -# uninformative: 1,023 B to 3,900,518 B, failures at both extremes. -# -# WHAT IS STILL NOT COVERED, said plainly. These 197 make the 36 KNOWN PATHS safe -# across their whole history. They do NOT prove the 19 projects will complete. A -# file whose HEAD version parses but whose older versions crash is invisible both -# to the sweep (HEAD-only) and to this probe (these 36 paths only), and would -# still stop its project at exit 6. Establishing otherwise needs a full-history -# probe of every mask-selected blob in those 19 projects, which was not run: -# that is hundreds of thousands of blobs. Two further gaps are inherited from the -# sweep: `torvalds__linux` has no Parquet and was never measured, and -# `tencent__tencentkona-21` lives only in a backup with no Parquet to join. +# Class 1: srcML Java parser hangs on annotated array types (srcML/srcML#2361). +# Class 2: srcML C parser hangs on C++ inside a .h (no upstream issue). # # sha citation reason 2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65 srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar @@ -298,7 +19,7 @@ afbd81e5789ba060147b0de3be4f01759963299d srcML/srcML#2361 srcML 1.1.0 does not t 99c191e9a208547f5c9b1224c290f106a640de59 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); a third historical version of eden/fs/utils/StatTimes.h, 2,443 B dbb885e0bf046755c4fcfbddfc2454f90c550c62 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); the fourth historical version of eden/fs/utils/StatTimes.h, 2,442 B # -# --- CLASS 3 (srcML signal death under --position); 197 blobs, 36 files, 18 projects --- +# Class 3: srcML C/C++ parsers die on a signal under --position (no upstream issue). 046a9a1a8b09089625dc77d04185415a710cd1aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,549 B 0cdf543499d0ee5e962b37fdd574131f8b03f711 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 117,124 B d1d06fc15b4e6abbd1dffb7c11e6d5c7bdff736e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,431 B diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala index 222ec72d5..3a4806ebb 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala @@ -6,28 +6,10 @@ import java.nio.file.{Files, Path} import scala.jdk.CollectionConverters._ import scala.util.Using -/** Blobs that must never be handed to the tokenizer, keyed by git blob id. - * - * The corpus holds four blobs on which srcML 1.1.0 does not terminate: it - * produces 0 bytes, pegs one core and never finishes, at every budget from 5 s to - * 600 s. The per-blob timeout (see [[BlobExec]]) excludes such a blob too, but - * only after it has spent the whole budget on it in every run. This hang is - * diagnosed, with a minimal reproducer and an open upstream defect, so the list - * excludes it at once and records the reason. - * - * So a denylisted blob takes the exclusion path instead, the same one an - * oversized blob takes: excluded in milliseconds, dropped from every rewritten - * tree, counted on its own, logged with its sha, path and reason, and NOT gating - * the exit status. The timeout stays exactly as it was. - * - * The entries live in a data file rather than in this code - * ([[BlobDenylist.ResourcePath]]), because the list is a documented property of - * the published dataset and a paper has to be able to cite it. - */ +/** Blobs never handed to the tokenizer, keyed by git blob id. The list is a data + * file ([[BlobDenylist.ResourcePath]]) so the dataset can cite it. */ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.Entry]) { - /** The entry for `sha`, or None. Case-insensitive, because a git id is hex and - * both cases name the same object. */ def entryFor(sha: String): Option[BlobDenylist.Entry] = if (bySha.isEmpty) None else bySha.get(sha.toLowerCase) @@ -39,37 +21,23 @@ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.En object BlobDenylist { - /** One excluded blob: the git blob id, an upstream citation, and one line of - * reason. All three are required — an exclusion with no citation and no reason - * is indistinguishable from data loss. */ final case class Entry(sha: String, citation: String, reason: String) { - /** The half of the log line that comes from the data file. */ def describe: String = s"$reason [$citation]" } - /** Classpath location of the shipped list. A resource, so it travels inside the - * assembly jar and cannot go missing between the build and the run. */ val ResourcePath = "/cregit/blobexec/blob-denylist.tsv" val empty: BlobDenylist = new BlobDenylist(Map.empty) private val ShaPattern = "^[0-9a-f]{40}$".r - /** Parse the TSV. Malformed input throws: a denylist that silently drops a line - * would silently re-introduce a blob that costs a 600 s timeout in every run, - * which is precisely the cost this list removes. - * - * `where` names the source in any error message, because the same parser reads - * the shipped resource and a test fixture. - */ + /** Throws on malformed input: a dropped line would silently re-admit a blob. */ def parse(lines: IterableOnce[String], where: String): BlobDenylist = { val acc = scala.collection.mutable.LinkedHashMap.empty[String, Entry] lines.iterator.zipWithIndex.foreach { case (raw, i) => val line = raw.trim if (line.nonEmpty && !line.startsWith("#")) { - // Trailing tabs are tolerated; an empty field in the middle is not. A - // missing citation or a missing reason has to be an error, or the list - // stops being something a paper can cite. + // Trailing tabs are tolerated; an empty field in the middle is not. val fields = line.split("\t", -1).map(_.trim).reverse.dropWhile(_.isEmpty).reverse if (fields.length != 3 || fields.exists(_.isEmpty)) throw new IllegalArgumentException( @@ -104,7 +72,6 @@ object BlobDenylist { parse(src.getLines().toVector, resource)) } - /** The shipped list. Parsed once, on first use, and a parse failure is fatal by - * design: running with a list nobody could read is worse than not running. */ + /** Parsed on first use; a parse failure is fatal by design. */ lazy val shipped: BlobDenylist = fromResource(ResourcePath) } diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala index 143181617..39962351c 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala @@ -9,75 +9,37 @@ import java.util.{Arrays => JavaArrays} import scala.io.Source import scala.sys.process.{Process, ProcessIO} -/** - * Run an external per-blob command and return the resulting `ObjectId`. - * - * Pure helper carved out of the old `BlobExecModifier`. Contract: - * - * - env `BFG_BLOB` = orig blob 40-hex - * - env `BFG_FILENAME` = blob basename (preserved for backward - * compatibility with cregit's `tokenBySha.pl`, - * which keys its on-disk memo by basename) - * - env `BFG_PATH` = blob's full repo-root relative path - * (e.g. `src/lib/foo.c`). Enables tokenizers to - * make path-aware decisions. - * - stdin = original blob bytes - * - stdout = replacement blob bytes - * - exit != 0 → `Skip` (or `Abort` if `abortOnError`) - * - timeout → child killed, `Skip` (never `Abort`, never `Replace`) - * - stdout == stdin → `Skip` (no inserter activity) - * - otherwise → `Replace(newBlob)` - */ +/** Runs a per-blob command: original bytes on stdin, replacement on stdout, env + * `BFG_BLOB`, `BFG_PATH` and `BFG_FILENAME` (the basename, which `tokenBySha.pl` + * keys its memo by). */ object BlobExec { /** Per-blob wall-clock budget for the external command, in seconds. */ val DefaultTimeoutSeconds: Int = 600 - /** Synthetic exit code reported by [[invoke]] when the child was killed for - * exceeding its budget. A real child can never produce it: on Unix a waited - * status is 0..255 (128+signal when killed), so negative values are free. */ + /** Reported for a killed child; negative, so no real exit status can equal it. */ val TimeoutExitCode: Int = -1 - /** Exit status by which the tokenizer reports "srcML died on a signal, or this - * tokenization is otherwise unusable" — the defect that used to arrive here as - * a zero exit with empty stdout and become a silent 0-byte blob. - * - * Emitted by `tokenize/tokenizeSrcMl.pl` (`$PARSER_CRASH_EXIT` there) and - * propagated unchanged by `tokenize.pl` and `tokenizeByBlobId/tokenBySha.pl`. - * The two constants must stay in step; 33 is below 128 so it cannot collide - * with a shell's 128+signal encoding, and is clear of GNU `timeout`'s 124/137 - * and of blobExec's own 2/3/4 exit statuses. */ + /** Must match `$PARSER_CRASH_EXIT` in `tokenize/tokenizeSrcMl.pl`. */ val ParserCrashExitCode: Int = 33 - /** "the waiter never observed an exit status" — distinct from - * [[TimeoutExitCode]] so an interrupted wait can never be mistaken for a - * completed one, nor a completed one for a timeout. */ private val NotExitedSentinel: Int = Int.MinValue /** Seconds GNU `timeout` waits between its SIGTERM and its SIGKILL (`-k`). */ private val KillGraceSeconds: Int = 5 - /** Slack on the JVM-side backstop latch beyond the child's own budget plus - * the kill grace. The latch should only ever fire if `timeout` itself is - * missing or wedged. */ + /** Backstop for the JVM latch, which fires only if `timeout` is missing or wedged. */ private val BackstopSlackSeconds: Int = 25 /** Longest one [[invoke]] can take: the budget, the kill grace and the backstop. */ private[blobexec] def maxChildLifetimeSeconds(timeoutSeconds: Int): Int = math.min(math.max(1, timeoutSeconds).toLong + KillGraceSeconds + BackstopSlackSeconds, Int.MaxValue.toLong).toInt - /** GNU `timeout`'s own statuses: 124 = the budget expired, 137 = 128+SIGKILL, - * i.e. the command ignored SIGTERM and needed the `-k` follow-up. Both mean - * "we killed it", and both must reach the Skip branch rather than the - * `exitCode != 0` branch, which `--abort-on-error` turns into a run abort. */ + /** GNU `timeout`: 124 = budget expired, 137 = SIGKILL after `-k`. */ private val TimeoutStatuses: Set[Int] = Set(124, 137) - /** GNU `timeout`, if it is on PATH. It is the only cheap way to kill the - * whole child *process group*: `Process.destroy()` signals the direct child's - * pid alone, and the tokenizer chain is `tokenBySha.pl` -> `sh` -> `srcml`, - * so the grandchildren survive, inherit the JVM's stderr pipe, and keep the - * reader thread blocked — which is how one wedged blob wedged a whole run. - * Resolved once; the warning is therefore printed at most once per process. */ + /** GNU `timeout` kills the child's whole process group; `Process.destroy()` would + * kill only `tokenBySha.pl` and leave `srcml` holding our pipes open. */ private lazy val gnuTimeout: Option[String] = { val found = sys.env .getOrElse("PATH", "") @@ -104,11 +66,7 @@ object BlobExec { final case class Abort(stderr: String, exitCode: Int) extends Outcome } - /** - * Run `command` against `bytes`. Pure aside from the JVM process and the - * (single-threaded-per-call) jgit inserter. Thread-safe so long as - * `inserter` is confined to the calling thread. - */ + /** Thread-safe if `inserter` is confined to the calling thread. */ def run( bytes: Array[Byte], origSha: String, @@ -128,12 +86,7 @@ object BlobExec { } if (exitCode == TimeoutExitCode) { - // A timed-out child skips exactly one blob. Deliberately *not* routed - // through `abortOnError`: one wedged tokenizer must never take down a - // run that has already folded thousands of commits, and its (discarded, - // possibly truncated) stdout must never be mistaken for a tokenization. - // `onTimeout` is how the caller counts it: a Skip that leaves raw source - // where tokens belong must not be invisible to the run's statistics. + // Not routed through `abortOnError`: one wedged blob must not abort a long run. System.err.println( s"Warning: command [$command] timed out after ${timeoutSeconds}s on blob $origSha " + s"at path [$fullPath]: child killed, blob excluded" @@ -141,15 +94,7 @@ object BlobExec { onTimeout() Outcome.Skip } else if (exitCode == ParserCrashExitCode) { - // srcML died on a signal (SIGSEGV on 32 of the 36 known corpus files, - // SIGABRT on the other 4), or the token stream came back empty, which a - // healthy srcML parse never is. Treated exactly like a timeout in flow, and - // deliberately so: Skip rather than Replace, so the (empty or truncated) - // stdout can never become a blob; and NOT routed through `abortOnError`, so - // one crashing blob cannot take down a run that has folded thousands of - // commits. Counted separately from a timeout via `onParserCrash` because it - // is a different defect with a different fix — more time never helps a - // segfault — and because the operator needs to know which one happened. + // Skip like a timeout, but counted apart: more time never fixes a crash. System.err.println( s"Warning: command [$command] reported a parser crash (exit $exitCode) on blob $origSha " + s"at path [$fullPath]: srcML died or produced no tokens, so this blob is excluded " + @@ -167,17 +112,7 @@ object BlobExec { logError(command, origSha, fullPath, exitCode, stderr) if (abortOnError) Outcome.Abort(stderr, exitCode) else Outcome.Skip } else if (stdout.isEmpty && bytes.nonEmpty) { - // Defence in depth, and the other half of "zero output is never success". - // The tokenizer now detects its own empty stream and exits - // ParserCrashExitCode, but this is the backstop for every way that could be - // bypassed — a wrapper that swallows the status, a different tokenizer, a - // future parser. Consuming real source and emitting nothing is a failure - // whatever the exit code says, and inserting it is precisely how a 1,263-byte - // C file became a 0-byte blob in a published dataset. - // - // The two cases the brief asks to be told apart are told apart here by - // `bytes.nonEmpty`: a genuinely empty input producing empty output is - // untouched and falls through to the `equals` branch below as a plain Skip. + // Backstop for any tokenizer that exits 0 with no output: never insert it. System.err.println( s"Warning: command [$command] exited 0 but produced no output for the ${bytes.length}-byte " + s"blob $origSha at path [$fullPath]. Refusing to write an empty tokenization: counting " + @@ -192,11 +127,7 @@ object BlobExec { } } - /** Visible for testing. Runs the process and returns (exit, stdout, stderr). - * - * The child is bounded by `timeoutSeconds`; on expiry its whole process - * group is killed (via GNU `timeout`) and [[TimeoutExitCode]] is returned - * with empty stdout/stderr. */ + /** Returns (exit, stdout, stderr), or [[TimeoutExitCode]] and no output if killed. */ private[blobexec] def invoke( bytes: Array[Byte], origSha: String, @@ -219,19 +150,11 @@ object BlobExec { try stderrBuilder.append(src.mkString) finally src.close() } - // daemonizeThreads = true: the stdout/stderr readers must not keep the JVM - // alive. `exitValue()` still joins them on the healthy path, so this only - // matters at JVM exit — and on the timeout path they are deliberately - // abandoned while blocked in `FileInputStream.readBytes`. Left non-daemon, - // two such threads kept the process running after the walk had finished, - // reintroducing the very stall this timeout exists to remove. + // Daemon readers: on timeout they are abandoned while blocked and must not + // keep the JVM alive. val io = new ProcessIO(writeStdin, readStdout, readStderr, daemonizeThreads = true) val secs = math.max(1, timeoutSeconds) - // Prefer the kernel over a pid-by-pid kill: GNU `timeout` runs the command - // in its own process group and signals the *group* on expiry, so `sh`, - // `tokenize.pl`, `srcml` and `srcml2token` all die together instead of - // orphaning themselves onto the JVM's pipes. val argv = gnuTimeout match { case Some(bin) => Seq(bin, "-k", KillGraceSeconds.toString, secs.toString, command) case None => Seq(command) @@ -246,12 +169,8 @@ object BlobExec { ) val proc = pb.run(io) - // A stuck srcml/ctags used to park the caller forever: the child stops - // producing output but holds its stdout open, the reader thread blocks in - // pipe_read, `exitValue()` (which joins the io threads) never returns, and - // the whole project goes silent. Bound the child instead of trusting it to - // exit. `exitValue()` is moved onto a daemon thread so the timeout can be - // observed even when that join is the thing that is wedged. + // `exitValue()` joins the io threads, which a wedged child can block forever, + // so it runs on a daemon thread and the caller waits on a latch. val finished = new java.util.concurrent.CountDownLatch(1) val exitHolder = new java.util.concurrent.atomic.AtomicInteger(NotExitedSentinel) val waiter = new Thread( @@ -265,16 +184,10 @@ object BlobExec { waiter.setDaemon(true) waiter.start() - // `timeout` is the primary kill, so the latch is only a backstop: give it - // the child's own budget, the -k grace, and slack. It fires only when - // `timeout` is absent or itself wedged. val latchBudget = maxChildLifetimeSeconds(secs).toLong - // The io threads are abandoned rather than joined on every timeout path: if - // a surviving grandchild still holds a pipe, joining them is exactly the - // hang we are escaping. That makes `stdoutBuilder`/`stderrBuilder` live, - // racy state, so neither is read there — a timed-out blob's output is - // discarded by the caller anyway. + // On the timeout paths the io threads are abandoned, so the builders are racy: + // do not read them there. if (!finished.await(latchBudget, java.util.concurrent.TimeUnit.SECONDS)) { System.err.println( s"blobExec: no exit from [$command] ${latchBudget}s after start on blob $origSha " + @@ -286,9 +199,7 @@ object BlobExec { } else exitHolder.get() match { case NotExitedSentinel => - // The waiter was interrupted (e.g. pool.shutdownNow) before any status - // was observed. The blob may well have succeeded, but we cannot claim - // that, so it is reported as a timeout: Skip, never Replace. + // Interrupted before any status: the result is unknown, so Skip. System.err.println( s"blobExec: wait for blob $origSha ($fullPath) was interrupted before the child " + "reported a status; treating it as a timeout and discarding its output" diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index efcc33307..d8c55df67 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -29,22 +29,11 @@ import java.nio.file.{Files, Paths} */ object Main { - /** Exit status when `--retokenize` was asked for and would have invalidated - * nothing. - * - * Its own status, and not 3, because it is a different kind of answer. 3 means - * "this memo does not match this run, so the run did not start". This means - * "the run could have started, and the invalidation you asked for was a no-op". - * An operator scripting a corpus-wide re-tokenization needs to tell those - * apart: the first is a refusal to proceed, the second is a request that did - * nothing, and treating the second as success is how a poisoned cache gets - * published. 7 is the next free status after 1, 2, 3, 4, 5, 6; - * `retokenizeStatusIsUnique` in MainOptionsSpec pins that. */ + /** Exit status when `--retokenize` would invalidate nothing. Not 3 (memo mismatch): + * a no-op request must be told apart from a refusal, and never read as success. */ private[blobexec] val RetokenizeIneffectiveExitStatus = 7 - /** The process exit status for a finished walk: 2 on an abort, else 0. - * Excluded blobs (oversized, denylisted, or whose tokenizer failed) are - * named on EXCLUDED lines and do not change it. */ + /** 2 on an abort, else 0: excluded blobs are named on EXCLUDED lines, not failed. */ private[blobexec] def exitStatus(stats: WalkStats): Int = if (stats.aborted) 2 else 0 @@ -364,9 +353,6 @@ object Main { sys.exit(1) } - // A shard writes its own fresh dst.git and blobmap.db every run, so there is - // never a recorded mask for --mask-widened to widen. Refusing says so instead - // of letting the flag be a silent no-op; --warm is the sharded equivalent. if (shard.isDefined && maskWidened) { System.err.println("Error: --mask-widened has nothing to do under --shard: each shard builds a " + "fresh dst.git and blobmap.db, so no mask is recorded to widen. Reuse a prior run's " + @@ -375,9 +361,6 @@ object Main { sys.exit(1) } - // Every way --retokenize could end up doing nothing, refused before the walk. - // A flag whose whole purpose is to force work must never be able to run and - // force none. if (retokenizeExtensions.nonEmpty) { if (tokenizerIdentity.isEmpty) { System.err.println( @@ -470,9 +453,7 @@ object Main { val dbParent = dbPath.getParent if (dbParent != null && !Files.isDirectory(dbParent)) Files.createDirectories(dbParent) - // Loaded here rather than on first use: a jar built without the resource, or a - // malformed line in it, must stop the run now and say so, not silently hand a - // known non-terminating blob to srcml an hour into the walk. + // Eager, so a missing or malformed denylist stops the run now, not mid-walk. val denylist = try BlobDenylist.shipped catch { @@ -499,9 +480,6 @@ object Main { val src: FileRepository = openSrc(srcPath) val dst: FileRepository = openOrInitDst(dstPath) - // The reachability probe the widening needs. It reads dst, which is why the - // whole decision lives here and not inside Mapping.open's signature alone: - // `blob_map` is only meaningful next to the repository its new_blob ids are in. val widening = if (!maskWidened) None else Some(Mapping.MaskWidening( @@ -514,10 +492,8 @@ object Main { report = msg => println(s"blobExec: $msg") )) - // The invalidation, wired to the two things only this layer can supply: the - // dst reachability probe, and a memo purge that reads the original blobs out - // of src to recompute their content sha1 — the memo's only key - // (tokenizeByBlobId/tokenBySha.pl:76). + // The memo is keyed on content sha1 alone (tokenizeByBlobId/tokenBySha.pl:76), + // so the purge re-reads the original blobs from src. val retokenize = if (retokenizeExtensions.isEmpty) None else Some(Mapping.Retokenize( @@ -566,15 +542,12 @@ object Main { src.close(); dst.close() sys.exit(3) case n: Mapping.NothingInvalidatedException => - // Deliberately NOT 0. The walk could have run; the invalidation asked for - // could not. Exiting 0 here would let a corpus script publish a project - // whose poisoned entries were never touched. + // Not 0: a corpus script would publish entries that were never invalidated. System.err.println(s"Error: ${n.getMessage}") src.close(); dst.close() sys.exit(RetokenizeIneffectiveExitStatus) case i: IllegalArgumentException => - // Mapping.open's own preconditions on the flag combinations, re-checked - // there so a library caller cannot bypass what main() validates. + // Mapping.open re-checks main()'s flag rules, for library callers. System.err.println(s"Error: ${i.getMessage}") src.close(); dst.close() sys.exit(1) @@ -634,9 +607,7 @@ object Main { } if (stats.blobsOversized > 0) { - // Reported, never fatal. An oversized blob is deterministic and fully - // explained by the EXCLUDED lines above, so gating publication on it would - // only mean this project could never publish while telling us nothing new. + // Not fatal: an oversized blob is deterministic and named on its EXCLUDED line. System.err.println( s"blobExec: ${stats.blobsOversized} blob(s) were excluded as oversized (>= " + s"${Walker.MaxBlobBytes} bytes, JGit's stream-file threshold). Each one is named with " + @@ -657,10 +628,7 @@ object Main { ) } - // Always exit explicitly. A timed-out blob abandons its (daemon) reader - // threads while they are blocked on a pipe, and on the pre-fix build the - // JVM outlived the finished walk on exactly those threads — a silent stall - // behind a done-line that read `aborted=false`. + // Exit explicitly: a timed-out blob's daemon reader threads can keep the JVM alive. sys.exit(exitStatus(stats)) } diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala index 3c09c4911..d6739ae77 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala @@ -105,19 +105,14 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends def setMeta(key: String, value: String): Unit = Mapping.execute(insMeta, key, value) - /** The tokenizer identity recorded for `extension`, if any. */ def storedTokenizerId(extension: String): Option[String] = getMeta(Mapping.TokenizerIdKeyPrefix + extension) def setTokenizerId(extension: String, value: String): Unit = setMeta(Mapping.TokenizerIdKeyPrefix + extension, value) - /** Every extension whose recorded tokenizer identity differs from `identity`. - * - * An extension with nothing recorded is NOT a change: it is a map written - * before identities were recorded at all, which describes every blob map on - * this machine today. Treating that as a change would refuse all 188 - * projects at once and say nothing true. */ + /** Extensions whose recorded identity differs from `identity`. Nothing recorded + * is not a change: older maps carry no identity at all. */ def tokenizerIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = identity.byExtension.toVector.sorted.flatMap { case (ext, requested) => storedTokenizerId(ext) match { @@ -127,28 +122,13 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } } - /** The subset of [[tokenizerIdentityChanges]] that can actually poison this run: - * the extensions whose tokenizer moved AND which have cached tokenizations to - * reuse. - * - * The filter is not an optimisation, it is what keeps the mechanism usable. The - * pipeline reports one identity per extension it can parse, for every project, - * because the identity is a property of the checkout and not of the project. A - * Java-only project therefore carries a `c` identity it has no rows for — and - * refusing that project because the C toolchain changed would be a refusal with - * no remedy: `--retokenize=c` would then fail too, correctly, with "no tokenized - * row carries that extension". A deadlock where the only way forward is to - * delete the work. - * - * With no rows there is nothing to serve from cache, so the new identity is - * simply recorded: whatever gets tokenized from now on is tokenized by the - * tokenizer that is recorded. */ + /** The changed extensions that also have cached tokenizations. Refusing one + * with no rows would deadlock: `--retokenize` of it fails too, as a no-op. */ def cachePoisoningIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = tokenizerIdentityChanges(identity) .filter(c => countTokenizedRowsForExtensions(Set(c.extension)) > 0L) - /** How many TOKENIZED rows sit on a path with one of these extensions. Zero is - * what makes a `--retokenize` request a no-op, and a no-op has to fail. */ + /** Zero makes a `--retokenize` request a no-op, and a no-op has to fail. */ def countTokenizedRowsForExtensions(extensions: Set[String]): Long = { val st = conn.createStatement() try { @@ -159,14 +139,8 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } - /** The distinct ORIGINAL blob ids of the tokenized rows for these extensions. - * Distinct because the memo is keyed on content, so the same blob reached - * under two paths is one memo entry, and hashing it twice is wasted reads. - * - * Materialised rather than streamed: the caller hands the whole vector to the - * memo purge, which must complete before anything is deleted from the map. - * The corpus figure is 741,869 rows for a `.rs` invalidation, which is a - * vector of 40-character strings — tens of megabytes, not gigabytes. */ + /** Distinct, because the memo is keyed on content: one blob under two paths is + * one memo entry. */ def tokenizedOrigBlobsForExtensions(extensions: Set[String]): Vector[String] = { val st = conn.createStatement() try { @@ -196,7 +170,6 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } - /** How many blob_map rows satisfy `predicate`. */ private def countWhere(predicate: String): Long = { val st = conn.createStatement() try { @@ -205,7 +178,6 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } - /** Row counts of the four maps, for the mask-widening report. */ def rowCounts: Mapping.RowCounts = { def one(sql: String): Long = { val st = conn.createStatement() @@ -223,27 +195,8 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends ) } - /** The first `blob_map` path of a TOKENIZED row (orig_blob <> new_blob) that - * `newMask` no longer selects, or None when every one still matches. - * - * Restricted to tokenized rows on purpose, and that restriction is the whole - * correctness argument. `blob_map` also holds IDENTITY rows (orig == new) for - * every blob the walker merely passed through — `README.md`, `CMakeLists.txt`, - * a `.pdf` — written by Walker's `unchangedBlobs`. Those paths match no mask - * that ever existed, so a check over all rows refuses every project on this - * machine (measured: 198 of 198). A tokenized row, by contrast, exists only - * because the old mask selected its path, so "does the new mask still select - * it?" is exactly the superset question, asked of the data rather than of two - * regexes. - * - * Matched against the BASENAME, because that is what Walker matches - * (`fileMask.findFirstIn(tw.getNameString)`). Anchoring makes the two - * equivalent for every mask this pipeline has used, but replicating the - * walker is the point: the check must answer what the walker will do, not - * what a reader assumes it does. - * - * Streams the result set rather than materialising it: torvalds__linux holds - * 2.7 M tokenized rows. */ + /** The first tokenized-row path `newMask` no longer selects. Identity rows are + * skipped: their paths match no mask. Matched on the basename, as Walker does. */ def firstPathNarrowedBy(newMask: Regex): Option[String] = { val st = conn.createStatement() try { @@ -259,16 +212,12 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } - /** Up to `size` (path, new_blob) pairs from the TOKENIZED rows, spread across - * the table rather than taken from its head, so the sample is not confined to - * whatever the first commit happened to touch. Deterministic: the same DB - * gives the same sample, so a refusal is reproducible. */ + /** Up to `size` tokenized (path, new_blob) pairs, spread across the table and + * deterministic, so a refusal is reproducible. */ def sampleTokenizedNewBlobs(size: Int, excludeExtensions: Set[String] = Set.empty): Vector[(String, String)] = { require(size > 0, "sample size must be positive") - // The probe must ask about the rows that will be KEPT. Under --retokenize the - // rows for the named extensions are about to be deleted, and their new_blob - // ids may legitimately have been garbage-collected out of dst already; asking - // about those would refuse a correct invalidation. + // Skip rows --retokenize is about to delete: their new_blob may already be + // gone from dst. val keep = if (excludeExtensions.isEmpty) Mapping.TokenizedPredicate else s"${Mapping.TokenizedPredicate} AND NOT ${Mapping.extensionPredicate(excludeExtensions)}" @@ -293,31 +242,9 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } - /** Discard everything a wider mask invalidates and record the new mask. - * - * Four deletions, in one transaction, each for its own reason: - * - * - `commit_map`: a commit names a tree, and every tree gains entries, so - * every rewritten commit id changes. - * - `ref_map`: its rows name commits. The FK cascades from `commit_map`, but - * deleting it explicitly means this does not depend on a per-connection - * pragma being on. - * - `tree_map`: trees gain entries. - * - `blob_map` IDENTITY rows only. This is the load-bearing one. An identity - * row says "this path was not selected, so its bytes pass through". Under a - * wider mask some of those paths ARE selected now, and `getBlob` would - * serve the identity row as a cache hit — putting RAW SOURCE into the - * tokenized repository, invisibly, for exactly the files this widening - * exists to tokenize. Measured on this corpus: 456,363 such rows across 88 - * of 198 projects. Identity rows on still-unselected paths are never read - * (`getBlob` is only called inside the mask branch), so dropping all of - * them is free and re-inserting them is a walk, not a tokenize. - * - * What survives is every row where orig_blob <> new_blob: a real - * tokenization, keyed by (blob, path), so same content and same extension and - * therefore same language and same tokens. That is 94.4% of the work. - * - * Returns the counts before and after, for the caller to report. */ + /** Drop commit, ref and tree maps and the identity blob rows, which a wider mask + * would otherwise serve as cache hits of raw source, and record the new mask. + * ref_map is deleted explicitly, not left to the per-connection FK pragma. */ def applyMaskWidening(oldMask: String, newMask: String, atEpochSeconds: Long): Mapping.WideningResult = { val before = rowCounts inTx { @@ -335,35 +262,9 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends Mapping.WideningResult(before, rowCounts) } - /** Discard everything a changed tokenizer invalidates for `extensions`, and - * record the new identity for them. - * - * Four deletions and the identity update, in ONE transaction, because a - * partially invalidated map is worse than either a stale one or a clean one: - * it is a state no flag describes and no later run can detect. - * - * - `blob_map` TOKENIZED rows for these extensions. These are the poisoned - * entries: same bytes, same path, but produced by the tokenizer that has - * since been corrected. - * - `tree_map`. A tree names the blobs under it, so a tree containing a - * re-tokenized file has a different id; and a retained `tree_map` row - * short-circuits the re-walk of that whole subtree (buildTreePlan consults - * `getTree` before it looks at anything below), so keeping it would leave - * the tree pointing at the old token blob and invalidate nothing. - * - `commit_map`, because commits name trees, and `ref_map`, because refs - * name commits. ref_map would cascade from commit_map's FK, but only if the - * per-connection pragma is on, so it is deleted explicitly. - * - * What survives is every tokenized row for every OTHER extension. That is the - * whole point: re-tokenizing a corpus costs 88% of total pipeline time, and a - * `.rs`-only defect has no business touching the C work. Trees and commits are - * rebuilt by a walk, which is the cheap part. - * - * Identity pass-through rows (orig == new) are kept. They record "this path was - * not selected, its bytes pass through", the mask has not changed here, and so - * the same paths are still not selected. (That is exactly the case - * `--mask-widened` must handle differently, which is why the two flags refuse - * to run together.) */ + /** Drop the tokenized blob rows for `extensions` plus tree, commit and ref maps, + * and record the new identity, in one transaction: a partial invalidation is + * undetectable. Identity rows stay, because the mask has not changed. */ def applyRetokenize( extensions: Set[String], identity: TokenizerIdentity, @@ -383,9 +284,8 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends s"DELETE FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + s"AND ${Mapping.extensionPredicate(extensions)}") } finally st.close() - // Every extension the run knows about gets its identity recorded, not just - // the invalidated ones: an extension with nothing recorded is a hole the - // next run cannot detect a change through. + // Record every extension, not only the invalidated ones: an unrecorded one + // is a hole the next run cannot detect a change through. identity.byExtension.foreach { case (ext, value) => if (extensions.contains(ext) || storedTokenizerId(ext).isEmpty) setTokenizerId(ext, value) } @@ -426,72 +326,35 @@ object Mapping { /** Mismatch between the recorded command/mask and the values passed in. */ final class MetaMismatchException(message: String) extends RuntimeException(message) - /** `--mask-widened` was passed but the new mask does not select something the - * old one did, so it is not a widening. Distinct from MetaMismatchException - * because the cause is different — the flag was given and the data refused it, - * rather than the flag being absent — but Main maps both to exit 3, which - * already means "this memo does not match this run". */ + /** `--mask-widened` was passed but the new mask is not a widening. */ final class MaskNarrowedException(message: String) extends RuntimeException(message) - /** `--mask-widened` was passed and the map is reusable, but the `new_blob` ids - * it holds do not resolve in dst, so reusing it would dangle every reference. */ + /** Reusing the map would dangle: its `new_blob` ids do not resolve in dst. */ final class DanglingNewBlobException(message: String) extends RuntimeException(message) - /** A recorded tokenizer identity does not match this run's, and the operator - * did not ask for the affected entries to be invalidated. Reusing them would - * reproduce the tokens of the tokenizer that has since been corrected — - * silently, because nothing downstream can tell one token stream from - * another. Same exit status as a meta mismatch (3): the cause is the same - * class of thing, "this memo does not match this run". */ + /** A recorded tokenizer identity changed and `--retokenize` did not name it. */ final class TokenizerChangedException(message: String) extends RuntimeException(message) - /** `--retokenize` was passed and would have invalidated nothing. - * - * This exception is the whole safety property of the flag. An invalidation - * that quietly invalidates nothing and exits 0 is indistinguishable, in a log, - * from one that worked — and the operator then publishes a dataset believing - * the defect is out of it. Every path that reaches "nothing changed" raises - * this instead of returning. */ + /** `--retokenize` would invalidate nothing; exiting 0 would look like success. */ final class NothingInvalidatedException(message: String) extends RuntimeException(message) - /** A recorded identity that differs from the one this run reports. */ final case class IdentityChange(extension: String, stored: String, requested: String) - /** What distinguishes a real tokenization from a pass-through. A tokenized row - * exists only because the mask selected its path; an identity row exists - * because it did not. Every decision in the mask-widening path turns on this - * one predicate, so it is written once. */ private[blobexec] val TokenizedPredicate = "orig_blob <> new_blob" - /** meta keys recording that a widening happened, so the provenance of a reused - * blob_map is in the database rather than only in a log. */ private[blobexec] val MaskWidenedFromKey = "mask_widened_from" private[blobexec] val MaskWidenedAtKey = "mask_widened_at" - /** meta key prefix for the per-extension tokenizer identity: one row per - * extension, `tokenizer_id.rs`, `tokenizer_id.c`. Per extension rather than - * one combined value, because `--retokenize` has to be able to invalidate one - * language without disturbing the recorded state of the others. */ + /** One meta row per extension, so `--retokenize` can invalidate one language. */ private[blobexec] val TokenizerIdKeyPrefix = "tokenizer_id." - /** meta keys recording that an invalidation happened, and what it replaced, so - * a reused blob map carries its own provenance. */ private[blobexec] val RetokenizedAtKey = "retokenized_at" private[blobexec] val RetokenizedExtensionsKey = "retokenized_extensions" private[blobexec] val RetokenizedFromKey = "retokenized_from" - /** "Does this path carry one of these extensions?", as SQL. - * - * `lower(path) LIKE '%.rs'` rather than a suffix computed in Scala, so the - * whole selection stays one statement over a table that holds 2.7 M rows for - * the largest project. Case-insensitive because the mask is (`(?i)` in - * CregitLanguages::file_mask) and `.C` and `.H` are real files in this corpus. - * - * The dot is part of the pattern, which is what stops `.c` from selecting - * `.cc` and `.cpp`. There is nothing to escape: [[TokenizerIdentity]] admits - * only `[a-z0-9+]+`, an alphabet with no quote, no `%` and no `_`. The require - * is there so that stays true if a future caller builds the set some other - * way. */ + /** SQL for "path has one of these extensions", case-insensitive like the mask. + * The dot stops `.c` from matching `.cc`; TokenizerIdentity's alphabet leaves + * nothing to escape. */ private[blobexec] def extensionPredicate(extensions: Set[String]): String = { require(extensions.nonEmpty, "extensionPredicate needs at least one extension") extensions.foreach(e => require( @@ -500,10 +363,7 @@ object Mapping { extensions.toVector.sorted.map(e => s"lower(path) LIKE '%.$e'").mkString("(", " OR ", ")") } - /** How many `new_blob` ids to probe in dst. Every retained row's id must exist - * there, but reading 2.7 M of them is minutes of object lookup before the walk - * starts; a spread sample catches the failure this guards against, which is a - * dst that was wiped or never written, not one blob gone missing. */ + /** A sample, not all 2.7 M ids: it guards against a wiped dst, not one lost blob. */ private[blobexec] val ReachabilitySampleSize = 256 final case class RowCounts(blobTokenized: Long, blobIdentity: Long, tree: Long, @@ -518,26 +378,9 @@ object Mapping { blobsInvalidated: Long, memo: TokenizerMemo.PurgeReport) - /** The opt-in behind `--retokenize`, and the four things it has to be given. - * - * `extensions` is what makes it surgical. Re-tokenizing a corpus is 88% of - * total pipeline time, so a defect in one language's tokenizer must invalidate - * one language's entries. - * - * `newBlobResolves` answers "does this object id exist in dst?", for the rows - * that are KEPT. Not optional and with no default, for the same reason as in - * [[MaskWidening]]: `blob_map`'s `new_blob` ids live in dst and nowhere else. - * - * `purgeMemo` is handed the original blob ids being invalidated and must delete - * their entries from the content-addressed memo. It is not optional either, and - * that is the point: the memo is keyed on `sha1(contents)` alone - * (tokenizeByBlobId/tokenBySha.pl:76), with no tokenizer in the key, so - * dropping a `blob_map` row on its own just moves the stale answer one layer - * down. Two layers, one flag, no way to do one without the other. - * - * `report` receives the human-readable account of what was invalidated and - * what was kept. Injected so the decision is testable without capturing - * stdout. */ + /** Opt-in for `--retokenize`. `purgeMemo` is mandatory: the memo is keyed on + * contents alone (tokenBySha.pl), so dropping only blob_map rows would serve + * the stale tokens from one layer down. */ final case class Retokenize( extensions: Set[String], newBlobResolves: String => Boolean, @@ -613,41 +456,21 @@ object Mapping { } finally st.close() } - /** The opt-in behind `--mask-widened`, and the two things it has to be given. - * - * `newBlobResolves` answers "does this object id exist in dst?". It is not - * optional and has no default, because `blob_map`'s `new_blob` ids live in - * dst and nowhere else: reusing the map against a dst that was wiped, or was - * never written, leaves every retained reference dangling and the failure - * surfaces as a corrupt repository several steps later. A default would let a - * caller skip the check by forgetting about it. - * - * `report` receives the human-readable account of what was kept and what was - * dropped. Injected so the decision is testable without capturing stdout. */ + /** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids + * live only in dst, and a wiped dst would dangle every reused row. */ final case class MaskWidening(newBlobResolves: String => Boolean, report: String => Unit, nowEpochSeconds: () => Long = () => System.currentTimeMillis() / 1000L, sampleSize: Int = ReachabilitySampleSize) - /** Open/create the SQLite DB at `path` (schema created if absent) and validate - * `command`/`mask` against `meta`: mismatch throws MetaMismatchException, - * absence records them. - * - * `maskWidening` is the ONLY way past the mask refusal, and it is deliberately - * narrow. `command` is still checked exactly as before: a different tokenizer - * means different tokens for the same bytes, and no amount of mask reasoning - * covers that. */ + /** Open/create the DB at `path` and check `command`/`mask` against `meta`. + * `maskWidening` relaxes only the mask check, never the command check. */ def open(path: Path, command: String, mask: String, warm: Option[Path] = None, maskWidening: Option[MaskWidening] = None, tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, retokenize: Option[Retokenize] = None): Mapping = { - // Refused rather than composed. Both invalidations are verified against the - // rows before they touch anything, and running them together would mean - // verifying each against a state the other was about to change: a widening - // decides what to do with identity rows on the assumption the tokenizations - // are valid, and an invalidation decides which tokenizations to drop on the - // assumption the mask has not moved. One at a time, each with its own - // verified report, and the second run is a resume. + // Refused, not composed: each invalidation is verified against a state the + // other would change. Run one, then the other as a resume. require(!(maskWidening.isDefined && retokenize.isDefined), "--mask-widened and --retokenize cannot be used in the same run: each verifies its own " + "precondition against the rows, and together each would be verifying against a state the " + @@ -681,8 +504,7 @@ object Mapping { case Some(w) => widenOrCheckMask(m, mask, w) case None => checkOrSetMeta(m, "mask", mask) } - // Last, because it is the only check that can delete rows, and it must not do - // so against a map whose command or mask this run has already refused. + // Last: the only check that deletes rows must not run on a refused map. retokenize match { case Some(r) => retokenizeOrRefuse(m, tokenizerIdentity, r) case None => if (tokenizerIdentity.nonEmpty) checkOrRecordTokenizerIdentity(m, tokenizerIdentity) @@ -690,55 +512,21 @@ object Mapping { m } - /** The default path: record what is not recorded yet, and refuse if anything - * recorded disagrees. - * - * Refusing is not the same thing as invalidating, and the distinction is the - * whole reason this is safe to make the default. Nothing is deleted here. The - * 186 already-published projects are untouched — a project that is never run - * again is never refused either — and a project that IS re-run with a changed - * tokenizer stops instead of quietly reproducing the old tokens. - * - * An extension with nothing recorded is not a change. That is what every blob - * map in this corpus looks like today, and calling it a change would refuse - * every project at once on the strength of no evidence at all. */ + /** Record identities not yet recorded; refuse on a recorded mismatch. Deletes + * nothing, so a re-run with a changed tokenizer stops instead of reusing. */ private def checkOrRecordTokenizerIdentity(m: Mapping, identity: TokenizerIdentity): Unit = { val changes = m.cachePoisoningIdentityChanges(identity) if (changes.nonEmpty) throw new TokenizerChangedException(tokenizerChangedMessage(changes)) - // Nothing recorded yet, or recorded but with no rows to poison: record the - // current value either way. Leaving an extension unrecorded is the hole this - // whole mechanism exists to close, and leaving a superseded value in place for - // an extension with no rows would refuse the next run for no reason. + // Also overwrite a superseded value with no rows behind it, or it would + // refuse the next run for no reason. identity.byExtension.foreach { case (ext, value) => if (!m.storedTokenizerId(ext).contains(value)) m.setTokenizerId(ext, value) } } - /** The `--retokenize` path. Five outcomes, in this order, and the order is the - * safety property: nothing is deleted until every check has passed. - * - * 1. A changed tokenizer the flag did not name: refuse, naming it. Invalidating - * the named extensions first and then refusing would leave a map no flag - * describes — part redone under a new tokenizer, part stale, and the - * recorded identity a mixture of both. - * 2. No tokenized row carries any of the named extensions: refuse. This is the - * no-op case, and a no-op that exits 0 is the failure this flag exists to - * prevent. - * 3. A sampled `new_blob` of a RETAINED row does not resolve in dst: refuse. - * Those ids live only there, so the map is reusable only alongside it. - * 4. The memo purge. It runs BEFORE the transaction because the two failure - * directions are not symmetric: a memo entry deleted for nothing costs one - * tokenizer invocation, a memo entry kept serves the stale tokens the whole - * exercise is meant to remove. If the purge throws, nothing has been - * invalidated and the run stops. - * 5. The purge found no entries at all: refuse. With a non-empty blob_map the - * tokenized rows were produced by tokenBySha.pl, which memoizes every one - * of them, so "none of them are in this directory" means the directory is - * not the memo — and the real memo would answer the re-tokenization with - * the stale tokens. - * - * Only after all five does anything change, and then it changes in one - * transaction. */ + /** Refuse an unnamed change, a no-op, a dangling dst sample, or a purge that + * found nothing. The purge runs before the transaction: a needless purge costs + * a re-tokenize, a skipped one serves stale tokens. */ private def retokenizeOrRefuse(m: Mapping, identity: TokenizerIdentity, r: Retokenize): Unit = { val uncovered = m.cachePoisoningIdentityChanges(identity).filterNot(c => r.extensions.contains(c.extension)) if (uncovered.nonEmpty) @@ -821,18 +609,8 @@ object Mapping { "extension's tokenizations, and requires --memo-dir and a step-2 resume." } - /** The `--mask-widened` path. Four outcomes, in this order, and the order is - * the safety property: nothing is written until both checks have passed. - * - * 1. No stored mask (a fresh DB): record it. Nothing to reuse, nothing to - * check — the flag is a no-op and says so. - * 2. Stored mask equals the new one: an ordinary resume. The flag is a no-op. - * 3. A tokenized row's path is no longer selected: this is a NARROWING, not a - * widening. Refuse, naming the path, having written nothing. - * 4. A sampled `new_blob` does not resolve in dst: refuse, naming it, having - * written nothing. - * - * Only after 3 and 4 both pass does anything change. */ + /** No stored mask or an equal one: no-op. Otherwise refuse a narrowing or a + * dangling dst sample before writing anything. */ private def widenOrCheckMask(m: Mapping, mask: String, w: MaskWidening): Unit = m.getMeta("mask") match { case None => diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala index 0aa8bbbd9..f457d65d1 100644 --- a/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala @@ -1,25 +1,8 @@ package cregit.blobexec -/** Which tokenizer produced the tokens for each file extension. - * - * The problem this type exists for. `blob_map` reuse was decided on `command` - * and `mask` alone (see `Mapping.open`), and `command` is the fixed path - * `tokenizeByBlobId/tokenBySha.pl` — a constant for the life of the pipeline. - * So when the Rust tokenizer's output format was corrected in 729643e, nothing - * about the recorded metadata changed, every cached `.rs` row stayed a cache - * hit, and a run with the corrected tokenizer reproduced the old, wrong tokens - * exactly. The mask cannot stand in for this: the mask decides WHICH files are - * tokenized and never HOW. - * - * The identity is per EXTENSION rather than per language, because the extension - * is what the cache rows can be selected by: `blob_map` stores paths, not - * languages. `CregitLanguages.pm` maps extension to language and language to - * parser, so the pipeline collapses that chain into one value per extension - * before handing it over (see `tokenize/tokenizerIdentity.pl`). - * - * The value is opaque here on purpose. This code never needs to know whether it - * is a hash of a binary, a version string the tokenizer reports, or a digest of - * a whole toolchain — only whether it is the same string as last time. */ +/** Opaque tokenizer identity per file extension, so `blob_map` reuse notices a + * changed tokenizer. Per extension because `blob_map` stores paths, not + * languages (see `tokenize/tokenizerIdentity.pl`). */ final case class TokenizerIdentity(byExtension: Map[String, String]) { def isEmpty: Boolean = byExtension.isEmpty @@ -27,8 +10,7 @@ final case class TokenizerIdentity(byExtension: Map[String, String]) { def extensions: Set[String] = byExtension.keySet def get(extension: String): Option[String] = byExtension.get(extension) - /** Canonical, sorted rendering: the same map always prints the same string, so - * it can go in a log or a refusal message and be compared by eye. */ + /** Sorted, so the same map always renders the same string. */ def render: String = byExtension.toVector.sorted.map { case (e, id) => s"$e=$id" }.mkString(",") } @@ -37,26 +19,15 @@ object TokenizerIdentity { val empty: TokenizerIdentity = TokenizerIdentity(Map.empty) - /** Extensions as `CregitLanguages.pm` spells them: lowercase, no leading dot. - * `c++` and `h++` are real entries in that table, hence the `+`. The pattern - * is also a safety property: these strings are interpolated into a SQL LIKE - * pattern in [[Mapping]], and this alphabet contains no quote, no `%` and no - * `_`, so there is nothing to escape and nothing to inject. */ + /** As `CregitLanguages.pm` spells them (`c++` is real). Interpolated into a SQL + * LIKE in [[Mapping]]: no quote, `%` or `_` allowed, so nothing to escape. */ private[blobexec] val ExtensionPattern = "[a-z0-9+]+" - /** The identity value: hex, and long enough to be a digest rather than a - * guess. Bounded above so a malformed argument cannot become a meta row of - * arbitrary size. */ private[blobexec] val ValuePattern = "[0-9a-f]{8,128}" private val EntryRe = s"($ExtensionPattern)=($ValuePattern)".r - /** Parse `ext=value[,ext=value]...`. - * - * Left on anything malformed, naming the offending entry. Refusing a - * malformed identity matters more than it looks: a value this code does not - * understand would be recorded verbatim, compare unequal to the next run's, - * and turn every subsequent run into a refusal nobody can explain. */ + /** Parse `ext=value[,ext=value]...`; Left names the malformed entry. */ def parse(spec: String): Either[String, TokenizerIdentity] = { val trimmed = spec.trim if (trimmed.isEmpty) @@ -81,8 +52,7 @@ object TokenizerIdentity { } } - /** Parse a `--retokenize=` extension list. Same alphabet as above, so the two - * cannot disagree about what an extension looks like. */ + /** Parse a `--retokenize=` extension list. */ def parseExtensions(spec: String): Either[String, Set[String]] = { val trimmed = spec.trim if (trimmed.isEmpty) diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala index f1f60d78a..d2f773ee5 100644 --- a/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala @@ -3,32 +3,13 @@ package cregit.blobexec import java.nio.file.{Files, Path} import java.security.MessageDigest -/** The content-addressed memo `tokenizeByBlobId/tokenBySha.pl` keeps, described - * from this side so it can be invalidated. - * - * That script is the `` blobExec runs per blob. Before invoking a - * tokenizer it hashes the bytes on its stdin and looks for - * `$BFG_MEMO_DIR/xx/yy/`; on a hit it prints that file and no tokenizer - * runs at all. The key is `sha1_hex($contents)` and NOTHING else — - * tokenBySha.pl:76,83-84. No tokenizer version, no extension, no language. - * - * So dropping a `blob_map` row is not enough to force a re-tokenization. The - * walker re-runs the command for that blob, the command finds the memo entry, - * and the same stale tokens come back. Both layers have to go together, which - * is why `--retokenize` will not run without `--memo-dir`. - * - * Deleting a memo entry is always safe in the direction that matters: a miss - * costs one tokenizer invocation and produces correct tokens, while a kept - * entry produces wrong ones. That asymmetry is why the purge runs BEFORE the - * database transaction — if the purge fails, nothing has been invalidated and - * the run refuses; if the transaction fails afterwards, some cache entries were - * thrown away and nothing is wrong. */ +/** The tokenBySha.pl memo, keyed by `sha1_hex(contents)` alone (tokenBySha.pl:76,83-84). + * A dropped `blob_map` row would be refilled from it with stale tokens, so + * `--retokenize` purges both, the memo first: a lost entry only costs a rerun. */ object TokenizerMemo { - /** What the purge did. `missing` is not an error on its own: a blob excluded as - * oversized or denylisted never reached the tokenizer, so it never had an - * entry. `deleted == 0` with `examined > 0` IS, and [[Mapping]] refuses on it - * — see there. */ + /** `missing` is normal (excluded blobs never had an entry); [[Mapping]] refuses + * on `deleted == 0` with `examined > 0`. */ final case class PurgeReport(examined: Long, deleted: Long, missing: Long, unreadable: Long) { def render: String = s"examined=$examined deleted=$deleted missing=$missing unreadable=$unreadable" @@ -47,11 +28,7 @@ object TokenizerMemo { sb.toString } - /** The two-level fan-out tokenBySha.pl builds: `substr(0,2)/substr(2,2)/sha1` - * (tokenBySha.pl:83-84). Replicated rather than abstracted over, because the - * two have to agree byte for byte and a shared helper across a Perl/Scala - * boundary is not available. `TokenizerMemoSpec` pins it against a literal - * sha1 and, when perl is on PATH, against Digest::SHA itself. */ + /** Must match tokenBySha.pl:83-84 byte for byte; pinned by `TokenizerMemoSpec`. */ def relativePathFor(sha1Hex: String): String = { require(sha1Hex.length >= 4, s"not a sha1: [$sha1Hex]") s"${sha1Hex.substring(0, 2)}/${sha1Hex.substring(2, 4)}/$sha1Hex" @@ -66,14 +43,8 @@ object TokenizerMemo { catch { case _: java.io.IOException | _: java.nio.file.InvalidPathException => false } } - /** Delete the memo entries of `origBlobs`. - * - * `readBytes` turns an original blob sha into its bytes, or None when the - * object cannot be materialised (oversized). Injected so this is testable - * without a repository, and so the caller decides how the bytes are read. - * - * One blob at a time and nothing retained: the corpus figure for a `.rs` - * invalidation is 741,869 rows, and holding their contents is not an option. */ + /** Delete the memo entries of `origBlobs`. `readBytes` is None for a blob that + * cannot be materialised. One blob at a time: an invalidation can be ~10^6 rows. */ def purge(root: Path, origBlobs: Vector[String], readBytes: String => Option[Array[Byte]]): PurgeReport = { var deleted = 0L var missing = 0L diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index 502a6579e..f686f8953 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -20,18 +20,11 @@ final case class WalkStats( blobsCacheHit: Int, refsProjected: Int, aborted: Boolean, - /** Blobs whose command was killed for exceeding its budget, and so - * excluded from the rewrite as failed. */ blobsTimedOut: Long, - /** Distinct mask-matched blobs excluded because JGit will not materialise an - * object that large: see [[Walker.MaxBlobBytes]]. */ + /** Too large for JGit to materialise: see [[Walker.MaxBlobBytes]]. */ blobsOversized: Long, - /** Distinct mask-matched blobs excluded because they are on the shipped blob - * denylist ([[BlobDenylist]]). */ blobsDenylisted: Long, - /** Blobs whose tokenizer reported [[BlobExec.ParserCrashExitCode]] - * (srcML died on a signal, or the token stream came back empty), and so - * excluded from the rewrite as failed. */ + /** Tokenizer exited [[BlobExec.ParserCrashExitCode]]. */ blobsParserCrashed: Long = 0L, blobCommandExecutions: Long, originalBlobCopyRequests: Long, @@ -61,9 +54,7 @@ final class Walker( blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, workerPool: Option[TokenizerWorkerPool] = None, - // The shipped list by default, so a caller cannot forget it and hand a known - // non-terminating blob to srcml. A parameter only so a test can supply its - // own fixture; nothing at run time chooses a different list. + // A parameter only so tests can supply a fixture. denylist: BlobDenylist = BlobDenylist.shipped ) { import Walker._ @@ -87,17 +78,9 @@ final class Walker( private val originalBlobBytesAvoided = new LongAdder // -- stall watchdog ------------------------------------------------------ - // - // The consumer's `Await.result` calls are unbounded again, and deliberately. - // A duration budget cannot tell a wedged run from an honestly large one: the - // predecessor of this watchdog computed 3,806,400s for tencentkona-21's - // first-import commit and clamped to 30 days, so on the very run that - // motivated this work it would never have fired. Instead of predicting how - // long honest work takes, assert that work is *happening*: every blob, tree, - // commit and original-blob copy stamps `lastProgressNanos`, and if nothing at - // all completes within `stallTimeoutSeconds` the watchdog kills the process. - // Safe because GNU `timeout` hard-bounds every child at `blobTimeoutSeconds`, - // so a healthy run always completes *something* well inside the window. + // `Await.result` is unbounded: a duration cannot tell a wedged run from a large + // one, so the watchdog checks that some work completes. Safe because `timeout` + // hard-bounds every child at `blobTimeoutSeconds`. private val lastProgressNanos = new AtomicLong(System.nanoTime()) private val lastProgressWhat = new AtomicReference[String]("startup") @@ -147,10 +130,7 @@ final class Walker( killed } - /** Run `body` under the stall watchdog. The watchdog is a daemon so it can - * never keep the JVM alive, and it halts rather than exiting: a stalled run - * may well have a shutdown hook that would block on the same wedged thread, - * and an exit path that can hang is not a fix for a hang. */ + /** Halts rather than exits: a shutdown hook could block on the wedged thread. */ private def withStallWatchdog[A](body: => A): A = { val done = new AtomicBoolean(false) val tick = math.max(1L, math.min(30L, math.max(1, stallTimeoutSeconds).toLong / 4L)) @@ -161,10 +141,7 @@ final class Walker( catch { case _: InterruptedException => Thread.currentThread().interrupt() } if (!done.get()) stalled().foreach { stalledNanos => System.err.println(watchdogReport(stalledNanos)) - // Halting leaves nothing behind to reap the children, and an - // orphaned srcml tree is what survived its parent by 51 hours in - // the incident this work comes from. Kill the whole descendant - // tree first — `timeout`, the shell, the tokenizer, all of it. + // halt() reaps nothing: kill the whole descendant tree first. val killed = killDescendants() System.err.println(s"blobExec: killed $killed leftover child process(es) before exiting") System.err.flush() @@ -371,8 +348,7 @@ final class Walker( private def markUninteresting(revWalk: RevWalk): Unit = { mapping.allCommitOrigShas.foreach { sha => val id = ObjectId.fromString(sha) - // Scanning a large memo's frontier happens before any blob runs, and on a - // long history it is not fast; it must not look like a stall. + // A long frontier scan must not look like a stall. progress(s"frontier $sha") try { val rc = revWalk.parseCommit(id) @@ -779,10 +755,7 @@ final class Walker( * into dst). Never touches `mapping`, keeping the hot path lock-free. */ private def executeBlobTask(task: BlobMissTask): BlobResult = readBlob(task) match { - // None means "excluded from the rewrite": too large for jgit to - // materialise, or on the blob denylist. Both take the same downstream path - // — no id, no tree entry, no blob_map row — which is why one result covers - // them. readBlob has already counted and explained whichever it was. + // None: excluded, and already counted and logged (see readBlob). case None => BlobResult.Excluded(task.origId) case Some(bytes) => executeBlobTask(task, bytes) } @@ -923,8 +896,7 @@ final class Walker( // -- parallel blob resolution ------------------------------------------- - /** Run each miss through the external command on the pool. Returns the ids the - * tree should reference and an aborted flag. Excluded blobs have no id. */ + /** Excluded blobs get no id in the result. */ private def resolveMisses( misses: Vector[BlobMissTask], pool: java.util.concurrent.ExecutorService @@ -935,8 +907,6 @@ final class Walker( // single commit (e.g. the same (blob, path) reached via two subtrees). val unique = misses.map(m => (m.origId.name, m.fullPath) -> m).toMap.values.toVector - // An oversized blob yields None: it contributes no id, so no tree entry, no - // blob_map row and no dataset row. See readBlob and resolveEntry. val futures = unique.map { task => Future { readBlob(task).flatMap { bytes => @@ -960,11 +930,7 @@ final class Walker( ) val failed = Option(failure.get()) failed.foreach(noteFailed(task, _)) - // For Skip outcomes (identical output, or a non-zero exit with - // abortOnError=false) we keep the original blob id, so the dst tree will - // reference it — meaning the bytes must exist in dst. For Replace - // outcomes the worker has already inserted the new blob. For Abort we do - // nothing (caller short-circuits). A failed blob is excluded. + // Skip keeps the original id, so its bytes must exist in dst. outcome match { case BlobExec.Outcome.Skip if failed.isEmpty => ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) @@ -981,8 +947,7 @@ final class Walker( } } - // Unbounded: the stall watchdog is the backstop, not a budget. `flatten` - // drops the excluded blobs: oversized, denylisted or failed. + // Unbounded: the stall watchdog is the backstop. `flatten` drops excluded blobs. val results = Await.result(Future.sequence(futures), Duration.Inf).flatten val abort = results.exists { case (_, o) => o.isInstanceOf[BlobExec.Outcome.Abort] } @@ -999,38 +964,10 @@ final class Walker( } } - /** Read one mask-matched blob, or `None` when it is too large for JGit to - * materialise and must therefore be excluded from the rewrite. - * - * `qualcomm/qcom-embedded-power-measurement` ended with rc 1 here: - * - * org.eclipse.jgit.errors.LargeObjectException: 205b0f65... exceeds size limit - * at cregit.blobexec.Walker.readBlob(Walker.scala:1047) - * - * on `src/libraries/libexcel/excel.cpp`, 102,897,757 bytes and 2,398,232 - * lines. The mask *did* select it (`.cpp`), so unlike the pass-through path - * this is not a binary that slipped through. It is machine-generated: its own - * header says it is a `dumpcpp` dump of Microsoft Excel's COM type library. - * It carries no contributor-behaviour signal, and the tokenizer cannot be run - * on it in any case, so it is excluded — and the size alone is not the reason - * worth writing down, the provenance is. - * - * Two layers, because one is not enough: - * - the size check is a fast path, at the threshold JGit itself uses; - * - the `LargeObjectException` catch is the real mechanism, and covers any - * blob that sits between [[Walker.MaxBlobBytes]] and a JGit configured - * lower than its own default. A bare size check with a hand-picked - * constant above JGit's threshold would still throw in that band. - * - * Streaming the bytes instead (as the pass-through path does) is not an - * option here: the tokenizer would then be handed 98 MB of generated C++ on - * a box with about 5 GiB to spare. - */ + /** `None` when the blob is excluded: denylisted, failed earlier, or oversized. + * The size check is only a fast path; the `LargeObjectException` catch also + * covers a JGit configured below its default threshold. */ private def readBlob(task: BlobMissTask): Option[Array[Byte]] = { - // Before anything is read or opened: a denylisted blob is one srcML cannot be - // trusted with, so the cheapest possible check is the right one. This is a map - // lookup on a small map, and it is what turns a 600 s timeout into a - // microsecond and a logged exclusion. val denied = denylist.entryFor(task.origId.name) if (denied.isDefined) { noteDenylisted(task, denied.get); return None } if (failedKeys.contains((task.origId.name, task.fullPath))) return None @@ -1050,22 +987,12 @@ final class Walker( } finally r.close() } - /** `(origSha, fullPath)` of every blob excluded as oversized. Read by - * [[resolveEntry]], which drops those entries from the rewritten tree: the - * key's absence from `resolved` is what omits the file, and this set is what - * distinguishes a deliberate omission from a missing-key bug. */ + /** Lets [[resolveEntry]] tell a deliberate omission from a missing-key bug. */ private val oversizedKeys = ConcurrentHashMap.newKeySet[(String, String)]() - /** `(origSha, fullPath)` of every blob excluded by the denylist. Read by - * [[resolveEntry]] for the same reason as [[oversizedKeys]]: the key's absence - * from `resolved` is what omits the path, and this set is what separates a - * deliberate omission from a missing-key bug. Kept separate from - * `oversizedKeys` so the two exclusions can never be confused in the counts. */ private val denylistedKeys = ConcurrentHashMap.newKeySet[(String, String)]() - /** `(origSha, fullPath)` of every blob whose tokenizer timed out or crashed in - * this run. Excluded like a denylisted blob, and checked in [[readBlob]] so - * that one failing blob costs one tokenizer run, not one per commit. */ + /** Checked in [[readBlob]] so a failing blob costs one tokenizer run, not one per commit. */ private val failedKeys = ConcurrentHashMap.newKeySet[(String, String)]() private def noteFailed(task: BlobMissTask, reason: String): Unit = @@ -1075,17 +1002,9 @@ final class Walker( s"reason=$reason. The blob is left out of the rewritten tree, so it produces no " + "blame and no dataset row. The walk carries on.") - /** Count and explain one denylisted blob, once per `(sha, path)`. The sha, the - * path, the reason and the citation are all in the line, because this line is - * the only per-blob record of what the dataset does not contain, and "we could - * not parse it" is not a defensible sentence in a paper. */ private def noteDenylisted(task: BlobMissTask, entry: BlobDenylist.Entry): Unit = { val key = (task.origId.name, task.fullPath) - // Counted and logged once per (sha, path), exactly like an oversized blob, so - // the two counters mean the same thing. One blob reached through two paths is - // therefore two, which is the honest figure for "paths the dataset is missing" - // — but note the four shipped entries are ONE file's history, so a count of 4 - // is not four distinct files. + // Once per (sha, path), like noteOversized: one blob at two paths counts twice. if (denylistedKeys.add(key)) { blobsDenylisted.increment() System.err.println( @@ -1100,9 +1019,6 @@ final class Walker( } } - /** Count and explain one exclusion, once per `(sha, path)`. The path and the - * size are in the line on purpose: a bare sha cannot be cited in a paper, - * and this line is the only durable record of what the dataset is missing. */ private def noteOversized(task: BlobMissTask, sizeBytes: Long): Unit = { val key = (task.origId.name, task.fullPath) if (oversizedKeys.add(key)) { @@ -1120,23 +1036,8 @@ final class Walker( } /** Copy one original blob from src into dst by streaming it. Returns (id, size). - * - * Replaces readBlob on the pass-through path. `ObjectLoader.getBytes` throws - * LargeObjectException above JGit's stream threshold, and a blob that does not - * match the mask is copied verbatim, so its size is whatever the project - * committed rather than the size of a source file. redis/redis failed this way - * after 1,684s of work: - * - * org.eclipse.jgit.errors.LargeObjectException: 12e1ac54... exceeds size limit - * at cregit.blobexec.Walker.ensureOriginalBlobAvailable(Walker.scala:912) - * - * Streaming also removes the memory spike. Raising the threshold instead would - * still hold the whole blob, and three concurrent projects on a 30 GiB box - * cannot each afford a large fixture. - * - * The reader stays open for the whole copy: closing it before the stream is - * consumed would invalidate the stream. - */ + * Streams because a pass-through blob can exceed what `getBytes` will load. + * The reader must stay open until the stream is consumed. */ private def streamBlobInto(id: ObjectId, inserter: ObjectInserter): (ObjectId, Long) = { val r = src.newObjectReader() try { @@ -1163,8 +1064,7 @@ final class Walker( case TreeExisting(id) => id case TreeBuild(origId, entries) => val tf = new org.eclipse.jgit.lib.TreeFormatter - // flatMap, not map: an oversized blob resolves to no entry at all, so the - // rewritten tree simply does not contain that path. + // flatMap: an excluded blob resolves to no entry, so the path is dropped. val resolvedEntries = entries.flatMap(resolveEntry(_, resolved, inserter, subtreeAcc)) // jgit requires sorted entries (git tree order). The TreeWalk visited // them in tree order already, so we keep that order. @@ -1199,13 +1099,8 @@ final class Walker( val key = (origId.name, fullPath) resolved.get(key) match { case Some(newId) => Some(ResolvedEntry(name, mode, newId, copyBytes = false)) - // Excluded as oversized or by the denylist (see readBlob): drop the entry, - // so the file is absent from the rewritten tree rather than present as raw - // source. case None if oversizedKeys.contains(key) || denylistedKeys.contains(key) || failedKeys.contains(key) => None - // Anything else missing is a bug, and used to surface as a bare - // NoSuchElementException. Keep it fatal and say which blob it was. case None => throw new IllegalStateException( s"blob ${origId.name} ($fullPath) is a mask-matched miss with no resolution " + @@ -1492,29 +1387,11 @@ final class Walker( object Walker { - /** Size at which a mask-matched blob stops being tokenizable and is excluded. - * - * Deliberately not a number of our own choosing. It is read from JGit, which - * refuses to materialise any object at or above its stream-file threshold - * (`core.streamFileThreshold`, default 50 MiB) — `ObjectLoader.getBytes` - * throws `LargeObjectException` instead. Picking a larger constant, as an - * earlier draft of this fix did with 64 MiB, leaves a band (50-64 MiB) in - * which the size check passes and `getBytes` then throws anyway; picking a - * smaller one would silently drop files JGit could have handled. - * - * Reading a *fresh* `WindowCacheConfig` gives JGit's default rather than - * whatever is currently installed, which cannot be read back through public - * API. That is why this is only a fast path and [[Walker#readBlob]] also - * catches `LargeObjectException`: with the threshold lowered below the - * default, the catch is what handles the band. - * - * `>=`, not `>`, because that is JGit's own comparison: `Pack.load` returns a - * streaming (non-materialisable) loader once `size >= streamFileThreshold`. - */ + /** JGit's default `core.streamFileThreshold`, at or above which `getBytes` + * throws. Only a fast path: [[Walker#readBlob]] also catches the exception. */ private[blobexec] val MaxBlobBytes: Long = new org.eclipse.jgit.storage.file.WindowCacheConfig().getStreamFileThreshold.toLong - /** Pure form of the exclusion decision, so it can be checked without a repo. */ private[blobexec] def isOversized(sizeBytes: Long): Boolean = sizeBytes >= MaxBlobBytes private val OriginalBlobCacheSize = 1 << 16 @@ -1569,9 +1446,7 @@ object Walker { * memory stays modest. */ private val PipelineWindow = 32 - /** Window with no completed work of any kind after which the run is declared - * stalled. Generous because it is a watchdog, not a schedule: every child is - * separately hard-bounded at `--blob-timeout`. */ + /** Generous: every child is separately hard-bounded at `--blob-timeout`. */ private[blobexec] val DefaultStallTimeoutSeconds = 1800 /** Exit status when the watchdog kills a stalled run. Distinct from 4 (a blob @@ -1607,7 +1482,6 @@ object Walker { fullMessage: String ) - /** What [[Walker.resolveMisses]] hands back: the ids the tree references. */ final case class MissResolution(ids: IMap[(String, String), ObjectId], abort: Boolean) /** Per-blob worker result handed from a pool thread back to the consumer. */ @@ -1617,10 +1491,7 @@ object Walker { final case class Resolved(newId: ObjectId) extends BlobResult /** abort-on-error tripped by a non-zero command exit. */ final case class Aborted(stderr: String, exitCode: Int) extends BlobResult - /** Excluded from the rewrite: oversized, denylisted, or its tokenizer failed. - * It is deliberately not `Resolved`: it contributes no id, so - * every `collect` that builds a tree/blob_map map drops it, `resolveEntry` - * omits the path, and the run is still publishable. */ + /** Oversized, denylisted or failed: no id, so the path is left out of the tree. */ final case class Excluded(origId: ObjectId) extends BlobResult } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala index c4ab0174e..66091588c 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala @@ -5,18 +5,10 @@ import org.scalatest.matchers.should.Matchers import java.nio.file.Files -/** The denylist is a DATA file, because the list of blobs a published dataset - * omits is a property of the dataset and a paper has to cite it. That makes the - * file's contents part of the contract, so they are asserted here and not only - * parsed. */ +/** The shipped file's contents are part of the contract, so they are asserted, not only parsed. */ class BlobDenylistSpec extends AnyFunSuite with Matchers { - // CLASS 1 — the Java parser on type-annotated array types, upstream - // srcML/srcML#2361. Eight blobs, all in tencent__tencentkona-21's bare clone, - // all OpenJDK langtools regression tests. They are the historical versions of - // TWO files, one of which lives at two paths after a repository reorganisation — - // which is why the list is keyed on content, not on path. The second file - // carries a distinct sufficient construct within the same defect. + // Class 1: the Java parser hang, srcML/srcML#2361. private val JavaAnnotationShas = Set( // TestNewCastArray.java "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65", @@ -30,12 +22,7 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { "afbd81e5789ba060147b0de3be4f01759963299d" ) - // CLASS 2 — the C parser on C++ inside a `.h`, because CregitLanguages maps that - // extension to language C. A DIFFERENT defect in a DIFFERENT language, with NO - // upstream issue, so it cites the analysis document instead. Asserted separately - // because the one thing a reader must not do is read this list as one defect. - // Four historical versions of eden/fs/utils/StatTimes.h in facebook__sapling; - // the file's other four versions return by value and parse in milliseconds. + // Class 2: the C parser hang on C++ in a `.h` (StatTimes.h). No upstream issue. private val CParserShas = Set( "5460d43f1884e3be230e02215151b9b963844a48", "720914b27e170b73b38ef662fb04239273dd0a34", @@ -43,25 +30,11 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { "dbb885e0bf046755c4fcfbddfc2454f90c550c62" ) - // CLASS 3 — the C and C++ parsers CRASHING ON A SIGNAL (SIGSEGV or SIGABRT) - // under `--position`. A THIRD defect, and the only one that is not a hang: srcml - // is dead in milliseconds, so no timeout budget is ever involved. 197 blobs, the - // whole history of 36 files across 18 projects, found by running the tokenizer's - // real chain over all 904 historical versions of those paths (707 of the 904 - // parse cleanly — this is content-triggered, not a property of the path). - // - // Asserted by SIZE and by spot-checked members rather than by listing 197 - // literals: a 197-line literal set duplicated here would be maintained by - // copy-paste and would stop being a check. What must not drift is the count, the - // citation, and the fact that class 3 never claims to be #2361. + // Class 3: C/C++ parser crashes under `--position`. Checked by count and spot + // checks, not by duplicating 197 literals. private val ParserCrashCount = 197 - // Spot checks, one per distinguishing property, so the assertions below are - // anchored on real entries rather than on whatever happens to be first. - // - the sweep's own control file, 1,263 B, which rules out a size limit - // - a SIGABRT entry, where the signal is NOT a segfault - // - a blob vendored byte-identically into TWO projects, which is why the list - // is keyed on content and not on path + // crc32_small.c (position-only), a SIGABRT, and one blob vendored into two projects. private val CrashPositionOnlySha = "4a62830c807a5a2aff73f474891c38ed913d39c4" private val CrashAbortSha = "37cf5d5b1985a7510ad9c952897ef253231e80f7" private val CrashSharedBlobSha = "012ea8148e8d21cf515a9ff47d4af719d049b276" @@ -69,10 +42,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { private val KnownHangShas = JavaAnnotationShas ++ CParserShas test("the shipped list is on the classpath and holds the twelve hangs plus 197 crashes") { - // A jar built without the resource would hand all of these back to srcml. The - // twelve would hang for 600s each and then exit 4; the 197 would segfault or - // abort, and blobExec would exit 6 on `blobsParserCrashed`. Either way the - // project cannot publish, so "the resource is present" is itself a requirement. BlobDenylist.shipped.size shouldEqual 12 + ParserCrashCount JavaAnnotationShas.size shouldEqual 8 CParserShas.size shouldEqual 4 @@ -84,9 +53,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("class 3 holds exactly the 197 signal-death blobs, and no hang is counted among them") { - // The two groups are separated by citation, which is the only thing in the data - // file that distinguishes them, so this also pins that the citations did not - // get pasted across classes. val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) crashes.size shouldEqual ParserCrashCount crashes.map(_.sha).toSet.size shouldEqual ParserCrashCount // no duplicates @@ -94,9 +60,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("a class 3 entry says CRASH, names its signal, and disclaims #2361") { - // "we could not parse it" is indefensible in a paper; so is calling a segfault - // a hang. Every class 3 reason has to carry the defect, the signal and the - // srcML version, and has to say which invocation produced it. val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) crashes.foreach { e => withClue(s"${e.sha}: ") { @@ -105,20 +68,14 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { e.reason should include("--position") e.reason.toUpperCase should include("CRASH") e.reason should (include("SIGSEGV (shell status 139)") or include("SIGABRT (shell status 134)")) - // the flag is not optional, and the reason has to say why e.reason should include("token format carries line:col positions") - // a crash is not the hang, and must not be mistaken for it e.reason should include("not the non-termination") } } } test("each class 3 entry records whether --position is the trigger, measured per blob") { - // silent-empty-sweep.md claims --position is the trigger for all of them. It is - // not: measured per blob, 132 are position-only, 49 die either way, and 16 exit - // 1 without the flag. Inheriting the sweep's single claim would have put a - // false statement on 65 entries, so each one states its own measurement and - // this holds the three groups to their counts. + // silent-empty-sweep.md says --position is the trigger for all; it is not for 65. val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) val positionOnly = crashes.count(_.reason.contains("--position IS the trigger")) val eitherWay = crashes.count(_.reason.contains("--position is NOT the trigger")) @@ -131,34 +88,21 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("the spot-checked class 3 blobs are present, with the property each was chosen for") { - // crc32_small.c, 1,263 B — the sweep's control, and the file the committed - // reproducer tests/t/fixtures/srcml-position-crash.c is derived from. + // Source of the reproducer tests/t/fixtures/srcml-position-crash.c. val posOnly = BlobDenylist.shipped.entryFor(CrashPositionOnlySha).getOrElse( fail(s"$CrashPositionOnlySha (crc32_small.c) is missing from the shipped denylist")) posOnly.reason should include("--position IS the trigger") posOnly.reason should include("SIGSEGV (shell status 139)") - // An abort, not a segfault. The defect is broader than "segfault", and an - // abort evades a timeout-shaped denylist just as thoroughly. val abort = BlobDenylist.shipped.entryFor(CrashAbortSha).getOrElse( fail(s"$CrashAbortSha (util-linux swaplabel.c) is missing from the shipped denylist")) abort.reason should include("SIGABRT (shell status 134)") - // One blob, two projects: zlib's deflate.c is byte-identical in ossec__ossec-hids - // and azerothcore__azerothcore-wotlk. A path-keyed list would need two entries - // and would be wrong the moment a third project vendored the same bytes. BlobDenylist.shipped.entryFor(CrashSharedBlobSha) should not be empty } test("every entry carries a one-line reason and a citation for its own defect") { - // Without both, an exclusion is indistinguishable from data loss. The partner - // rejected "we could not parse it" as indefensible in a paper, and this is the - // assertion that keeps the answer in the artefact. - // - // The citation is per-CLASS, not global. Citing #2361 for the C parser hang - // would be a false attribution in a published artefact: it is a different - // language and a different construct, and no upstream issue exists for it. - // The same applies to class 3, which is not even a hang. + // Per class: citing #2361 for class 2 or 3 would be a false attribution. BlobDenylist.shipped.entries.foreach { e => withClue(s"${e.sha}: ") { if (JavaAnnotationShas.contains(e.sha)) e.citation shouldEqual "srcML/srcML#2361" @@ -172,9 +116,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("every citation belongs to exactly one of the three classes, so none is unattributed") { - // A pasted or empty citation would leave an omission in a published dataset with - // nothing to look up. Partitioning the whole list proves there is no fourth, - // accidental citation string hiding in the file. val byClass = BlobDenylist.shipped.entries.groupBy { e => if (e.citation == "srcML/srcML#2361") "class1" else if (e.citation.contains("srcml-nontermination-analysis.md")) "class2" @@ -188,8 +129,7 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("the C parser entries do not claim to be srcML#2361, and carry their reproducer") { - // The evidence for these is a minimised reproducer in the artefact rather than - // an upstream issue number, so the reproducer is part of the contract. + // With no upstream issue, the reproducer in the reason is the evidence. CParserShas.foreach { sha => val e = BlobDenylist.shipped.entryFor(sha).getOrElse( fail(s"the C parser blob $sha is missing from the shipped denylist")) @@ -225,7 +165,7 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { "", " ", s" $sha1\tsrcML/srcML#2361\tnon-termination ", - s"$sha2\tsrcML/srcML#2361\tnon-termination\t" // a trailing tab is tolerated + s"$sha2\tsrcML/srcML#2361\tnon-termination\t" ), "fixture") list.size shouldEqual 2 @@ -233,8 +173,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("a line that is not three fields is refused, naming the line") { - // Silently skipping a malformed line would re-admit a blob that hangs the run - // for 600s and then blocks publication — the exact failure this list removes. val bad = intercept[IllegalArgumentException] { BlobDenylist.parse(Vector(s"$sha1\tsrcML/srcML#2361"), "fixture") } @@ -248,9 +186,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("a key that is not a 40-hex git blob sha is refused") { - // The memo's key is sha1 of the file contents with no git header, so a - // plausible-looking wrong hash is an easy mistake to make. A 39-character or - // non-hex key would simply never match, silently. intercept[IllegalArgumentException] { BlobDenylist.parse(Vector(s"${"a" * 39}\tc\tr"), "fixture") } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala index ef77f040a..51c6606cb 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala @@ -101,12 +101,6 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } } - // CHANGED ASSERTION. This test used to be "zero exit, empty stdout against - // non-empty input → Replace with empty blob", and it pinned the defect: a - // crashed srcML exits 0 through the wrapper with empty stdout, and that empty - // stdout was inserted as the file's tokenization. 36 files across 19 projects - // were published as 0-byte blobs that way, with nothing counting them. Zero - // output from non-empty input is now a counted failure, never a blob. test("zero exit, empty stdout against non-empty input → Skip, counted, no blob") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) val cmd = shellScript("cat > /dev/null; true") @@ -117,9 +111,6 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { crashes.get shouldEqual 1 } - // The other half of the distinction the fix has to make: an input that is - // genuinely empty may legitimately produce empty output, and must stay a plain - // Skip that is NOT counted as a crash. test("zero exit, empty stdout against empty input → Skip, not counted a crash") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) val cmd = shellScript("cat > /dev/null; true") @@ -132,9 +123,6 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { test("the parser-crash exit status is Skip, counted, and never a Replace") { val crashes = new java.util.concurrent.atomic.AtomicInteger(0) - // Writes plausible-looking output first, so the test proves the status is what - // rejects it rather than the emptiness check: a crashed srcML can emit a - // truncated prefix before dying, and that prefix must never become a blob. val cmd = shellScript(s"echo 'partial tokens'; exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, abortOnError = false, inserter, @@ -144,9 +132,6 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } test("a parser crash skips one blob even with abortOnError, like a timeout") { - // The hostile case: one crashing blob must not take down a run that has - // already folded thousands of commits. It gates publication through the exit - // status instead (Main.exitStatus), which is where a timeout gates it too. val cmd = shellScript(s"exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, abortOnError = true, inserter) shouldBe BlobExec.Outcome.Skip @@ -169,8 +154,6 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { timeouts.get shouldEqual 1 crashes.get shouldEqual 1 // a timeout is not a crash - // A healthy blob and an ordinary failure must leave both counts alone, or a - // project would be held back from publication for nothing. run(shellScript("tr a-z A-Z")) run(shellScript("exit 7")) crashes.get shouldEqual 1 @@ -178,9 +161,8 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } test("a child that never exits is killed at the timeout, not awaited forever") { - // `sleep 30` stands in for the wedged srcml chain: it reads nothing and - // writes nothing, and it holds its stdout open so the reader thread parks - // in pipe_read. With a 1 second budget the call must return quickly. + // `sleep 30` is the wedged srcml chain: it reads nothing, writes nothing, and + // holds its stdout open so the reader parks in pipe_read. val cmd = shellScript("sleep 30") val started = System.currentTimeMillis() val (exit, _, _) = BlobExec.invoke( @@ -207,12 +189,8 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } test("an orphan grandchild holding the pipes cannot outlive the timeout") { - // The tencentkona-21 shape: tokenBySha.pl -> sh -> srcml, where killing the - // direct child leaves grandchildren holding the JVM's stdout/stderr pipes, - // the reader threads blocked, and `exitValue()`'s join unreturnable. The - // background `sleep` is that grandchild. Only a process-group kill returns - // near the budget; a destroy() of the direct child alone would park here - // until the JVM-side backstop latch, far past this assertion. + // The real process shape, tokenBySha.pl -> sh -> srcml: killing only the + // direct child leaves the background `sleep` holding this JVM's pipes. val cmd = shellScript("sleep 30 & sleep 30") val started = System.currentTimeMillis() val (exit, _, _) = BlobExec.invoke( diff --git a/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala index 5028d6ab4..8674eab5f 100644 --- a/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala @@ -13,9 +13,6 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Files, Path} -/** A blob whose tokenizer times out or crashes is excluded from the rewritten - * tree, like a denylisted blob: the walk carries on, the other blobs keep their - * tokens, and the run is not aborted. */ class FailedBlobSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { private val workRoot: Path = Files.createTempDirectory("failed-blob-") @@ -126,8 +123,6 @@ class FailedBlobSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { } } - // -- helpers --------------------------------------------------------------- - private def blobIdAt(repo: Repository, commitId: ObjectId, path: String): ObjectId = { val rw = new RevWalk(repo) try { diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index f40df421f..c23ad4615 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -45,12 +45,6 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } // -- the two timeouts are coupled ------------------------------------------- - // - // A pure-blob commit's only progress stamp is a blob finishing or being killed, - // so the watchdog window must exceed the per-blob budget. The defaults (600 and - // 1800) satisfy it; following the advice to raise --blob-timeout to 1800 without - // touching the window would not, and a legitimately slow blob would then race - // its own watchdog. test("a window at or above the floor is accepted unchanged") { Main.resolveStallTimeout(600, 1800, stallExplicit = false) shouldEqual Right(1800) @@ -94,8 +88,6 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("a defaulted window is widened to fit a raised --blob-timeout") { - // The exact collision the printed advice used to create: --blob-timeout=1800 - // against the 1800 default window. Main.resolveStallTimeout(1800, 1800, stallExplicit = false) shouldEqual Right(5400) Main.resolveStallTimeout(3600, 1800, stallExplicit = false) shouldEqual Right(10800) } @@ -160,11 +152,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.exitStatus(stats(aborted = true, blobsParserCrashed = 1)) shouldEqual 2 } - // The ineffective-invalidation status must be distinct: run_pipeline_process.sh maps blobExec's statuses onto different - // remedies and different marker files, so a collision sends the operator to the - // wrong one. This status specifically must not be 0 — "--retokenize did - // nothing" reported as success is the failure the flag exists to prevent — and - // must not be 3, which means the run never started. + // run_pipeline_process.sh maps each status to its own remedy and marker file. test("the ineffective-retokenize status collides with no other blobExec exit status") { val others = Map( "clean" -> 0, @@ -181,7 +169,6 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("an ineffective --retokenize is not reported as a clean walk") { - // Belt and braces on the one collision that would actually lose data. Main.RetokenizeIneffectiveExitStatus should not equal 0 } diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala index 1b00aafc6..46af59266 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala @@ -126,15 +126,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } finally Files.deleteIfExists(tmp) } - // The mask widening (per-language masks -> one universal mask) is exactly the - // change this refusal exists for, so pin it with the real strings rather than - // ".*" vs "\.c$". A project whose mask changes MUST be rebuilt: its tree_map - // rows were assembled from the files the old mask selected, so reusing them - // would produce a repository that silently omits every newly selected file — - // and the omission is invisible afterwards, because the trees look complete. - // - // If someone later "fixes" this into a warning to make a resume work, this test - // is what says no. The rows below are the ones that would be reused. + // Reused tree_map rows would silently omit every newly selected file, so a + // mask change must rebuild. Pinned with the real mask strings. test("a widened mask is refused on resume, not silently reused") { val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" @@ -154,8 +147,6 @@ class MappingSpec extends AnyFunSuite with Matchers { ex.getMessage should include(oldMask) ex.getMessage should include(newMask) - // Same mask still resumes, and the rows are still there — the refusal is - // about the mask changing, not about resuming at all. val m2 = Mapping.open(tmp, "/bin/cat", oldMask) try { m2.getTree("origtree") shouldBe Some("newtree") @@ -165,9 +156,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the warm-DB fallback refuses a widened mask too") { - // shard_build.sh passes a prior run's DB as --warm. Its blob ids were minted - // under that run's mask, so a mask change makes them foreign; openWarm has - // its own check and it must not drift from checkOrSetMeta's. + // openWarm (shard_build.sh --warm) has its own mask check; it must not drift + // from checkOrSetMeta's. val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" val warm = Files.createTempFile("mapping-warm-", ".db") @@ -187,15 +177,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } } - // -- --mask-widened --------------------------------------------------------- - // - // The opt-in beside the refusal above. Everything here exists because reusing - // blob_map across a widening is only safe under conditions that have to be - // CHECKED, not assumed, and one of those conditions is not obvious: blob_map - // holds identity rows for blobs the old mask did NOT select, and a wider mask - // selects some of them. Serving one of those as a cache hit puts raw source - // into the tokenized repository, which is the one outcome that silently - // corrupts the dataset's meaning. + // --mask-widened. Trap: blob_map holds identity rows for blobs the old mask + // skipped; a wider mask selects some, and a cache hit would ship raw source. private val OldCMask = """\.[ch]$""" private val OldCppMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" @@ -211,19 +194,16 @@ class MappingSpec extends AnyFunSuite with Matchers { ), log) } - /** A blob map as a real run leaves it: tokenized rows for the paths the mask - * selected, identity rows for every other blob the walker passed through, plus - * trees, a commit and a ref. */ + /** Tokenized rows for selected paths, identity rows for the rest, plus trees, a commit and a ref. */ private def seedUnderOldMask(tmp: java.nio.file.Path, mask: String): Unit = { val m = Mapping.open(tmp, "/bin/cat", mask) try { m.putBlob("b1", "src/a.c", "tok1") m.putBlob("b2", "src/b.h", "tok2") - // Identity rows. These are the dangerous ones: under the universal mask - // a.cpp and T.java ARE selected, so a reused row would serve raw source. + // Identity rows the universal mask selects: reusing them would serve raw source. m.putBlob("b3", "src/a.cpp", "b3") m.putBlob("b4", "src/T.java", "b4") - // And these are paths no mask will ever select. + // Never selected by any mask. m.putBlob("b5", "README.md", "b5") m.putBlob("b6", "CMakeLists.txt", "b6") m.putTree("t1", "nt1") @@ -240,7 +220,6 @@ class MappingSpec extends AnyFunSuite with Matchers { Mapping.open(tmp, "/bin/cat", UniversalMask) } ex.getMessage should include("mask") - // ...and it changed nothing, so the flag can still be used afterwards. val m = Mapping.open(tmp, "/bin/cat", OldCMask) try { m.getMeta("mask") shouldBe Some(OldCMask) @@ -258,21 +237,16 @@ class MappingSpec extends AnyFunSuite with Matchers { val (w, log) = widening() val m = Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) try { - // Kept: the real tokenizations. m.getBlob("b1", "src/a.c") shouldBe Some("tok1") m.getBlob("b2", "src/b.h") shouldBe Some("tok2") - // Gone: every identity row, so the newly selected paths are cache MISSES - // and get tokenized instead of passing through as raw source. m.getBlob("b3", "src/a.cpp") shouldBe None m.getBlob("b4", "src/T.java") shouldBe None m.getBlob("b5", "README.md") shouldBe None m.getBlob("b6", "CMakeLists.txt") shouldBe None - // Gone: trees, commits and the refs that name them. m.getTree("t1") shouldBe None m.getCommit("c1") shouldBe None m.getRef("refs/heads/main") shouldBe None m.allCommitOrigShas shouldBe empty - // The stored mask is now the new one, with the old one kept as provenance. m.getMeta("mask") shouldBe Some(UniversalMask) m.getMeta(Mapping.MaskWidenedFromKey) shouldBe Some(OldCMask) m.getMeta(Mapping.MaskWidenedAtKey) shouldBe Some("1700000000") @@ -294,13 +268,9 @@ class MappingSpec extends AnyFunSuite with Matchers { } finally Files.deleteIfExists(tmp) } - // THE check. A narrowing is refused on the evidence of the rows, and the - // databases are left exactly as they were — the operator can still resume - // under the old mask afterwards, which is the proof that nothing was touched. test("a NARROWING mask is refused, names the path, and leaves the databases untouched") { val tmp = Files.createTempFile("mapping-narrow-", ".db") try { - // Tokenized under the C++ mask, so src/a.cpp is a real tokenization. val seed = Mapping.open(tmp, "/bin/cat", OldCppMask) try { seed.putBlob("b1", "src/a.c", "tok1") @@ -313,7 +283,6 @@ class MappingSpec extends AnyFunSuite with Matchers { val (w, log) = widening() val ex = intercept[Mapping.MaskNarrowedException] { - // C only. It drops .cpp, so it is a narrowing wearing the flag's name. Mapping.open(tmp, "/bin/cat", OldCMask, None, Some(w)) } ex.getMessage should include("src/a.cpp") @@ -322,7 +291,6 @@ class MappingSpec extends AnyFunSuite with Matchers { ex.getMessage should include("NARROWING") log.toString shouldBe "" - // Untouched: every row and the recorded mask. val m = Mapping.open(tmp, "/bin/cat", OldCppMask) try { m.getMeta("mask") shouldBe Some(OldCppMask) @@ -337,10 +305,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } finally Files.deleteIfExists(tmp) } - // An identity row on a path no mask selects must NOT be read as a narrowing. - // Every blob map on this machine holds such rows (README.md, CMakeLists.txt, - // .pdf), so a check over all rows rather than tokenized ones refuses every - // project and the feature never fires once. + // Every blob map holds such rows (README.md, ...); counting them would refuse + // every project. test("identity rows on never-selected paths do not count as a narrowing") { val tmp = Files.createTempFile("mapping-identity-ok-", ".db") try { @@ -358,8 +324,7 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the narrowing check reads the basename, the way Walker matches the mask") { - // Walker matches against tw.getNameString, not the full path. Replicate it, - // or the check answers a question the walker never asks. + // Walker matches tw.getNameString, not the full path. Mapping.basename("src/kernel/a.c") shouldEqual "a.c" Mapping.basename("a.c") shouldEqual "a.c" Mapping.basename("dir.c/plain") shouldEqual "plain" @@ -387,9 +352,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the reachability check probes only retained rows, never identity ones") { - // An identity row's new_blob is the ORIGINAL blob id, which lives in src and - // need not be in dst at all until tree assembly copies it. Probing one would - // refuse a perfectly good resume. + // An identity row's new_blob is the source blob id, which need not be in dst + // yet; probing it would refuse a good resume. val tmp = Files.createTempFile("mapping-probe-set-", ".db") try { seedUnderOldMask(tmp, OldCMask) @@ -401,8 +365,7 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the command is still checked under the flag: a different tokenizer is refused") { - // The widening argument is about WHICH blobs, and says nothing about a - // different program producing different tokens from the same bytes. + // Widening is about which blobs, not about a different tokenizer. val tmp = Files.createTempFile("mapping-widen-cmd-", ".db") try { seedUnderOldMask(tmp, OldCMask) @@ -454,11 +417,10 @@ class MappingSpec extends AnyFunSuite with Matchers { try { (0 until 100).foreach(i => m.putBlob(f"orig$i%03d", f"f$i%03d.c", f"tok$i%03d")) m.sampleTokenizedNewBlobs(10) should have size 10 - // Spread, not the first ten: a head sample only sees the first commit. + // A spread, not the head: a head sample only sees the first commit. m.sampleTokenizedNewBlobs(10).map(_._2) shouldEqual m.sampleTokenizedNewBlobs(10).map(_._2) m.sampleTokenizedNewBlobs(10).map(_._2).toSet should contain("tok090") - // Asking for more than exists returns what exists. m.sampleTokenizedNewBlobs(1000) should have size 100 } finally m.close() } finally Files.deleteIfExists(tmp) diff --git a/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala index 34992c000..85b8c5af1 100644 --- a/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala @@ -5,14 +5,7 @@ import org.scalatest.matchers.should.Matchers import java.nio.file.{Files, Path} -/** Tokenizer-identity recording, the refusal it makes possible, and the - * `--retokenize` invalidation, at the level of the blob map itself. - * - * The defect these exist for: `Mapping.open` decided reuse on `command` and - * `mask`, and `command` is the constant path `tokenizeByBlobId/tokenBySha.pl`. - * Correcting the Rust tokenizer (729643e) therefore changed nothing either - * check looks at, every cached `.rs` row stayed a hit, and a run with the fixed - * binary reproduced the broken tokens. */ +/** Tokenizer-identity recording, refusal and `--retokenize`, at the blob-map level. */ class RetokenizeSpec extends AnyFunSuite with Matchers { private val cmd = "/bin/cat" @@ -38,9 +31,7 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } } - /** A blob map that looks like a finished run: two tokenized `.rs` rows, two - * tokenized `.c` rows, one identity (pass-through) row, and the tree, commit - * and ref rows above them. */ + /** A finished run: two `.rs` rows, two `.c` rows and one identity row. */ private def seed(db: Path, identity: TokenizerIdentity): Unit = { val m = Mapping.open(db, cmd, mask, tokenizerIdentity = identity) try { @@ -76,8 +67,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { private val emptyMemo: Vector[String] => TokenizerMemo.PurgeReport = blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, 0L, blobs.size.toLong, 0L) - // -- recording and refusing ------------------------------------------------ - test("an identity absent from meta is recorded, so the first run establishes it") { withDb { db => seed(db, id("rs" -> rustV1, "c" -> cV1)) @@ -104,13 +93,11 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { val ex = intercept[Mapping.TokenizerChangedException] { Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1)) } - // An operator has to be able to act on it without reading this source. ex.getMessage should include("rs") ex.getMessage should include(rustV1) ex.getMessage should include(rustV2) ex.getMessage should include("--retokenize=rs") - // The extension whose tokenizer did NOT change must not be named, or the - // hint tells the operator to throw away work that is still valid. + // Naming the unchanged extension would tell the operator to discard valid work. ex.getMessage should not include "--retokenize=c,rs" } } @@ -130,8 +117,7 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } test("no identity passed at all leaves behaviour exactly as it was") { - // The 186 already-published projects must not become refusals or - // invalidations because this mechanism now exists. + // Already-published projects must not start refusing. withDb { db => seed(db, TokenizerIdentity.empty) val m = Mapping.open(db, cmd, mask) @@ -143,12 +129,8 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } test("a changed identity for an extension with no cached rows is recorded, not refused") { - // The pipeline reports one identity per extension the checkout can parse, for - // every project, because the identity describes the checkout and not the - // project. A Java-only project carries a `c` identity it has no rows for. - // Refusing it when the C toolchain moves would be a refusal with no remedy: - // --retokenize=c would then fail too, correctly, for having nothing to - // invalidate. Nothing can be served from a cache with no entries. + // Every project gets an identity for every extension the checkout parses, so a + // refusal here would have no remedy: --retokenize would have nothing to drop. withDb { db => seed(db, id("rs" -> rustV1, "java" -> "9999999999999999")) val m = Mapping.open(db, cmd, mask, @@ -172,8 +154,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } } - // -- the invalidation ------------------------------------------------------ - test("--retokenize drops only the named extension's tokenized rows") { withDb { db => seed(db, id("rs" -> rustV1, "c" -> cV1)) @@ -182,11 +162,9 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), retokenize = Some(retokenize(Set("rs"), report = s => reported = s))) try { - // The assertion the whole flag exists for. m.getBlob("rsblob1", "src/one.rs") shouldBe None m.getBlob("rsblob2", "src/two.rs") shouldBe None - // And the assertion that makes it worth having rather than --drop-memo: - // the C tokenizations, which are 88% of the pipeline's time, are intact. + // The C rows survive: what makes this cheaper than --drop-memo. m.getBlob("cblob1", "src/one.c") shouldBe Some("newc1") m.getBlob("cblob2", "lib/two.h") shouldBe Some("newc2") m.storedTokenizerId("rs") shouldBe Some(rustV2) @@ -203,9 +181,7 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { tokenizerIdentity = id("rs" -> rustV2), retokenize = Some(retokenize(Set("rs")))) try { - // A retained tree_map row would short-circuit the re-walk of the subtree - // holding the .rs file, and the tree would keep pointing at the old - // token blob — a cache invalidation that invalidates nothing. + // A retained tree_map row would skip the re-walk and keep the old token blob. m.getTree("tree1") shouldBe None m.getCommit("commit1") shouldBe None m.getRef("refs/heads/master") shouldBe None @@ -236,9 +212,8 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { }))) try { purged.sorted shouldEqual Vector("rsblob1", "rsblob2") - // The memo is keyed on sha1(contents) with no tokenizer in the key - // (tokenBySha.pl:76), so a dropped blob_map row alone would be answered - // by the memo with the same stale tokens. + // The memo key has no tokenizer in it (tokenBySha.pl:76), so it would + // serve the same stale tokens. purged should not contain "cblob1" } finally m.close() } @@ -272,7 +247,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } ex.getMessage should include("--memo-dir") ex.getMessage should include("$BFG_MEMO_DIR") - // And it changed nothing. val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) try m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") finally m.close() @@ -314,8 +288,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } } - // -- the ways it must refuse rather than no-op ----------------------------- - test("--retokenize on an extension with no tokenized rows refuses, it does not exit quietly") { withDb { db => seed(db, id("rs" -> rustV1, "java" -> "cccccccccccccccc")) @@ -325,7 +297,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { retokenize = Some(retokenize(Set("java")))) } ex.getMessage should include("java") - // Nothing was dropped on the way to discovering it. val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) try { m.rowCounts.blobTokenized shouldEqual 4L @@ -343,9 +314,7 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { retokenize = Some(retokenize(Set("rs")))) } ex.getMessage should include("c") - // If this had invalidated .rs and then refused, the map would be in a state - // no flag describes: .rs redone under a new tokenizer, .c stale, and the - // recorded identity a mixture. Half a cache invalidation is a corruption. + // Invalidating .rs and then refusing would leave a half-invalidated map. val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1, "c" -> cV1)) try { m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") @@ -400,8 +369,6 @@ class RetokenizeSpec extends AnyFunSuite with Matchers { } } - // -- selection details ----------------------------------------------------- - test("extensions are matched case-insensitively, as the mask is") { withDb { db => val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala index 51b0e5d08..09a03dc77 100644 --- a/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala @@ -14,8 +14,6 @@ class TokenizerIdentitySpec extends AnyFunSuite with Matchers { val id = TokenizerIdentity.parse("h=aaaaaaaa,c=bbbbbbbb,rs=cccccccc").toOption.get id.extensions shouldBe Set("c", "h", "rs") id.render shouldEqual "c=bbbbbbbb,h=aaaaaaaa,rs=cccccccc" - // A rendering that depended on map order would make every log line and every - // refusal message a different string for the same fact. TokenizerIdentity.parse("c=bbbbbbbb,rs=cccccccc,h=aaaaaaaa").toOption.get.render shouldEqual id.render } @@ -25,8 +23,6 @@ class TokenizerIdentitySpec extends AnyFunSuite with Matchers { } test("an empty spec is refused, not read as 'no extensions'") { - // "no extensions" would make --tokenizer-identity a silent no-op, which is - // the whole failure mode this mechanism exists to remove. TokenizerIdentity.parse("").isLeft shouldBe true TokenizerIdentity.parse(" ").isLeft shouldBe true } @@ -47,8 +43,7 @@ class TokenizerIdentitySpec extends AnyFunSuite with Matchers { } test("a quote or a LIKE wildcard cannot get into an extension") { - // These strings reach a SQL LIKE pattern in Mapping. The alphabet is the - // defence, so it is asserted here rather than trusted there. + // Mapping interpolates extensions into a SQL LIKE. Seq("r's", "r%s", "r_s", "rs;", "rs)", "r s").foreach { bad => TokenizerIdentity.parse(s"$bad=deadbeefcafe").isLeft shouldBe true TokenizerIdentity.parseExtensions(bad).isLeft shouldBe true diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala index 739ec5d84..cbc45a54d 100644 --- a/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala @@ -6,10 +6,7 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Files, Path} -/** The memo layout has to match tokenizeByBlobId/tokenBySha.pl exactly, and the - * two are written in different languages, so the agreement is asserted rather - * than assumed. A silent disagreement here would look like a successful - * invalidation that purged nothing. */ +/** The memo layout must match tokenizeByBlobId/tokenBySha.pl byte for byte. */ class TokenizerMemoSpec extends AnyFunSuite with Matchers { private def withTempDir[A](body: Path => A): A = { @@ -38,15 +35,12 @@ class TokenizerMemoSpec extends AnyFunSuite with Matchers { val sha = "56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" TokenizerMemo.relativePathFor(sha) shouldEqual "56/f5/56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" - // Both halves come from the FRONT of the sha (substr 0,2 then 2,2), which is - // easy to get wrong as 0,2 then -2. + // Both halves from the front (substr 0,2 then 2,2), not 0,2 then -2. TokenizerMemo.relativePathFor(sha) should startWith("56/f5/") } test("sha1Hex agrees with perl's Digest::SHA on bytes perl might mangle") { - // The real inputs are arbitrary source bytes, including CRLF and high bytes. - // Perl reads the blob from stdin and hashes it; if either side treated the - // bytes as text the keys would diverge for exactly those files. + // If either side hashed text rather than bytes, CRLF and non-ASCII keys would diverge. val perl = Option(System.getenv("PATH")).toVector .flatMap(_.split(java.io.File.pathSeparator)) .map(p => Path.of(p, "perl")) @@ -104,8 +98,6 @@ class TokenizerMemoSpec extends AnyFunSuite with Matchers { Files.exists(TokenizerMemo.entryFor(root, TokenizerMemo.sha1Hex(contents("blob-rs-1").getBytes(UTF_8)))) shouldBe false - // The whole point of being selective: the C entry is still a valid - // tokenization and re-doing it would cost 88% of the pipeline for nothing. Files.exists(TokenizerMemo.entryFor(root, TokenizerMemo.sha1Hex(contents("blob-c-1").getBytes(UTF_8)))) shouldBe true } diff --git a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala index 26e94fe95..293d25409 100644 --- a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala @@ -87,8 +87,6 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // exercised by accident. The denylist tests pass their own. denylist: BlobDenylist = BlobDenylist.empty, maskWidened: Boolean = false, - // Empty by default so every test written before the tokenizer-identity - // mechanism existed keeps its exact previous behaviour. tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, retokenizeExtensions: Set[String] = Set.empty, memoDir: Option[Path] = None @@ -671,18 +669,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // -- large pass-through blob (regression) ---------------------------------- - // - // redis/redis failed after 1,684s of real work with: - // org.eclipse.jgit.errors.LargeObjectException: 12e1ac54... exceeds size limit - // at cregit.blobexec.Walker.ensureOriginalBlobAvailable - // - // A blob that does not match the mask is copied verbatim, so its size is - // whatever the project committed, not the size of a source file. - // ObjectLoader.getBytes refuses anything above JGit's stream threshold, so the - // copy path must stream instead of materialising. - // - // The threshold is lowered here so a small blob reproduces it in milliseconds - // rather than needing a 50 MB fixture. + // getBytes refuses blobs above JGit's stream threshold, so the verbatim copy + // must stream. The threshold is lowered so a small blob reproduces it. private def withStreamFileThreshold[T](bytes: Int)(body: => T): T = { val lowered = new WindowCacheConfig lowered.setStreamFileThreshold(bytes) @@ -742,33 +730,20 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- oversized MASK-MATCHED blob (regression) ------------------------------- - // - // qualcomm/qcom-embedded-power-measurement ended a whole project with rc 1: - // org.eclipse.jgit.errors.LargeObjectException: 205b0f65... exceeds size limit - // at cregit.blobexec.Walker.readBlob - // on src/libraries/libexcel/excel.cpp — 102,897,757 bytes, 2,398,232 lines, a - // dumpcpp dump of Microsoft Excel's COM type library. The mask DID select it - // (.cpp), so the streaming fix on the pass-through path does not apply: the - // tokenizer cannot be handed 98 MB, so the blob has to be excluded outright. + // Streaming cannot help here: the tokenizer cannot take it, so it is excluded. test("the oversized limit is jgit's own threshold, not a constant above it") { val jgitDefault = new WindowCacheConfig().getStreamFileThreshold.toLong - // Read from jgit, so the two cannot drift. An earlier draft of this fix used - // a hand-picked 64 MiB, which leaves 50-64 MiB passing the size check and - // then throwing inside getBytes anyway. + // Read from jgit, so the two cannot drift. Walker.MaxBlobBytes shouldEqual jgitDefault jgitDefault shouldEqual 50L * 1024 * 1024 Walker.isOversized(200L * 1024 * 1024) shouldBe true - Walker.isOversized(55L * 1024 * 1024) shouldBe true // the band the 64 MiB constant missed + Walker.isOversized(55L * 1024 * 1024) shouldBe true Walker.isOversized(4096L) shouldBe false } - /** Run the exclusion end to end on `mode`, with jgit's threshold lowered so a - * 78 KB fixture reproduces a 98 MB blob in milliseconds. - * - * The fixture size is the point: 78 KB is far BELOW Walker.MaxBlobBytes, so - * the size fast path cannot see it and only the LargeObjectException catch - * can. That is exactly the band a bare size check gets wrong. */ + /** The 78 KB fixture is below Walker.MaxBlobBytes, so only the LargeObjectException + * catch (under a lowered jgit threshold) can exclude it. */ private def checkOversizedExclusion(label: String, pipeline: Boolean, pipelineTrees: Boolean): Unit = { val dir = freshWorkDir("oversized-masked-" + label) val git = initSrc(dir.resolve("src")) @@ -780,7 +755,6 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte "add a normal source file and a generated one") } finally git.close() - // Proof that the fast path is not what fires here. Walker.isOversized(generated.length.toLong) shouldBe false val srcRepo = openBare(dir.resolve("src/.git")) @@ -830,18 +804,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- DENYLISTED mask-matched blob ------------------------------------------- - // - // tencent__tencentkona-21 holds four blobs on which srcML 1.1.0 does not - // terminate: 0 bytes of output, one core at 101%, at every budget from 5s to - // 600s. Upstream srcML/srcML#2361 is open with no patch and v1.1.0 is the latest - // release, so there is nothing to upgrade to. The timeout path contains the hang - // but answers it with exit 4, "incomplete, do not publish", which would leave - // that project permanently unpublishable over a diagnosed third-party defect. - // - // So a denylisted blob takes the EXCLUSION path instead: dropped in - // microseconds, never handed to the tokenizer, counted on its own, and NOT - // gating the exit status. A timeout still gates it, because a timeout is a hang - // nobody has explained yet. + // srcML 1.1.0 never terminates on these (srcML/srcML#2361). A denylisted blob is + // excluded and does not gate the exit status; an unexplained timeout still does. /** The git blob id of `content`, computed the way git computes it, so a fixture * can be denylisted without first asking the repository what its sha is. */ @@ -919,8 +883,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } test("a denylisted blob is excluded without running the tokenizer (pipeline)") { - // The mode run_pipeline_process.sh actually uses for S/M projects, which is - // what tencent__tencentkona-21 runs as. + // The mode run_pipeline_process.sh uses for small and medium projects. checkDenylistExclusion("pipeline", pipeline = true, pipelineTrees = false) } @@ -954,14 +917,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- --mask-widened, end to end --------------------------------------------- - // - // The one test that matters for the mask widening, because it is the only one - // that asks what ends up in dst. Reusing blob_map has to do two things at once: - // skip the work already done, AND tokenize the files the wider mask newly - // selects. The second is the trap. Those files already have an identity row - // from the first run, saying "not selected, bytes pass through", and a naive - // reuse serves that row as a cache hit — so the file lands in the TOKENIZED - // repository holding RAW SOURCE, and nothing downstream can tell. + // Trap: a newly selected file already has an identity row; serving it as a cache + // hit would put raw source in the tokenized repository. test("a widened mask reuses the tokenized rows and still tokenizes the newly selected ones") { val w = freshWorkDir("mask-widened") @@ -980,23 +937,18 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte "README.md" -> "leave-me-alone\n" ), "only commit") - // Run 1, narrow mask: two files tokenized, c.cpp merely passed through. val first = runWalker(git.getRepository, dstDir, db, cmd, narrow) first.blobCommandExecutions shouldEqual 2L locally { val dst = openBare(dstDir) try { fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") - // Raw, and correctly so: the narrow mask did not select it. fileAtHead(dst, "master", "c.cpp") shouldBe Some("int c;\n") } finally dst.close() } - // Run 2, wider mask, same dst and same db, with the opt-in. val second = runWalker(git.getRepository, dstDir, db, cmd, wide, maskWidened = true) - // Exactly ONE tokenizer invocation: a.c and b.h came from blob_map, c.cpp is - // new. That ratio is the entire point of reusing the map. second.blobCommandExecutions shouldEqual 1L second.blobsCacheHit shouldEqual 2 @@ -1004,9 +956,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte try { fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") fileAtHead(dst, "master", "b.h") shouldBe Some("INT B;\n") - // The assertion this whole task turns on. Under a reused blob_map without - // the identity-row purge this reads "int c;\n" — raw source, in the - // tokenized repository, invisibly. + // Without the identity-row purge this would be the raw "int c;\n". fileAtHead(dst, "master", "c.cpp") shouldBe Some("INT C;\n") fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") } finally dst.close() @@ -1032,19 +982,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- tokenizer identity ---------------------------------------------------- - // - // The defect, end to end. blob_map reuse was decided on `command` + `mask`, and - // `command` is the constant path tokenizeByBlobId/tokenBySha.pl. So correcting a - // tokenizer changed nothing either check looks at: every cached row for that - // language stayed a cache hit, and a run with the corrected tokenizer - // reproduced the OLD tokens exactly, with no error anywhere. Measured on the - // corpus for the Rust fix (729643e): 741,869 .rs (blob, path) pairs in 45 - // projects would have been served from the poisoned cache. - // - // These two tests are the deliverable. The first says a changed tokenizer can - // no longer be silent. The second says the opt-in actually re-tokenizes the - // affected files, and ONLY those — re-tokenizing everything costs 88% of total - // pipeline time, so a `.rs`-only fix must not touch the C work. + // `command` is a constant path, so a corrected tokenizer must be detected by its + // identity, and --retokenize must redo only the affected extension. test("a changed tokenizer identity refuses the incremental run instead of reusing its tokens") { val w = freshWorkDir("tokid-refuse") @@ -1059,9 +998,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte runWalker(git.getRepository, dstDir, db, v1, mask, tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "1111111111"))) - // Same content, same mask, same command path — only the Rust tokenizer's - // identity moved. Before this mechanism the run reused every .rs row and said - // nothing. + // Only the Rust tokenizer's identity moved. val ex = intercept[Mapping.TokenizerChangedException] { runWalker(git.getRepository, dstDir, db, v1, mask, tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "2222222222"))) @@ -1082,12 +1019,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte Files.createDirectories(memo) val mask = """\.(c|rs)$""" - // ONE command path whose behaviour changes between the two runs, because that - // is exactly the pipeline's situation: is the constant path - // tokenizeByBlobId/tokenBySha.pl, which dispatches by extension to a tokenizer - // that can be rebuilt underneath it. Two different script paths would trip the - // existing `command` meta check and prove nothing — the defect is precisely - // that the command string does NOT change when the tokenizer does. + // One command path for both runs, as in the pipeline (tokenBySha.pl): a second + // path would trip the `command` meta check and prove nothing. val cmd = w.resolve("tokenize.sh") def writeTokenizer(rustBody: String): String = { Files.writeString(cmd, @@ -1100,7 +1033,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte cmd.toFile.setExecutable(true) cmd.toAbsolutePath.toString } - // v1 emits the `line:col` prefix the real defect emitted; v2 does not. + // v1 emits the buggy `line:col` prefix; v2 does not. val v1 = writeTokenizer("""sed 's/^/1:1\t/'""") val git = initSrc(srcDir) @@ -1119,16 +1052,12 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte locally { val dst = openBare(dstDir) try { - // The defect's signature: a line:col prefix the rest of the pipeline does - // not expect, shifting every later field by one. fileAtHead(dst, "master", "b.rs") shouldBe Some("1:1\tfn b() {}\n") fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") } finally dst.close() } - // Plant the memo entries the real tokenizeByBlobId/tokenBySha.pl would have - // written, so the purge has something to purge and the test can assert the - // .c entry survives. + // The entries tokenBySha.pl would have written. val memoEntries = Map( "fn b() {}\n" -> "1:1\tfn b() {}\n", "fn c() {}\n" -> "1:1\tfn c() {}\n", @@ -1140,33 +1069,25 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte content -> p } - // The tokenizer is corrected in place. Same path, same mask, same everything - // the old checks looked at. val v2 = writeTokenizer("cat") v2 shouldEqual v1 val second = runWalker(git.getRepository, dstDir, db, v2, mask, tokenizerIdentity = idV2, retokenizeExtensions = Set("rs"), memoDir = Some(memo)) - // Two invocations, not three: the .rs files were re-tokenized and a.c was not. - // That ratio IS the requirement — a flag that re-tokenized a.c as well would - // cost the 88% this exists to avoid. + // Only the .rs files are re-tokenized; a.c stays a cache hit. second.blobCommandExecutions shouldEqual 2L second.blobsCacheHit shouldEqual 1 val dst = openBare(dstDir) try { - // The assertion the whole task turns on: the corrected tokenizer's output, - // not the cached wrong one. fileAtHead(dst, "master", "b.rs") shouldBe Some("fn b() {}\n") fileAtHead(dst, "master", "c.rs") shouldBe Some("fn c() {}\n") fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") } finally dst.close() - // The memo, the second cache layer. Without purging it the walker would have - // re-run the command and tokenBySha.pl would have answered from sha1(contents) - // — the same stale tokens, through a different door. + // An unpurged memo would serve the same stale tokens by sha1(contents). Files.exists(memoEntries("fn b() {}\n")) shouldBe false Files.exists(memoEntries("fn c() {}\n")) shouldBe false Files.exists(memoEntries("int a;\n")) shouldBe true From 4d8c4ff1fe20449a067b4ee2790cdfe1045b91f3 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:40:44 +0000 Subject: [PATCH 03/15] docs: tighten README, DATASET and worker protocol Cut repetition and incident history from the prose the merge added: the -C100 and --reblame notes, the --retokenize section, the firm columns and the worker protocol. Every fact a user needs stays. --- README.md | 79 +++++++++++------------------ generate_dataset/DATASET.md | 29 +++++------ tokenizeByBlobId/WORKER_PROTOCOL.md | 12 ++--- 3 files changed, 47 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 1bdbbfb32..be918806b 100644 --- a/README.md +++ b/README.md @@ -169,17 +169,13 @@ Flags (see `./run_pipeline_process.sh --help` for the full list): | `--reblame` | re-blame every file in step 7, replacing existing `.blame` output | off — a resume skips files already blamed | **`git blame` runs with `-C100` copy detection** (`blameRepo/formatBlame.pl`), so a -token moved between files keeps its original author. Upstream shipped that commented -out. Enabling it changed the author of up to 25% of a project's tokens, so blame output -produced before 2026-09-22 is not comparable with output produced after it. - -**Pass `--reblame` whenever the blame itself changed**, not the file list. -`blameRepoFiles.pl` skips any file whose `.blame` output already exists. That skip makes -a resume cheap, and it makes a re-blame a silent no-op: step 7 reports every file as -already done and exits 0, then step 10 rebuilds from the old blame. One pilot changed -**0 of 15,036,195 tokens** that way, with identical row counts and `rc=0`. Read step 7's -summary to confirm a real re-blame — `Already done [0]` is the proof. Do **not** pass the -flag to resume an interrupted run. +token moved between files keeps its original author. It changes up to 25% of a +project's authors, so blame from before 2026-09-22 is not comparable with later blame. + +**Pass `--reblame` whenever the blame itself changed**, not the file list. Without it +step 7 skips every file whose `.blame` exists, exits 0, and step 10 rebuilds from the +old blame. A real re-blame shows `Already done [0]` in step 7's summary. Do **not** +pass the flag to resume an interrupted run. A full run starts by **deleting the work directory** — to keep several target repositories side by side, give each its own `--work`. To resume a failed run @@ -211,35 +207,24 @@ p.s.: long pauses are trimmed. ### When a tokenizer is corrected: `--retokenize` -A corrected tokenizer does **not** by itself produce corrected tokens on a -resume. blobExec decides whether to reuse a cached tokenization from the -recorded `command` and `mask`. `command` is the constant path -`tokenizeByBlobId/tokenBySha.pl`, and `mask` says *which* files to tokenize, -never *how* — so rebuilding a tokenizer moves neither value, every cached row for -that language stays a cache hit, and the run reproduces the old tokenizer's -output and exits 0. Measured, for the Rust fix in `729643e`: 741,869 `.rs` -(blob, path) pairs across 45 projects. - -Two mechanisms close that: - -1. **Every run reports a tokenizer identity** — one opaque digest per file - extension, covering that extension's whole parser toolchain - (`tokenize/tokenizerIdentity.pl`). blobExec records it in the blob map's - `meta` table and compares it on every later run. A mismatch, on an extension - that actually has cached rows, **refuses the run** (exit 3) and names the flag - to fix it. Nothing is invalidated automatically. -2. **`--retokenize EXTS` is the opt-in past that refusal**, and it is selective: - re-tokenizing is 88% of total pipeline time, so a `.rs`-only defect costs - `.rs` entries only. +A rebuilt tokenizer does **not** by itself change cached tokens: blobExec's cache +key (`command`, `mask`) stays the same, so a resume reuses the old output and exits 0. + +1. **Every run records a tokenizer identity** (`tokenize/tokenizerIdentity.pl`), one + digest per extension over its parser toolchain, in the blob map's `meta` table. + A mismatch on an extension with cached rows **refuses the run** (exit 3). + Nothing is invalidated automatically. +2. **`--retokenize EXTS` is the opt-in past that refusal**, for those extensions + only (re-tokenizing is 88% of pipeline time). To re-tokenize only the `.rs` entries of one project, after rebuilding the tokenizer: ```sh -# 1. make sure the binary is actually current (this also proves the identity moved) +# 1. rebuild whatever is stale ./run_pipeline_process.sh --ensure-artifacts -# 2. resume that project at step 2 — never step 1, which deletes the work +# 2. resume at step 2 — never step 1, which deletes the work ./run_pipeline_process.sh \ --repo-url --repo-name \ --work ../cregit-files- \ @@ -247,23 +232,17 @@ tokenizer: 2 ``` -`--retokenize` needs `FROM_STEP=2` exactly: step 1 deletes the work directory -(so there would be nothing cached to invalidate) and step 3 or later skips the -invalidation entirely. It is not available with `--mode sharded`. - -What it does, in one transaction: drops the `blob_map` rows for those extensions, -purges **their entries in the memo** (which is keyed on `sha1` of the file's -content with no tokenizer in the key, so dropping only the `blob_map` row would -let the memo answer with the same stale tokens), and drops `tree_map`, -`commit_map` and `ref_map` because a tree names its blobs. Every other -extension's tokenizations survive. - -It cannot quietly do nothing. blobExec refuses, **before changing anything**, if -no cached row carries a named extension, if the memo held none of the affected -blobs (which means `--memo-dir` is not this project's), if another extension's -tokenizer also changed and was not named, or if a retained `new_blob` id is -missing from the cregit repository. An ineffective `--retokenize` exits 7, never -0. +`--retokenize` needs `FROM_STEP=2` exactly: step 1 deletes the cache and step 3 or +later skips the invalidation. It is not available with `--mode sharded`. + +In one transaction it drops those extensions' `blob_map` rows and **their memo +entries** (the memo is keyed on content alone), plus `tree_map`, `commit_map` and +`ref_map`. Other extensions' tokenizations survive. + +blobExec refuses **before changing anything** if no cached row carries a named +extension, the memo held none of the affected blobs (wrong `--memo-dir`), another +extension's tokenizer changed and was not named, or a retained `new_blob` id is +missing from the cregit repository. An ineffective `--retokenize` exits 7, never 0. ### Outputs diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 621c777a3..213100411 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -114,19 +114,17 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or | `person_name` | `TEXT` | persons | Canonical display name for this person. Derived via `coalesce(p.personname, e.personid)`. If neither is available, this is `NULL`. | | `person_email` | `TEXT` | emails | Email address that matched this commit's author name/email pair. | | `person_domain` | `TEXT` | emails | Domain part of the email address. | -| `firm_raw` | `TEXT` | `--firm-map` | The map's `company` string for `person_domain`, exactly as the map gives it. `''` when the domain is not in the map. | -| `firm` | `TEXT` | `--firm-canonical` | The canonical firm name. Equals `firm_raw` unless the reviewed canonical table renames it. Never overwrites `firm_raw`. | -| `firm_source` | `TEXT` | `--firm-map` | The map's `source` for that domain: `patch`, `gitdm`, `rich`, `builtin`, `correction` (all hand-curated), `cncf-gitdm`, `cncf-gitdm-single` (a single-person inference, 2,771 of 4,049 rows), `spinellis[-sec]`. `''` means the domain is not in the map, so this is the column to filter on for "attributed at all". | +| `firm_raw` | `TEXT` | `--firm-map` | The map's `company` for `person_domain`, verbatim. `''` when the domain is not in the map. | +| `firm` | `TEXT` | `--firm-canonical` | Canonical firm name: `firm_raw` unless the canonical table renames it. | +| `firm_source` | `TEXT` | `--firm-map` | The map's `source`: `patch`, `gitdm`, `rich`, `builtin`, `correction` (hand-curated), `cncf-gitdm`, `cncf-gitdm-single` (single-person inference, 2,771 of 4,049 rows), `spinellis[-sec]`. `''` when the domain is not in the map. | | `repo_tag` | `TEXT` | commitmap | Repository tag indicating the origin repository. Values: `'p'` (pre-history), `'b'` (BitKeeper), `'l'` (Linux), or `''` (unknown/single repo). | ### Firm attribution -The 29 provenance columns are per-project constants. Firm is resolved **per row** -from `person_domain` by a LEFT JOIN against `--firm-map` -(`cregit-token-pipeline/data/affiliation.merged.csv`), and `firm` comes from the -reviewed canonical-name table `--firm-canonical`. `firm_raw` is never overwritten. -Both files must have a unique key: a repeated `domain` or `firm_raw` would -multiply token rows, so the generator refuses such a file before Phase 1. +Firm is resolved **per row** from `person_domain` by a LEFT JOIN against +`--firm-map` (`cregit-token-pipeline/data/affiliation.merged.csv`); `firm` comes +from `--firm-canonical`. A repeated key in either file would multiply token rows, +so the generator refuses it before Phase 1. ```sql -- Firm-level token counts, excluding single-person inferences @@ -372,11 +370,10 @@ persons.db ─────────────┤ For each `.blame` file: 1. Parse each line as `commit_sha;token_content` -2. Repair the token line: remove the Rust tokenizer's `line:col` prefix, and, - in a strict UTF-8 file that is not `.rs` and has no byte-order mark, undo the - srcML 1.1.0 Latin-1 misreading (`ö` becomes `ö`). The source is read as - `utf-8-sig`, so a byte-order mark does not shift the walk. `main` prints the - counts on a `Repaired:` line. +2. Repair the token line: drop the Rust tokenizer's `line:col` prefix and, + in a strict UTF-8, non-`.rs` file without a byte-order mark, undo srcML + 1.1.0's Latin-1 misreading (`ö` becomes `ö`). The source is read as + `utf-8-sig`, so a BOM does not shift the walk. Counts go to a `Repaired:` line. 3. Walk through the original source file character-by-character to match tokens 4. Classify each token (structural vs content, type, value) 5. Insert into SQLite `token_map` @@ -416,8 +413,8 @@ uv run python generate_dataset/generate_dataset.py \ | `--repo-name` | no | Repository name. Default: inferred from output filename. | | `--project-meta` | no | JSON sidecar of per-project provenance, from `project_meta.py`. Omit and those columns are written empty. | | `--project-key` | no | Which key of the sidecar this project is. Default: `--repo-name`. A key the sidecar does not hold is an error, not a blank row. | -| `--firm-map` | no | CSV of `domain,company,kind,source`, joined per row against `person_domain`. Omit and `firm_raw`, `firm` and `firm_source` are written empty. A repeated `domain` is an error. | -| `--firm-canonical` | no | CSV of `firm_raw,firm,…`, the reviewed canonical-name table that fills `firm`. Needs `--firm-map`. Omit and `firm` repeats `firm_raw`, so split spellings stay split. | +| `--firm-map` | no | CSV of `domain,company,kind,source`. Omit and the three firm columns are written empty. A repeated `domain` is an error. | +| `--firm-canonical` | no | CSV of `firm_raw,firm,…` that fills `firm`. Needs `--firm-map`. Omit and `firm` repeats `firm_raw`. | | `--verbose` | no | Enable info-level logging to stderr. | ## Cross-reference: Perl ↔ Python diff --git a/tokenizeByBlobId/WORKER_PROTOCOL.md b/tokenizeByBlobId/WORKER_PROTOCOL.md index d416f29dd..a427c122d 100644 --- a/tokenizeByBlobId/WORKER_PROTOCOL.md +++ b/tokenizeByBlobId/WORKER_PROTOCOL.md @@ -5,16 +5,14 @@ sends each blob to an idle worker. A worker handles one request at a time. The worker reads `BFG_MEMO_DIR` and `BFG_TOKENIZE_CMD` from its environment, -as `tokenBySha.pl` does. Its tokens, its memo files and its exit statuses are -the same as those of `tokenBySha.pl`. +as `tokenBySha.pl` does, and gives the same tokens, memo files and exit statuses. For C, C++ and Java, the worker does the work of `tokenizeSrcMl.pl` in its own process (`tokenize/CregitSrcMl.pm`) if `BFG_TOKENIZE_CMD` is `tokenize.pl` with -only the options `--srcml2token=`, `--ctags=` and `--position`. It keeps one -`ctags --_interactive` for each language alive, and starts only `srcml2token` -for each file. For all other languages and commands, it starts -`BFG_TOKENIZE_CMD` for each blob. The worker does not start if `srcml2token` -does not answer `--libsrcml-path`. +only the options `--srcml2token=`, `--ctags=` and `--position`: one +`ctags --_interactive` per language stays alive and only `srcml2token` starts per +file. Otherwise it starts `BFG_TOKENIZE_CMD` per blob. It refuses to start if +`srcml2token` does not answer `--libsrcml-path`. ## Messages From d1a799bf6dbd622feb5887604b01c3a60cf12da9 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:40:44 +0000 Subject: [PATCH 04/15] docs(pipeline): cut comments the code already says Remove comments in the runner that restate the next lines or narrate incidents, and shorten the rest to one line of why. Reflow two help entries without dropping an option. No code changes. --- .github/workflows/ci.yml | 3 +- blameRepo/blameRepoFiles.pl | 3 +- run_pipeline_process.sh | 83 +++++++------------------------------ 3 files changed, 16 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94d48cc1a..d0f4b9ee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,8 +92,7 @@ jobs: - name: Dataset tests (pytest) run: devenv shell -- pytest -q --junitxml=junit-generate-dataset.xml - # `devenv shell -- pytest` does not flush its summary line, so read the counts - # from the JUnit XML. Standard library only, so it needs no devenv shell. + # `devenv shell -- pytest` does not flush its summary line; read the JUnit XML. - name: Re-check the pytest JUnit counts if: always() shell: bash diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index 41a5ad891..7036ef5ab 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -230,8 +230,7 @@ =head1 OPTIONS =item B<--overwrite> - By default, if an output file exists it is skipped. This changes that - behaviour and re-blames the file. + Re-blame files whose output already exists (by default they are skipped). =back diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 79c3c4cf8..275644ab1 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -13,9 +13,8 @@ artifacts (jars, tokenizers) are built automatically first — run inside Build: --build-only build all pipeline artifacts and exit, running nothing --ensure-artifacts - run the pre-run build guard and exit, building nothing that - is already current. That guard builds any MISSING artifact, - and additionally rebuilds the Rust tokenizer when its + run the pre-run build guard and exit. That guard builds any + MISSING artifact, and also rebuilds the Rust tokenizer when its sources are newer than the binary — it is the pipeline's only compiled tokenizer, and a stale one silently shifts every Rust token column by a field. The four jars and srcml2token @@ -67,9 +66,8 @@ Target repository: --blob-timeout N wall-clock budget in seconds for one blob's tokenizer (blobExec default: 600). A child that exceeds it is killed and that blob is excluded, as an EXCLUDED failed blob line - says. Also settable - as CREGIT_BLOB_TIMEOUT in the environment, which is how to - reach it through ctp.py. + says. Also settable as CREGIT_BLOB_TIMEOUT in the + environment, which is how to reach it through ctp.py. --stall-timeout N watchdog window in seconds (blobExec default: 1800). If no blob, tree, commit or blob copy completes anywhere in this window the run is killed with exit 5. Must be larger than @@ -222,9 +220,7 @@ DEFAULT_JOBS=4 JOBS=${CREGIT_JOBS:-$DEFAULT_JOBS} FROM_STEP=1 BUILD_ONLY=0 -# 1 means "run ensure_artifacts and exit". Distinct from --build-only, which -# rebuilds all six unconditionally: this one runs exactly the guard a normal run -# runs, so it is also how that guard is tested. +# Unlike --build-only, runs exactly the guard a normal run runs (and tests it). ENSURE_ARTIFACTS_ONLY=0 REPO_GIT_URL="" REPO_NAME="" @@ -699,8 +695,6 @@ build_all() { log "build complete" } -# Only the Rust tokenizer is checked for staleness: it is the one compiled -# tokenizer. The jars and srcml2token are only checked for existence. RUST_TOKENIZER_SOURCES=( "$CREGIT/tokenize/rustTokenizer/Makefile" "$CREGIT/tokenize/rustTokenizer/Cargo.toml" @@ -750,8 +744,7 @@ if [ "$BUILD_ONLY" = 1 ]; then exit 0 fi -# Auto-build any missing or out-of-date artifact BEFORE the work directory is -# touched, so a build failure never disturbs the outputs of a previous run. +# Before the work directory is touched, so a failed build keeps earlier outputs. ensure_artifacts if [ "$ENSURE_ARTIFACTS_ONLY" = 1 ]; then @@ -858,38 +851,9 @@ echo " Log: $LOG_FILE" echo "████████████████████████████████████████████████████████████████████████" echo "" -# --------------------------------------------------------------------------- -# --mask-widened / --retokenize: drop what a re-fold invalidates, keep what it -# reuses -# --------------------------------------------------------------------------- -# -# Both flags re-fold the whole history. A widening because every tree gains -# entries; an invalidation because a re-tokenized blob changes the tree above it, -# and blobExec empties tree_map and commit_map for exactly that reason. Either way -# every rewritten commit gets a new sha. Steps 3-10 are all derived from those shas, -# and two of them do NOT rebuild themselves, which makes this mandatory rather -# than tidy: -# -# step 6 `git clone` into a directory that already exists is `fatal: -# destination path already exists and is not an empty directory`. -# Verified. Any resume over a project that once completed dies here. -# step 7 blameRepoFiles.pl skips a file whose .blame output already exists -# (--overwrite is off by default). Those files name cregit commit shas -# from BEFORE the re-fold, so keeping them means step 10 joins blame -# against commits that no longer exist. That failure is quiet, which -# makes it worse than step 6's. -# -# Steps 3, 4 and 5 rebuild cleanly on their own (slickGitLog drops and recreates -# its schema), but their outputs are deleted anyway: "everything derived from the -# tokenized repository" is a rule someone can check, and "these three are -# self-cleaning and those two are not" is a rule that rots. -# -# KEPT, deliberately and by name: the original bare clone (step 2's input), the -# cregit bare repo (where blob_map's new_blob ids live), the blob map itself, and -# the memo. Those four are the entire point of the flag. -# -# Named paths only. No globs, and never $WORK itself — the wipe of $WORK is the -# expensive mistake this whole script is defended against. +# A re-fold gives every cregit commit a new sha: step 6 cannot clone over its old +# dir and step 7 would keep blame of the old shas. Keeps the bare repos, blob map +# and memo. Named paths only, never a glob or $WORK itself. drop_refold_derived_artifacts() { # $1 = which flag is asking, for the log log "$1: dropping the artifacts derived from the tokenized repo," log " because the re-fold gives every cregit commit a new sha." @@ -921,13 +885,7 @@ if [ "$MASK_WIDENED" = 1 ]; then drop_refold_derived_artifacts "--mask-widened" fi -# --retokenize does the same drop, but AFTER step 2 has actually invalidated -# something — see the end of step 2. blobExec refuses a --retokenize that would -# invalidate nothing, and doing the drop up front would mean a refused request had -# already deleted the blame output. Blame is this pipeline's bottleneck (measured: -# 8 files a minute, 5.6 days for the largest project), so destroying it to answer -# a request that was then rejected is the most expensive possible way to handle a -# typo in an extension name. +# --retokenize drops them at the end of step 2 instead. # --------------------------------------------------------------------------- # Step 1 — clone bare original repo @@ -950,15 +908,12 @@ if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then export BFG_MEMO_DIR="$MEMO_DIR" -# Route through the tokenize.pl dispatcher (not tokenizeSrcMl.pl directly) so it can -# fan out by language: srcML for .c/.h, rustTokenizer for .rs, etc. The -# --srcml2token / --ctags paths are forwarded only to the srcML parser. +# tokenize.pl dispatches by language; --srcml2token/--ctags reach only srcML. export BFG_TOKENIZE_CMD="${CREGIT}/tokenize/tokenize.pl \ --srcml2token=${SRCML2TOKEN} \ --ctags=$(which ctags)" -# Lets blobExec refuse a cache built by a different tokenizer; fatal when empty, -# because an empty identity turns the check off. +# Fatal when empty: an empty identity turns blobExec's cache check off. TOKENIZER_IDENTITY=$(perl "${CREGIT}/tokenize/tokenizerIdentity.pl" \ --srcml2token="${SRCML2TOKEN}" \ --ctags="$(which ctags)") \ @@ -973,8 +928,6 @@ TOKENIZE_RC=0 if [ "$MODE" = "sharded" ]; then # Memory-bounded path: N tree-only shards in parallel, then merge + serial # re-fold into $SHARD_OUT/final/{dst.git,blobmap.db} (byte-identical to serial). - # shard_build.sh propagates blobExec's 4 and 5 instead of collapsing them, so - # this branch gets the same marker and the same guard as the serial one. "${CREGIT}/blobExec/shard_build.sh" \ --src "$REPO_PATH_ORIGINAL_BARE" \ --out "$SHARD_OUT" \ @@ -986,11 +939,6 @@ if [ "$MODE" = "sharded" ]; then ${BLOB_TIMEOUT:+--blob-timeout "$BLOB_TIMEOUT"} \ ${STALL_TIMEOUT:+--stall-timeout "$STALL_TIMEOUT"} || TOKENIZE_RC=$? tokenize_gate "$TOKENIZE_RC" "sharded tokenize (shard_build.sh)" - # Said out loud rather than left to be discovered. The tokenizer identity is not - # plumbed through shard_build.sh / shard_merge.py, so the merged blobmap.db a - # sharded build produces carries no tokenizer_id.* rows. A later non-sharded - # resume over that database records the identity of whatever tokenizer is then - # current, without being able to check it against the one that built it. log "note: --mode sharded does not record a tokenizer identity in the merged blob map." log " A later resume cannot detect a tokenizer change against it. Non-sharded" log " modes record and check it; see --retokenize." @@ -1002,9 +950,7 @@ else [ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened" WORKER_FLAG="" [ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl" - # --retokenize carries --memo-dir with it, always, and blobExec refuses one - # without the other: the blob map and the memo are two caches of the same - # answer, and invalidating either alone invalidates nothing. + # blobExec refuses --retokenize without --memo-dir: the memo caches the same tokens. RETOKENIZE_FLAGS=() if [ -n "$RETOKENIZE" ]; then RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") @@ -1033,8 +979,7 @@ for _m in $KEEP_MARKERS; do fi done -# Only after blobExec accepted the invalidation: a refused one (exit 7) must not -# have cost the blame output. +# Only after blobExec accepted it: a refused one (exit 7) must not cost the blame. if [ -n "$RETOKENIZE" ]; then drop_refold_derived_artifacts "--retokenize $RETOKENIZE" fi From 6432c1695e3de4bded0b883fd2556a8a5f4eaff3 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:40:44 +0000 Subject: [PATCH 05/15] docs(dataset): shorten generate_dataset comments Docstrings and comments on the mojibake repair and the firm join are cut to at most three lines; the column reference lives in DATASET.md. --- generate_dataset/generate_dataset.py | 106 ++++++--------------------- 1 file changed, 22 insertions(+), 84 deletions(-) diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index e7d3af175..12aab66a3 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -291,18 +291,12 @@ def parse_blame_line(line: str) -> tuple[str, str] | None: return commit_sha, token_content -# The Rust tokenizer puts its position and a TAB before every token line: -# `5:7identifier|x`, `N:-` on a DECL line, `-:-` on a unit marker. The -# pipeline never asks for positions, and classify_and_skip splits on the first -# pipe, so the prefix went into token_type and the walk skipped its length as -# well. Nothing else writes a line that starts with digits, a colon and a TAB. +# The Rust tokenizer prefixes token lines with `L:C` (also `N:-`, `-:-`); +# left in, it lands in token_type and shifts the walk. _POSITION_PREFIX = re.compile(r"^(?:\d+|-):(?:\d+|-)\t") -# The prebuilt srcML 1.1.0 binary reads UTF-8 source as Latin-1: `ö` (C3 B6) -# comes out as the two characters `ö`. A UTF-8 lead byte read as Latin-1 lies -# in U+00C2..U+00F4, and a continuation byte in U+0080..U+00BF. Text the build -# wrote correctly almost never holds that pair, and undo_mojibake also needs -# the whole token to decode as UTF-8 before it changes anything. +# The prebuilt srcML 1.1.0 reads UTF-8 as Latin-1 (`ö` becomes `ö`): a lead +# byte lands in U+00C2..U+00F4, a continuation byte in U+0080..U+00BF. _MOJIBAKE = re.compile("[Â-ô][\u0080-¿]") _UTF8_BOM = b"\xef\xbb\xbf" @@ -317,19 +311,9 @@ def undo_mojibake(text: str) -> str: def read_source(source_path: Path) -> tuple[str, bool, bool]: - """Read a source file for the walk. Return its text, whether it started - with a byte-order mark, and whether the mojibake repair can apply to its - tokens. - - The text loses a leading byte-order mark. The tokenizers drop it, so a - BOM kept as a character puts the walk one character ahead to the end of - the file. - - The repair applies only when the file is strict UTF-8, has no BOM and is - not Rust. srcML decodes a BOM file correctly, and the Rust tokenizer does - not use srcML. In those files, and in a file that is not UTF-8, a token - that matches _MOJIBAKE is the true text. - """ + """Return (text, had a BOM, mojibake repair applies). The BOM is dropped as + the tokenizers drop it. Repair needs strict UTF-8, no BOM (srcML decodes + those right) and not Rust (no srcML).""" raw = source_path.read_bytes() has_bom = raw.startswith(_UTF8_BOM) try: @@ -337,8 +321,7 @@ def read_source(source_path: Path) -> tuple[str, bool, bool]: strict = True except UnicodeDecodeError: strict = False - # TextIOWrapper, not raw.decode: it keeps the newline translation of the - # text-mode open() that this replaces, so all other files read the same. + # TextIOWrapper, not raw.decode: it keeps text-mode newline translation. with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-8-sig", errors="replace") as f: text = f.read() @@ -348,13 +331,9 @@ def read_source(source_path: Path) -> tuple[str, bool, bool]: def repair_token_line(token_content: str, repairable: bool, stats: Counter) -> str: - """Return the token line with the Rust position prefix removed and, when - `repairable`, the srcML mojibake undone. Count each repair in `stats`. - - This must run before classify_and_skip: the walk consumes one source - character per token character, so a wrong token text also moves every - later position in the file. - """ + """Strip the Rust position prefix and, if `repairable`, undo srcML mojibake, + counting each in `stats`. Run before classify_and_skip: a wrong token + length shifts every later position in the file.""" prefix = _POSITION_PREFIX.match(token_content) if prefix: stats["position_prefix"] += 1 @@ -482,35 +461,13 @@ def sql_literal(value) -> str: return "'" + str(value).replace("'", "''") + "'" -# The three firm columns, in dataset order. They sit immediately after -# person_domain and before repo_tag, because firm is RESOLVED FROM person_domain: -# the key and the three values it produces belong together, and a reader who -# filters on person_domain finds the answer in the next three columns. -# -# Unlike the 29 metadata columns these are NOT per-project constants. They are -# per row, so they come from a real join against an external, auditable map -# (--firm-map, this repository's cregit-token-pipeline/data/affiliation.merged.csv) -# rather than from a SQL literal. The map stays a file on disk on purpose: baking -# 4,049 rows into a query would make the attribution unreviewable. -# -# firm_raw the map's `company` string, exactly as the map gives it -# firm the canonical name after --firm-canonical is applied -# firm_source the map's `source`: patch | gitdm | rich | builtin | correction | -# cncf-gitdm | cncf-gitdm-single | spinellis[-sec] | … -# -# An empty firm_source means "this person_domain is not in the map", so it is the -# column to filter on for "attributed at all". A reader who distrusts -# single-person inferences filters firm_source <> 'cncf-gitdm-single', which is -# 2,771 of the map's 4,049 rows. +# Joined per row from person_domain through --firm-map, kept a file so the +# attribution stays reviewable. Columns: generate_dataset/DATASET.md. FIRM_FIELDS = ("firm_raw", "firm", "firm_source") def read_csv_column(path, column): - """One column of a CSV, as a list. Used only for the duplicate checks below. - - csv, not duckdb: these checks run before Phase 1 and must not depend on the - query engine being reachable. - """ + """One CSV column, as a list. csv, not duckdb: it runs before Phase 1.""" import csv with open(path, newline="") as fh: @@ -518,14 +475,8 @@ def read_csv_column(path, column): def check_key_is_unique(path, column, what) -> int: - """Refuse a lookup table with a repeated key. Returns the row count. - - This is the one failure mode of the firm join that would be invisible. Both - tables are joined LEFT against token_map, so a domain appearing twice - DUPLICATES every token row of every person on that domain — the file still - validates, the schema still matches, and only the row count betrays it. - Failing here costs a second; finding it later costs the corpus. - """ + """Refuse a lookup table with a repeated key; return the row count. Through + the LEFT JOIN a repeated key silently duplicates token rows.""" keys = [k.strip().lower() for k in read_csv_column(path, column)] if len(keys) != len(set(keys)): seen, dupes = set(), [] @@ -541,17 +492,10 @@ def check_key_is_unique(path, column, what) -> int: def firm_sql(firm_map, firm_canonical) -> tuple[str, str]: - """(SELECT lines, JOIN lines) for the three firm columns. - - With no --firm-map the columns are three empty strings and no join is added, - so a caller that predates the flag still writes a schema-valid file — the - same bargain --project-meta makes. An empty firm_source then reads as "no - attribution", which is exactly true. - """ + """(SELECT lines, JOIN lines) for the firm columns. Without --firm-map they + are empty strings and no join is added, so the schema stays the same.""" if not firm_map: return ("".join(f" '' AS {f},\n" for f in FIRM_FIELDS), "") - # No canonical table means `firm` repeats `firm_raw`: the column still exists - # and still carries a name, it is just the unnormalised one. firm_expr = ("coalesce(fc.firm, fm.company, '')" if firm_canonical else "coalesce(fm.company, '')") select = ( @@ -559,10 +503,8 @@ def firm_sql(firm_map, firm_canonical) -> tuple[str, str]: f" {firm_expr} AS firm,\n" " coalesce(fm.source, '') AS firm_source,\n" ) - # lower() on both sides: the map is written lower-cased by build_domain_map, - # but persons.db's domain column is whatever the commit's e-mail carried. - # all_varchar=true so a company spelled like a number ('1&1', '360') cannot - # be sniffed into another type and change the Parquet's schema. + # persons.db keeps the e-mail's case, hence lower(). all_varchar stops a + # company like '1&1' or '360' being sniffed as a number. join = ( " LEFT JOIN (SELECT lower(domain) AS domain, company, source\n" f" FROM read_csv_auto({sql_literal(firm_map)},\n" @@ -656,8 +598,7 @@ def main(): metavar="PATH", help="CSV of domain,company,kind,source (cregit-token-pipeline/" "data/affiliation.merged.csv). Joined per row against person_domain to " - "fill firm_raw and firm_source. Empty means emit the three firm columns " - "as empty strings, so an older caller still produces a schema-valid file.", + "fill firm_raw and firm_source. Empty: the three firm columns are empty.", ) parser.add_argument( "--firm-canonical", @@ -665,7 +606,7 @@ def main(): metavar="PATH", help="CSV of firm_raw,firm,... — the REVIEWED canonical-name table that " "fills the `firm` column. Needs --firm-map. Without it `firm` repeats " - "`firm_raw`, so the split spellings stay split.", + "`firm_raw`.", ) parser.add_argument( "--memory-limit", @@ -725,9 +666,6 @@ def main(): project_meta = load_project_meta( args.project_meta, args.project_key or args.repo_name) - # Same rule for the firm map, and one extra reason: a repeated key in either - # lookup table multiplies token rows silently, so both are checked here - # rather than after the join has already written the Parquet. if args.firm_canonical and not args.firm_map: parser.error("--firm-canonical needs --firm-map: there is no firm_raw to " "canonicalise without a map to read it from") From 6938c3c416cf8a58d76820cab2c09057178d3033 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:40:45 +0000 Subject: [PATCH 06/15] docs(tokenize): trim the tokenizerIdentity header The 40-line header becomes three lines on what the identity covers. The script is not one of the files the identity hashes; its output is byte-identical before and after. --- tokenize/tokenizerIdentity.pl | 55 ++++------------------------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/tokenize/tokenizerIdentity.pl b/tokenize/tokenizerIdentity.pl index 62898711d..af630f376 100755 --- a/tokenize/tokenizerIdentity.pl +++ b/tokenize/tokenizerIdentity.pl @@ -13,44 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Prints blobExec's --tokenizer-identity argument: one opaque value per file -# extension, changing whenever the thing that turns that extension's bytes into -# tokens changes. -# -# ext=,ext=,... -# -# Why this exists. blobExec decided whether to reuse a cached tokenization on -# `command` and `mask`. `command` is the fixed path -# tokenizeByBlobId/tokenBySha.pl and `mask` says WHICH files to tokenize, never -# HOW. So when the Rust tokenizer's output format was corrected in 729643e, -# neither value moved: every cached .rs row stayed a cache hit and a run with -# the corrected binary reproduced the broken tokens, in 45 projects, with no -# error anywhere. An identity that tracks the TOOLCHAIN is the missing key. -# -# What goes into an extension's identity, and why each part: -# -# * its language's parser (CregitLanguages::LANG_PARSER_REL) — the tokenizer -# itself. For Rust that is the compiled cargo artifact, which is exactly the -# file that went stale. -# * srcml2token, the libsrcml it loads, ctags and CregitSrcMl.pm, for the -# srcML-routed languages only. srcml2token parses with libsrcml, and -# tokenizeSrcMl.pl (or CregitSrcMl.pm in the tokenizer worker) merges ctags -# output into its tokens. A change in any of them changes the tokens. -# * tokenize.pl and CregitLanguages.pm, for every extension. tokenize.pl is -# the dispatcher that decides which parser runs and with which flags -# (--position among them, which the token format depends on), and -# CregitLanguages.pm decides which language an extension is. Either one can -# change an extension's tokens without the parser changing at all. -# -# Including the shared files makes the identity CONSERVATIVE: editing -# tokenize.pl moves every extension's identity, so every project with cached -# rows refuses until an operator decides what to do. That is the right direction -# to be blunt in. A refusal costs a conversation; a false "unchanged" costs a -# republished corpus. -# -# The output is sorted and every value is a sha256 hex digest of a digest list, -# so the string is byte-stable for a given checkout: it is stored in the blob -# map's meta table and compared string-for-string on every resume. +# Prints blobExec's --tokenizer-identity, ext=,...: a digest of each +# extension's parser, tokenize.pl, CregitLanguages.pm and, for srcML languages, +# srcml2token, libsrcml, ctags and CregitSrcMl.pm. A change refuses old caches. use strict; use warnings; @@ -87,8 +52,6 @@ my $basedir = $FindBin::Bin; -# Every extension's identity includes these, because either one can change what -# an extension's tokens are without any parser changing. my @shared = ("$basedir/tokenize.pl", "$basedir/CregitLanguages.pm"); my %parsers = CregitLanguages::parsers($basedir); @@ -97,11 +60,8 @@ ? CregitLanguages::extensions_sorted() : CregitLanguages::masked_extensions_sorted(); -# A file that is named but absent is fatal. Emitting an identity that silently -# omitted a component would be worse than useless: it would compare EQUAL across -# a change in that component, which is the exact failure this program exists to -# make impossible. The Rust parser is a build artifact, so this is also where an -# unbuilt checkout is caught. +# A missing component is fatal: an identity without it would compare equal +# across a change in it. sub digest_of { my ($path) = @_; open(my $fh, '<', $path) @@ -146,8 +106,6 @@ sub cached_digest { my @components = (@shared, $parser); - # Required rather than optional: an identity for .c computed without - # srcml2token would not move when srcml2token does. if ($CregitLanguages::LANG_PARSER_REL{$language} eq $CregitLanguages::SRCML_PARSER_REL) { for my $pair (["--srcml2token", $srcml2tokenPath], ["--ctags", $ctagsPath]) { @@ -161,9 +119,6 @@ sub cached_digest { push @components, $libsrcml, "$basedir/CregitSrcMl.pm"; } - # Digest of the component digests, not of the concatenated bytes: the list is - # what identifies the toolchain, and a per-file digest makes a failure - # readable with --verbose-style debugging if it is ever needed. my $combined = sha256_hex(join("\n", map { cached_digest($_) } @components)); push @pairs, "$ext=$combined"; } From 1b589331ffa1df940ef9ae99d96e889dd83c75d6 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 03:40:45 +0000 Subject: [PATCH 07/15] test: trim comments and docstrings in tests Drop docstrings that repeat the test name and headers that narrate incidents; keep one-line notes on traps. Test counts are unchanged. --- tests/ensure_artifacts.sh | 85 ++++----------------------------- tests/reblame_passthrough.sh | 41 ++++------------ tests/retokenize_passthrough.sh | 60 +++++------------------ tests/t/languages.t | 5 +- tests/t/tokenizerIdentity.t | 51 +++----------------- tests/test_generate_dataset.py | 74 +++++++--------------------- 6 files changed, 56 insertions(+), 260 deletions(-) diff --git a/tests/ensure_artifacts.sh b/tests/ensure_artifacts.sh index 164d22762..ba7721927 100755 --- a/tests/ensure_artifacts.sh +++ b/tests/ensure_artifacts.sh @@ -1,39 +1,7 @@ #!/usr/bin/env bash -# Tests for ensure_artifacts(): the guard that decides whether a build artifact -# has to be rebuilt before a run. -# -# Why this exists. The guard used to be existence-only — -# -# [ -x "$RUST_TOKENIZER" ] || build_rust_tokenizer -# -# — so an artifact that EXISTED but was older than its sources was never -# rebuilt. Measured on this machine on 2026-09-20: -# tokenize/rustTokenizer/target/release/rust_tokenizer was dated 2026-09-04 -# 18:57 while tokenize/rustTokenizer/src/main.rs was dated 2026-09-20 21:30 — -# a 16-day-old binary running against source that had been fixed. The fix -# (729643e) removed a `line:col` prefix from every Rust token, and because -# the stale binary kept emitting it, every downstream consumer split the token -# on the wrong field and `token_type`, `token_value`, `source_text`, -# `source_line`, `source_col` and `is_structural` were all shifted by one. Not -# one step of the pipeline errored. -# -# So the guard has to compare mtimes for that binary. SCOPE, decided -# deliberately: the Rust tokenizer and nothing else. It is the pipeline's only -# COMPILED tokenizer — CregitLanguages.pm:109-115 routes C, C++ and Java to -# tokenizeSrcMl.pl and M4 to m4Tokenizer/m4.py, all interpreted and therefore -# re-read from disk every invocation. The four sbt jars and srcml2token keep -# their existence-only guards, and the last two cases in this file PIN that, so -# the gap is a recorded decision rather than something a later reader has to -# rediscover. -# -# The other half of the requirement is the fast path: an up-to-date checkout -# must still start a run without rebuilding anything, so every case below also -# asserts what was NOT built. -# -# No real building happens: `make` and `sbt` are stubbed on PATH and record -# their argv, so a case is milliseconds and nothing depends on cargo, g++, sbt -# or a JDK. The runner is driven through --ensure-artifacts, which runs exactly -# this guard and exits. +# Tests for ensure_artifacts(): only the Rust tokenizer (the one compiled +# tokenizer) is rebuilt when older than its sources; other artifacts are +# existence-only. make and sbt are stubbed on PATH and log their argv. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -49,15 +17,8 @@ check() { # $1 = description, $2 = condition result (0/1) fi } -# --------------------------------------------------------------------------- -# A fixture CREGIT tree: every source path the guard consults and every -# artifact it guards, and nothing else. The runner takes CREGIT from $(pwd), -# so the fixture is a directory holding a copy of the runner. -# -# Two mtime generations are used throughout: sources are stamped OLD and -# artifacts NEW, which is what an up-to-date checkout looks like. A case makes -# one source NEW to make one artifact stale. -# --------------------------------------------------------------------------- +# The runner takes CREGIT from $(pwd), so a fixture holds a copy of it. +# Sources are stamped OLD and artifacts NEW: an up-to-date checkout. OLD_STAMP=202601010000 NEW_STAMP=202602010000 @@ -107,10 +68,7 @@ fixture() { echo "source" > "$f/$p" touch -t "$OLD_STAMP" "$f/$p" done - # Directories are sources too — the guard passes whole source trees to find, - # and a directory's own mtime is what changes when a file is added or - # removed. Stamp them OLD as well, or a freshly created fixture looks like a - # checkout whose every source tree was just edited. + # The guard also compares directory mtimes; unstamped, they look just edited. find "$f" -type d -exec touch -t "$OLD_STAMP" {} + for p in "${ARTIFACTS[@]}"; do mkdir -p "$f/$(dirname "$p")" @@ -121,9 +79,7 @@ fixture() { printf '%s' "$f" } -# `make` and `sbt` that record their argv and do nothing else. The working -# directory goes in the record too: the three legacy jars share one sbt command -# line and are told apart only by the directory it runs in. +# cwd is logged too: the three legacy jars share one sbt command line. make_stubs() { # $1 = bin dir local b=$1 tool for tool in make sbt; do @@ -140,8 +96,6 @@ BIN=$(mktemp -d "${TMPDIR:-/tmp}/ensurebin-XXXXXX") make_stubs "$BIN" trap 'rm -rf "$BIN"' EXIT -# Run the guard in a fixture. Prints the runner's own output; the builds it -# invoked land in $ARGV. run_guard() { # $1 = fixture dir local f=$1 ARGV="$f/argv.log" @@ -166,8 +120,6 @@ build_count() { # $1 = fixture # --------------------------------------------------------------------------- echo "case 1: everything current — the fast path builds nothing" -# The whole reason the guard cannot simply always rebuild: a corpus run starts -# 188 times, and six builds each would dominate the schedule. W=$(fixture) OUT=$(run_guard "$W"); RC=$? [ "$RC" -eq 0 ]; check "exits 0" $? @@ -178,8 +130,7 @@ rm -rf "$W" echo "case 2: a newer Rust source — the exact defect this guard exists for" W=$(fixture) touch -t "$NEW_STAMP" "$W/tokenize/rustTokenizer/src/main.rs" -# The artifact and the source now share a timestamp, which is NOT stale: make -# the source strictly newer, as a real edit would. +# An equal timestamp is not stale; make the source strictly newer. touch "$W/tokenize/rustTokenizer/src/main.rs" OUT=$(run_guard "$W"); RC=$? [ "$RC" -eq 0 ]; check "exits 0" $? @@ -198,10 +149,6 @@ built "$W" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? N=$(build_count "$W"); [ "$N" -eq 1 ]; check "and nothing else was (built: $N)" $? rm -rf "$W" -# --------------------------------------------------------------------------- -# Each declared Rust source, one case each. Cargo.lock matters as much as -# main.rs: a dependency bump changes the tokenizer's behaviour without touching a -# line of this project's code. # --------------------------------------------------------------------------- rust_stale_case() { # $1 = source path, relative to the fixture echo "case: a newer $1 rebuilds the Rust tokenizer" @@ -222,15 +169,7 @@ rust_stale_case tokenize/rustTokenizer/Cargo.toml rust_stale_case tokenize/rustTokenizer/Makefile # --------------------------------------------------------------------------- -# The accepted gap, pinned so it is a decision and not a surprise. -# -# The four sbt jars and srcml2token are still guarded on existence alone. A -# stale one WOULD be used, silently. Rust was fixed alone because it is the only -# compiled tokenizer (CregitLanguages.pm:109-115 routes the other three -# languages to interpreted scripts, which cannot go stale this way) and because -# it is the artifact that actually failed. These cases assert the current -# behaviour so that whoever changes their minds has to change a test and read -# this comment. +# Pins the accepted gap: a stale sbt jar or srcml2token is used, not rebuilt. # --------------------------------------------------------------------------- gap_case() { # $1 = label, $2 = source to touch, $3 = grep pattern for its build echo "case: a newer $1 source does NOT rebuild it — known, accepted gap" @@ -251,8 +190,6 @@ gap_case slickGitLog slickGitLog/src/main/scala/gitLogToDb.scala 'one-j gap_case persons persons/src/main/scala/unifyPersons.scala 'one-jar' gap_case remapCommits remapCommits/src/main/scala/remapCommits.scala 'one-jar' -# But a MISSING one is still built, for all five. That behaviour predates this -# change and must survive it. missing_case() { # $1 = label, $2 = artifact to delete, $3 = grep pattern echo "case: a missing $1 is still built" local w @@ -272,9 +209,7 @@ missing_case slickGitLog slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNA # --------------------------------------------------------------------------- echo "case: a Rust source path the guard names but the checkout lacks is fatal" -# A typo in the source list would silently switch the guard off for that -# artifact, which is exactly the class of failure this whole file exists to -# stop. It must be loud instead. +# Otherwise a typo in the source list would silently disable the guard. W=$(fixture) rm -f "$W/tokenize/rustTokenizer/Cargo.lock" OUT=$(run_guard "$W"); RC=$? diff --git a/tests/reblame_passthrough.sh b/tests/reblame_passthrough.sh index 451799eab..549f608d6 100755 --- a/tests/reblame_passthrough.sh +++ b/tests/reblame_passthrough.sh @@ -1,18 +1,7 @@ #!/usr/bin/env bash -# Tests for the runner's half of the re-blame mechanism: the --reblame -# passthrough to step 7, and the guard that stops the flag being a quiet no-op. -# -# Why this exists. blameRepoFiles.pl skips any file whose .blame output already -# exists, and step 7 never passed --overwrite. So a deliberate re-blame — change a -# git blame flag, resume at step 7 — reported every file as already done, exited 0, -# and step 10 rebuilt the Parquet from the OLD blame. Measured on a three-project -# pilot before this flag existed: 0 of 15,036,195 tokens changed author, with -# identical schema and identical row counts, which is exactly what a successful run -# looks like. The unit test for --overwrite already existed and passed; nothing -# tested that the runner passes it. That is the gap these cases close. -# -# `blameRepoFiles.pl` and `java` are stubbed on PATH / in the fixture and record -# their argv, so no blame runs and no case takes more than a second or two. +# Tests that --reblame reaches step 7 as --overwrite (blameRepoFiles.pl skips +# existing .blame files) and is refused where step 7 would not run. +# blameRepoFiles.pl and java are stubbed and record their argv. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -36,15 +25,8 @@ STUB chmod +x "$1/java" } -# A throwaway CREGIT tree. The runner takes CREGIT from $(pwd), and its artifact -# staleness guard runs BEFORE any step — including a step-7 resume — so the tree -# needs tokenize/ and each module's build files plus placeholder artifacts, or the -# guard starts a real build and the run dies before step 7. The artifacts are -# placeholders: the guard digests their bytes, so any content works. -# -# On top of that, a stub blameRepoFiles.pl that appends its argv to $BLAME_ARGV. -# The real script is not used, because the assertion is about which flags arrive, -# not about what blame produces. +# The runner takes CREGIT from $(pwd) and runs its artifact guard before any +# step, so the tree needs build files and placeholder artifacts or it builds. repo_fixture() { local f p f=$(mktemp -d "${TMPDIR:-/tmp}/reblamerepo-XXXXXX") @@ -81,8 +63,7 @@ STUB printf '%s' "$f" } -# A work directory that looks finished through step 6: the non-bare cregit clone -# step 7 requires, and a blame directory holding output a re-blame must replace. +# A work dir finished through step 6, with blame output a re-blame must replace. fixture() { local w w=$(mktemp -d "${TMPDIR:-/tmp}/reblamework-XXXXXX") @@ -94,9 +75,7 @@ fixture() { run_from() { # $1 = work dir, $2 = FROM_STEP, rest = extra runner args local w=$1 from=$2; shift 2 - # BLAME_ARGV must be exported, not just assigned. `VAR=x somefunc` sets the - # variable for the function without exporting it, so the runner's child shell - # and the stub perl beneath it never saw it. + # Set on the command: `VAR=x somefunc` does not export VAR to the stub. ( cd "$REPO" \ && PATH="$BIN:$PATH" BLAME_ARGV="${BLAME_ARGV:-}" timeout 120 \ bash ./run_pipeline_process.sh \ @@ -130,17 +109,13 @@ OUT=$(BLAME_ARGV="$ARGV" run_from "$W" 7 --reblame); RC=$? [ -f "$ARGV" ]; check "step 7 invoked blameRepoFiles.pl" $? grep -q -- '--overwrite' "$ARGV"; check "--overwrite reached blameRepoFiles.pl" $? grep -q -- '--formatBlame=' "$ARGV"; check "and the formatter is still named" $? -# Match the log sentence, not the substring "reblame". The flag name appears in the -# usage text and in every refusal message too, so a bare substring match passed even -# when step 7 never ran. +# Not the bare word "reblame": usage and refusal messages contain it too. grep -q 'replacing every .blame file' <<<"$OUT" check "and the run says it is replacing every .blame file" $? rm -rf "$W" # --------------------------------------------------------------------------- echo "case 3: --reblame after step 7 is refused, because it would be skipped" -# The whole point of the flag is to close a silent no-op. Accepting it where it -# cannot act would reopen the same hole one step later. for FROM in 8 9 10; do W=$(fixture) OUT=$(BLAME_ARGV="$W/blame-argv.log" run_from "$W" "$FROM" --reblame); RC=$? diff --git a/tests/retokenize_passthrough.sh b/tests/retokenize_passthrough.sh index 6e839127f..e7107aa6e 100755 --- a/tests/retokenize_passthrough.sh +++ b/tests/retokenize_passthrough.sh @@ -1,32 +1,7 @@ #!/usr/bin/env bash -# Tests for the runner's half of the cache-invalidation mechanism: the tokenizer -# identity it computes and passes to blobExec on EVERY run, the --retokenize -# passthrough, the guards that stop the flag being a quiet no-op, and the -# derived-artifact drop that must happen only after the invalidation succeeded. -# -# Why each of these is here: -# -# * the identity has to arrive. If the runner does not pass -# --tokenizer-identity, blobExec records nothing, compares nothing, and the -# hole is exactly as open as it was — a corrected tokenizer reusing every -# cached row, silently, which is what put a line:col prefix into 741,869 .rs -# entries across 45 projects. -# * --retokenize has to arrive WITH --memo-dir. They are two caches of the same -# answer: the blob map, and the memo keyed on sha1 of the file's content with -# no tokenizer in the key. Invalidating either alone invalidates nothing. -# * the flag must not be reachable in a mode or a step where it would do -# nothing. FROM_STEP=1 deletes the caches first; FROM_STEP=3 skips step 2, so -# the invalidation never runs and the run exits 0 over poisoned tokens; -# sharded mode builds a fresh blob map per shard. -# * exit 7 must fail the run. blobExec returns it when --retokenize would have -# invalidated nothing. Treating that as success is the whole failure mode. -# * the derived artifacts (blame, html, the databases) must be dropped only -# AFTER step 2 succeeded. Blame is the pipeline's bottleneck — measured at 8 -# files a minute — so deleting it to serve a request blobExec then rejects is -# the most expensive way to handle a typo. -# -# `java` is stubbed on PATH and records its argv, so no tokenizing happens and no -# case takes more than a second or two. +# Tests the runner's tokenizer identity, the --retokenize passthrough (always +# with --memo-dir: the memo is keyed on content, not tokenizer) and its guards. +# java is stubbed on PATH and records its argv. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -42,8 +17,7 @@ check() { # $1 = description, $2 = condition result (0/1) fi } -# A `java` that exits with $STUB_RC, logs its argv, and (on 0) creates the -# destination repo its caller checks for. +# On success, creates the destination repo the caller checks for. make_stub_java() { # $1 = bin dir cat > "$1/java" <<'STUB' #!/usr/bin/env bash @@ -65,12 +39,8 @@ STUB } # --------------------------------------------------------------------------- -# A throwaway CREGIT tree. The runner takes CREGIT from $(pwd), and it needs the -# tokenize/ scripts (the identity is computed by tokenize/tokenizerIdentity.pl) -# plus each module's declared sources and artifacts, or the staleness guard would -# start real builds. The artifacts are placeholders: the identity digests their -# BYTES, so any content works, and a case that "changes a tokenizer" just writes a -# different string into one. +# The runner takes CREGIT from $(pwd) and needs tokenize/ for the identity, plus +# build files and placeholder artifacts, or the staleness guard would build. # --------------------------------------------------------------------------- repo_fixture() { local f p @@ -85,8 +55,6 @@ repo_fixture() { rm -rf "$f/$p/project/target" "$f/$p/project/project" mkdir -p "$f/$p/src" done - # Placeholder artifacts, stamped newer than everything above so the staleness - # guard is satisfied and no build runs. for p in tokenize/srcMLtoken/srcml2token \ tokenize/rustTokenizer/target/release/rust_tokenizer \ blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ @@ -104,8 +72,7 @@ repo_fixture() { printf '%s' "$f" } -# A work directory that looks like a finished run resumable at step 2: the bare -# original, a blob map, a memo, and the derived artifacts a re-fold invalidates. +# A finished run, resumable at step 2, with the artifacts a re-fold invalidates. fixture() { local w w=$(mktemp -d "${TMPDIR:-/tmp}/retokwork-XXXXXX") @@ -139,12 +106,10 @@ trap 'rm -rf "$BIN" "$REPO"' EXIT echo "case 1: every run passes a tokenizer identity, whether or not it invalidates" W=$(fixture); ARGV="$W/argv.log" OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" 2); RC=$? -# Step 2 is what these cases are about. The run goes on to die at step 5, because -# the stubbed java writes no databases; reaching step 3 is the signal that step 2 -# finished and the tokenize gate was satisfied. +# The run later dies at step 5 (the stub writes no databases); reaching step 3 +# means step 2 finished and the tokenize gate passed. grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? grep -q -- '--tokenizer-identity=' "$ARGV"; check "blobExec was given --tokenizer-identity" $? -# It must be a real per-extension map, not an empty string that records nothing. grep -qE -- '--tokenizer-identity=[a-z0-9+]+=[0-9a-f]{64}' "$ARGV" check "and the value is ext=sha256 pairs" $? grep -q -- '--retokenize=' "$ARGV"; [ $? -ne 0 ] @@ -166,9 +131,8 @@ rm -rf "$W" # --------------------------------------------------------------------------- echo "case 3: a successful invalidation drops the artifacts derived from the re-fold" -# Mandatory, not tidy: a re-tokenized blob changes the tree above it, so every -# cregit commit gets a new sha. Step 6's clone then fails on an existing directory -# and step 7's blame silently keeps .blame files naming commits that are gone. +# A re-tokenized blob gives every cregit commit a new sha, so old blame names +# commits that are gone and step 6's clone hits an existing directory. W=$(fixture); ARGV="$W/argv.log" OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? [ ! -f "$W/blame/one.c.blame" ]; check "the stale blame output is gone" $? @@ -199,8 +163,6 @@ rm -rf "$W" # --------------------------------------------------------------------------- echo "case 6: --retokenize at step 3 is refused, because it would be skipped entirely" -# The invalidation lives in step 2. From step 3 the flag does nothing, the poisoned -# tokens go through steps 3-10, and the run exits 0. W=$(fixture) OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs 3); RC=$? [ "$RC" -ne 0 ]; check "refused (exit $RC)" $? diff --git a/tests/t/languages.t b/tests/t/languages.t index 1096a4fe9..da03ac112 100644 --- a/tests/t/languages.t +++ b/tests/t/languages.t @@ -1,8 +1,7 @@ #!/usr/bin/env perl -# The extension table has been stable since 2017 and needs no test. These -# assertions cover the case-insensitive mask, which selects .C and .H, and a -# pipeline default derived from the table rather than written out a second time. +# Covers the case-insensitive mask (.C, .H) and the pipeline default derived +# from the table; the table itself is not tested. use strict; use warnings; diff --git a/tests/t/tokenizerIdentity.t b/tests/t/tokenizerIdentity.t index 18fb46ddb..49e4d145d 100644 --- a/tests/t/tokenizerIdentity.t +++ b/tests/t/tokenizerIdentity.t @@ -1,25 +1,7 @@ #!/usr/bin/env perl -# Tests for tokenize/tokenizerIdentity.pl, the value blobExec compares to decide -# whether a cached tokenization was made by the tokenizer this run is using. -# -# What has to be true of it, and why: -# -# 1. It changes when a tokenizer changes. That is the whole point: the defect -# it exists for is that `command` (the constant path -# tokenizeByBlobId/tokenBySha.pl) and `mask` both stayed the same while the -# Rust tokenizer's output format was corrected, so every cached .rs row -# stayed a cache hit. -# 2. It does NOT change when nothing changes. An identity that moved on every -# run would refuse every resume, which is indistinguishable from having no -# cache at all. -# 3. It changes the identity of the RIGHT extensions and no others. Selectivity -# is the difference between re-tokenizing 741,869 .rs entries and -# re-tokenizing the whole corpus at 88% of total pipeline time. -# 4. A component it cannot read is fatal, never omitted. An identity computed -# without a component compares EQUAL across a change in that component, -# which is exactly the silence being removed. -# 5. blobExec can parse what it prints. +# Tests for tokenize/tokenizerIdentity.pl: stable when nothing changes, moves +# only the affected extensions, and is fatal (never silent) on a missing part. use strict; use warnings; @@ -34,8 +16,6 @@ use CregitLanguages; my $realTokenizeDir = "$FindBin::Bin/../../tokenize"; -# A throwaway copy of tokenize/ with stand-in "binaries", so a test can change a -# tokenizer without touching the checkout or building anything. sub fixture { my $dir = tempdir(CLEANUP => 1); my $tok = "$dir/tokenize"; @@ -45,8 +25,7 @@ sub fixture { } chmod 0755, "$tok/tokenizerIdentity.pl", "$tok/tokenize.pl", "$tok/tokenizeSrcMl.pl"; - # Stand-ins for the parsers and the external binaries. Content is what the - # identity digests, so "changing a tokenizer" is writing a different string. + # The identity digests content, so "changing a tokenizer" is a new string. make_path("$tok/rustTokenizer/target/release"); write_file("$tok/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v1\n"); make_path("$tok/m4Tokenizer"); @@ -96,28 +75,21 @@ is($status, 0, "tokenizerIdentity.pl succeeds on a complete checkout"); my %baseMap = as_map($base); -# 3 — coverage. Every masked extension and nothing else, because those are the -# extensions the pipeline's mask can select and therefore the ones blob_map can -# hold rows for. is_deeply( [sort keys %baseMap], [CregitLanguages::masked_extensions_sorted()], "every masked extension gets an identity, and only those"); -# 5 — the format blobExec's TokenizerIdentity.parse accepts. +# The format blobExec's TokenizerIdentity.parse accepts. for my $ext (sort keys %baseMap) { like($ext, qr/^[a-z0-9+]+$/, "extension [$ext] is lowercase and dotless"); like($baseMap{$ext}, qr/^[0-9a-f]{64}$/, "identity for [$ext] is a sha256 hex digest"); } -# 2 — stability. Run twice, unchanged, and get the same string. my ($again) = identity($dir); is($again, $base, "an unchanged checkout yields a byte-identical identity"); # --------------------------------------------------------------------------- -# 1 and 3 — the Rust tokenizer changes, and ONLY .rs moves. This is the exact -# shape of the live defect: a corrected rustTokenizer binary, everything else -# untouched. write_file("$dir/tokenize/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v2 fixed\n"); my ($afterRust) = identity($dir); my %rustMap = as_map($afterRust); @@ -129,9 +101,6 @@ for my $ext (grep { $_ ne 'rs' } sort keys %baseMap) { } # --------------------------------------------------------------------------- -# 3, the other direction — the srcML chain moves the srcML-routed extensions and -# not Rust. Each of the three components counts: the token stream is the product -# of all of them. for my $component (qw(srcml2token libsrcml ctags)) { my $d = fixture(); my ($before) = identity($d); @@ -144,7 +113,7 @@ for my $component (qw(srcml2token libsrcml ctags)) { is($a{rs}, $b{rs}, "changing $component leaves .rs alone"); } -# tokenizeSrcMl.pl itself, and CregitSrcMl.pm, its copy in the tokenizer worker. +# CregitSrcMl.pm is tokenizeSrcMl.pl's copy in the tokenizer worker. for my $script (qw(tokenizeSrcMl.pl CregitSrcMl.pm)) { my $d = fixture(); my ($before) = identity($d); @@ -157,11 +126,8 @@ for my $script (qw(tokenizeSrcMl.pl CregitSrcMl.pm)) { } # --------------------------------------------------------------------------- -# The shared files move EVERY extension, deliberately. tokenize.pl decides which -# parser runs and with which flags (--position, which the token format depends -# on) and CregitLanguages.pm decides which language an extension is, so either can -# change an extension's tokens with no parser change at all. Blunt in the safe -# direction: a refusal costs a conversation, a false "unchanged" costs a corpus. +# Deliberately every extension: either file can change tokens with no parser +# change (tokenize.pl picks parser and flags, CregitLanguages.pm the language). for my $shared (qw(tokenize.pl CregitLanguages.pm)) { my $d = fixture(); my ($before) = identity($d); @@ -177,7 +143,6 @@ for my $shared (qw(tokenize.pl CregitLanguages.pm)) { } # --------------------------------------------------------------------------- -# 4 — a missing component is fatal, and says which file and why. { my $d = fixture(); unlink("$d/tokenize/rustTokenizer/target/release/rust_tokenizer"); @@ -205,8 +170,6 @@ for my $shared (qw(tokenize.pl CregitLanguages.pm)) { } # --------------------------------------------------------------------------- -# --all-extensions covers the routed-but-unmasked ones (.am/.ac), so the flag can -# be used the day the m4 lexer is fixed and M4 joins the mask. { my ($all) = identity($dir, "--all-extensions"); my %allMap = as_map($all); diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index 0337ed754..fd8111fa8 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -484,10 +484,6 @@ def footers_of(duckdb, path): @pytest.mark.parametrize("value", ["bob@example.com", "Bob bob@example.com"]) def test_a_bare_address_trailer_resolves_to_a_personid(monkeypatch, tmp_path, value): - """`Reviewed-by: bob@example.com`, no angle brackets. The trailer text - reached footer_reviewed_by while footer_personids stayed empty, because the - coalesce never reached the bare-address pattern and that pattern had no - capture group anyway.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( ("Reviewed-by", value), @@ -500,8 +496,7 @@ def test_a_bare_address_trailer_resolves_to_a_personid(monkeypatch, tmp_path, va def test_the_angle_bracket_form_still_wins_over_a_second_address(monkeypatch, tmp_path): - """Precedence, which the nullif must not disturb: a trailer holding both - forms resolves to the bracketed one.""" + """The nullif must not change precedence: the bracketed form wins.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( ("Reviewed-by", "noreply@example.com writing for Bob B "), @@ -513,10 +508,7 @@ def test_the_angle_bracket_form_still_wins_over_a_second_address(monkeypatch, tm def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): - """`Former-commit-id: ` carries no address, so it must contribute no - person. It used to join on '', which matches an emails row for an author - committing as `Name <>` — 27 of 199 corpus persons DBs hold one, so this - published a wrong personid on every commit carrying such a trailer.""" + """Joining on '' would match the emails row of an author committing as `Name <>`.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( ("Former-commit-id", "6a32a91a877bc2341810ef674dfdc3be21c500bc"), @@ -528,11 +520,8 @@ def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): # --------------------------------------------------------------------------- # -# firm attribution. The shape is different from everything above: the 29 -# metadata columns are per-project CONSTANTS injected as SQL literals, while -# firm is PER ROW and comes from a real join against an external CSV. So the -# failure modes are different too — a duplicate key in either lookup table -# multiplies token rows through the LEFT JOIN, and nothing downstream notices. +# firm attribution: a per-row LEFT JOIN, so a duplicate key in either lookup +# table silently multiplies token rows. # --------------------------------------------------------------------------- # FIRM_MAP_HEADER = "domain,company,kind,source\n" @@ -552,12 +541,7 @@ def write_canonical(tmp_path, *lines, name="canon.csv"): def generate_with_domain(monkeypatch, tmp_path, domain, argv_extra=()): - """The tiny project, plus one identified person on `domain`. - - build_tiny_project leaves emails and persons empty, so person_domain is NULL - there and every firm column is blank whatever the map says. A positive test - needs a person the commit actually joins to. - """ + """The tiny project plus one person on `domain`, so person_domain is set.""" import generate_dataset as gd blame, src, cregit_db, persons_db = build_tiny_project(tmp_path) @@ -596,16 +580,13 @@ def test_the_firm_field_list_is_three_names_in_dataset_order(): def test_no_firm_map_emits_no_join_and_three_empty_literals(): - """An older caller must still produce a schema-valid file, the same bargain - --project-meta makes. So the columns exist and the join does not.""" + """An older caller must still produce a schema-valid file.""" select, join = firm_sql("", "") assert join == "" assert select.count("'' AS") == 3 def test_a_map_without_a_canonical_table_makes_firm_repeat_firm_raw(): - """Honest rather than clever: without a reviewed table the split spellings - stay split, and `firm` says the same thing `firm_raw` does.""" select, join = firm_sql("/m.csv", "") assert "read_csv_auto('/m.csv'" in join assert "fc" not in join @@ -620,8 +601,7 @@ def test_the_map_path_is_escaped_like_every_other_literal(): def test_the_three_firm_columns_sit_between_person_domain_and_repo_tag( monkeypatch, tmp_path): - """Position is a claim, not a convenience: firm is resolved FROM - person_domain, so the key and its answers are adjacent.""" + """firm is resolved from person_domain, so they sit side by side.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path) names = [n for n, _ in schema_of(duckdb, out)] @@ -640,8 +620,7 @@ def test_a_call_with_no_firm_flags_writes_three_empty_strings( def test_a_domain_in_the_map_gets_its_firm_and_its_source(monkeypatch, tmp_path): - """The headline case, in miniature: the map says who, and firm_source says on - what evidence. `correction` is this repository's reviewed overlay.""" + """`correction` is this repository's reviewed overlay.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "qti.qualcomm.com,Qualcomm,company,correction") out = generate_with_domain(monkeypatch, tmp_path, "qti.qualcomm.com", @@ -652,8 +631,7 @@ def test_a_domain_in_the_map_gets_its_firm_and_its_source(monkeypatch, tmp_path) def test_the_canonical_table_fills_firm_and_never_touches_firm_raw( monkeypatch, tmp_path): - """The partner's standing preference: carry more, cut at publication. The raw - string is evidence and must survive beside the canonical name.""" + """The raw string is evidence and must survive beside the canonical name.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map( tmp_path, "au1.ibm.com,International Business Machines,company,cncf-gitdm") @@ -668,8 +646,7 @@ def test_the_canonical_table_fills_firm_and_never_touches_firm_raw( def test_a_name_the_canonical_table_does_not_mention_passes_through( monkeypatch, tmp_path): - """The table lists only the names that change. Everything else is already - canonical, and a missing row must not blank the column.""" + """The table lists only the names that change.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") @@ -682,8 +659,7 @@ def test_a_name_the_canonical_table_does_not_mention_passes_through( def test_a_domain_absent_from_the_map_gets_three_empty_strings( monkeypatch, tmp_path): - """An empty firm_source is the filter for 'not attributed at all', so it must - mean exactly that rather than 'the map had no source column'.""" + """An empty firm_source must mean exactly 'not attributed'.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") out = generate_with_domain(monkeypatch, tmp_path, "nowhere.example", @@ -692,8 +668,7 @@ def test_a_domain_absent_from_the_map_gets_three_empty_strings( def test_the_domain_match_ignores_case(monkeypatch, tmp_path): - """build_domain_map writes lower-cased domains, but persons.db carries - whatever the commit's e-mail header held.""" + """build_domain_map lower-cases domains; persons.db keeps the header's case.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "redhat.com,Red Hat,company,gitdm") out = generate_with_domain(monkeypatch, tmp_path, "RedHat.COM", @@ -702,9 +677,7 @@ def test_the_domain_match_ignores_case(monkeypatch, tmp_path): def test_a_company_spelled_like_a_number_stays_a_string(monkeypatch, tmp_path): - """all_varchar=true on the read. Without it DuckDB sniffs `360` as a number, - the firm columns change type, and validate_schema.py fails the whole corpus - on one project's map hit.""" + """Without all_varchar=true DuckDB reads `360` as a number and the schema breaks.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "360.cn,360,company,gitdm") out = generate_with_domain(monkeypatch, tmp_path, "360.cn", @@ -714,8 +687,6 @@ def test_a_company_spelled_like_a_number_stays_a_string(monkeypatch, tmp_path): def test_one_token_stays_one_row_when_the_map_matches(monkeypatch, tmp_path): - """The join must not fan out. This is the assertion that would catch a future - map keyed on something less unique than a domain.""" duckdb = pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") canon = write_canonical(tmp_path, "Google,Google LLC,merge,irrelevant here") @@ -727,9 +698,7 @@ def test_one_token_stays_one_row_when_the_map_matches(monkeypatch, tmp_path): def test_a_repeated_domain_in_the_map_stops_the_run_before_phase_1( monkeypatch, tmp_path): - """The one failure mode of this join that would be invisible. Two rows for one - domain duplicate every token row of every person on it: the file still - validates, the schema still matches, and only the row count betrays it.""" + """Otherwise invisible: the file still validates; only the row count grows.""" pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "google.com,Google,company,gitdm", "Google.com,Alphabet,company,gitdm") @@ -741,8 +710,7 @@ def test_a_repeated_domain_in_the_map_stops_the_run_before_phase_1( def test_a_repeated_name_in_the_canonical_table_stops_the_run( monkeypatch, tmp_path): - """Two canonical names for one raw string is an unresolved review, not a - default to pick from.""" + """Two canonical names for one raw string is an unresolved review.""" pytest.importorskip("duckdb") firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") canon = write_canonical(tmp_path, "Google,Alphabet,merge,one reviewer", @@ -760,8 +728,7 @@ def test_check_key_is_unique_returns_the_row_count(tmp_path): def test_a_canonical_table_without_a_map_is_refused(monkeypatch, tmp_path): - """argparse exits 2. There is no firm_raw to canonicalise without a map, and - accepting the pair would write `firm` out of nothing.""" + """Without a map there is no firm_raw to canonicalise.""" canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") with pytest.raises(SystemExit): run_main(monkeypatch, tmp_path, @@ -786,8 +753,7 @@ def test_a_missing_canonical_table_stops_the_run_before_phase_1( # --------------------------------------------------------------------------- # # Token-line repairs before the source walk: srcML mojibake, the Rust position -# prefix, and the byte-order mark. Each one moved the walk, so each test checks -# the text and the position of the token after the defect. +# prefix and the BOM. Each test checks text and position after the defect. # --------------------------------------------------------------------------- # def walk(tmp_path, name, source: bytes, *token_lines, stats=None): @@ -831,7 +797,6 @@ def test_a_mojibake_token_gets_the_true_text_and_the_walk_stays_aligned(tmp_path rows = walk(tmp_path, "a.c", C_SOURCE, *C_MOJIBAKE) assert rows[0][1] == "/* Högskolan */" assert rows[0][2].startswith("/* Högskolan */") - # The next token is where the source has it: line 2, column 1. assert rows[1][1:5] == ("int", "int ", 2, 1) assert rows[2][3:5] == (2, 5) @@ -864,7 +829,6 @@ def test_a_rust_line_loses_its_position_prefix(tmp_path): assert [r[0] for r in rows] == ["begin_unit", "keyword", "identifier", "op", "op", "op", "op", "end_unit"] assert rows[0][5] == 1 and rows[-1][5] == 1 - # The walk now agrees with the position the tokenizer wrote. assert [r[3:5] for r in rows[1:7]] == [(1, 1), (1, 4), (1, 8), (1, 9), (1, 11), (1, 12)] assert rows[2][2] == "main" @@ -904,9 +868,7 @@ def test_the_repairs_are_counted(tmp_path): def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path): - # The tokenizer no longer writes the prefix without --position. Such a - # line must pass unchanged: the repair removes text, it never moves a - # position by a fixed amount. + # The repair removes text; it never shifts a position by a fixed amount. stats = Counter() rows = walk(tmp_path, "m.rs", b"fn main\n", "begin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", From f75ca67fcdc826e9960856f51575c64ee6e6fb93 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 04:51:00 +0000 Subject: [PATCH 08/15] refactor(dataset): remove firm attribution Firm attribution is the author's ongoing research and is not meant for master. It stays on feat/firm-attribution (6e25165, 7549e54, e215e16) and in the tag corpus-c100-2026-10. Removed: the firm_raw, firm and firm_source columns; --firm-map and --firm-canonical in generate_dataset.py and their pass-through, help, variables and checks in run_pipeline_process.sh; the firm join, the duplicate-key pre-flight and firm_sql; the firm section and rows in DATASET.md; the 19 firm tests. The provenance columns, --project-meta and the #97 repairs are unchanged. The dataset now has 67 columns. On the test fixture the other 67 columns are identical to the 70-column output. --- generate_dataset/DATASET.md | 29 +--- generate_dataset/generate_dataset.py | 98 +---------- run_pipeline_process.sh | 24 --- tests/test_generate_dataset.py | 247 +-------------------------- 4 files changed, 12 insertions(+), 386 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 213100411..f08e53dd2 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -4,19 +4,19 @@ `generate_dataset.py` produces a unified Parquet dataset containing every token from a tokenized git repository, annotated with commit metadata, authorship information, -person identity, firm attribution, and the commit's git trailers. It is the final +person identity, and the commit's git trailers. It is the final output of the CreGit pipeline (Step 10). -**The schema is 70 columns**, in this order: +**The schema is 67 columns**, in this order: | Block | Columns | # | |-------|---------|---| | [Token data](#token-data) | `repo_name`, then `file_path` … `is_structural` | 1 + 8 | | [Per-project provenance](#per-project-provenance) | `clone_url` … `file_mask` | 29 | | [Git commit metadata](#git-commit-metadata) | `cregit_commit_sha` … `commit_summary` | 9 | -| [Person identity](#person-identity) | `personid` … `repo_tag`, including the three [firm](#firm-attribution) columns | 8 | +| [Person identity](#person-identity) | `personid` … `repo_tag` | 5 | | [Commit trailers](#commit-trailers-footers) | `footer_*` | 15 | -| | **total** | **70** | +| | **total** | **67** | `repo_name` is column 1 and the 29 provenance columns are 2-30, so `file_path` is column 31. @@ -114,26 +114,8 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or | `person_name` | `TEXT` | persons | Canonical display name for this person. Derived via `coalesce(p.personname, e.personid)`. If neither is available, this is `NULL`. | | `person_email` | `TEXT` | emails | Email address that matched this commit's author name/email pair. | | `person_domain` | `TEXT` | emails | Domain part of the email address. | -| `firm_raw` | `TEXT` | `--firm-map` | The map's `company` for `person_domain`, verbatim. `''` when the domain is not in the map. | -| `firm` | `TEXT` | `--firm-canonical` | Canonical firm name: `firm_raw` unless the canonical table renames it. | -| `firm_source` | `TEXT` | `--firm-map` | The map's `source`: `patch`, `gitdm`, `rich`, `builtin`, `correction` (hand-curated), `cncf-gitdm`, `cncf-gitdm-single` (single-person inference, 2,771 of 4,049 rows), `spinellis[-sec]`. `''` when the domain is not in the map. | | `repo_tag` | `TEXT` | commitmap | Repository tag indicating the origin repository. Values: `'p'` (pre-history), `'b'` (BitKeeper), `'l'` (Linux), or `''` (unknown/single repo). | -### Firm attribution - -Firm is resolved **per row** from `person_domain` by a LEFT JOIN against -`--firm-map` (`cregit-token-pipeline/data/affiliation.merged.csv`); `firm` comes -from `--firm-canonical`. A repeated key in either file would multiply token rows, -so the generator refuses it before Phase 1. - -```sql --- Firm-level token counts, excluding single-person inferences -SELECT firm, COUNT(*) AS tokens -FROM 'dataset.parquet' -WHERE is_structural = 0 AND firm <> '' AND firm_source <> 'cncf-gitdm-single' -GROUP BY firm ORDER BY tokens DESC; -``` - ### Commit trailers (footers) Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with @@ -384,7 +366,6 @@ Join `token_map` with the three SQLite databases: - `commits` — commit metadata (author, committer, dates, summary) - `commitmap` — cregit → original commit SHA mapping - `emails` / `persons` — person identity resolution -- `--firm-map` / `--firm-canonical` — firm attribution, when given Output is written as ZSTD-compressed Parquet. @@ -413,8 +394,6 @@ uv run python generate_dataset/generate_dataset.py \ | `--repo-name` | no | Repository name. Default: inferred from output filename. | | `--project-meta` | no | JSON sidecar of per-project provenance, from `project_meta.py`. Omit and those columns are written empty. | | `--project-key` | no | Which key of the sidecar this project is. Default: `--repo-name`. A key the sidecar does not hold is an error, not a blank row. | -| `--firm-map` | no | CSV of `domain,company,kind,source`. Omit and the three firm columns are written empty. A repeated `domain` is an error. | -| `--firm-canonical` | no | CSV of `firm_raw,firm,…` that fills `firm`. Needs `--firm-map`. Omit and `firm` repeats `firm_raw`. | | `--verbose` | no | Enable info-level logging to stderr. | ## Cross-reference: Perl ↔ Python diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index 12aab66a3..587370f99 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -461,66 +461,6 @@ def sql_literal(value) -> str: return "'" + str(value).replace("'", "''") + "'" -# Joined per row from person_domain through --firm-map, kept a file so the -# attribution stays reviewable. Columns: generate_dataset/DATASET.md. -FIRM_FIELDS = ("firm_raw", "firm", "firm_source") - - -def read_csv_column(path, column): - """One CSV column, as a list. csv, not duckdb: it runs before Phase 1.""" - import csv - - with open(path, newline="") as fh: - return [(row.get(column) or "") for row in csv.DictReader(fh)] - - -def check_key_is_unique(path, column, what) -> int: - """Refuse a lookup table with a repeated key; return the row count. Through - the LEFT JOIN a repeated key silently duplicates token rows.""" - keys = [k.strip().lower() for k in read_csv_column(path, column)] - if len(keys) != len(set(keys)): - seen, dupes = set(), [] - for k in keys: - if k in seen and k not in dupes: - dupes.append(k) - seen.add(k) - raise SystemExit( - f"{path}: {what} repeats {len(dupes)} key(s) in `{column}`: " - f"{', '.join(dupes[:5])}{' …' if len(dupes) > 5 else ''}. " - "A repeated key multiplies token rows through the LEFT JOIN.") - return len(keys) - - -def firm_sql(firm_map, firm_canonical) -> tuple[str, str]: - """(SELECT lines, JOIN lines) for the firm columns. Without --firm-map they - are empty strings and no join is added, so the schema stays the same.""" - if not firm_map: - return ("".join(f" '' AS {f},\n" for f in FIRM_FIELDS), "") - firm_expr = ("coalesce(fc.firm, fm.company, '')" if firm_canonical - else "coalesce(fm.company, '')") - select = ( - " coalesce(fm.company, '') AS firm_raw,\n" - f" {firm_expr} AS firm,\n" - " coalesce(fm.source, '') AS firm_source,\n" - ) - # persons.db keeps the e-mail's case, hence lower(). all_varchar stops a - # company like '1&1' or '360' being sniffed as a number. - join = ( - " LEFT JOIN (SELECT lower(domain) AS domain, company, source\n" - f" FROM read_csv_auto({sql_literal(firm_map)},\n" - " header=true, all_varchar=true)) fm\n" - " ON fm.domain = lower(e.domain)\n" - ) - if firm_canonical: - join += ( - " LEFT JOIN (SELECT firm_raw, firm\n" - f" FROM read_csv_auto({sql_literal(firm_canonical)},\n" - " header=true, all_varchar=true)) fc\n" - " ON fc.firm_raw = fm.company\n" - ) - return (select, join) - - def load_project_meta(meta_path, project_key) -> dict: """The project's constants, or empty strings when no sidecar was given. @@ -592,22 +532,6 @@ def main(): metavar="NAME", help="key into --project-meta (the manifest name). Defaults to --repo-name.", ) - parser.add_argument( - "--firm-map", - default="", - metavar="PATH", - help="CSV of domain,company,kind,source (cregit-token-pipeline/" - "data/affiliation.merged.csv). Joined per row against person_domain to " - "fill firm_raw and firm_source. Empty: the three firm columns are empty.", - ) - parser.add_argument( - "--firm-canonical", - default="", - metavar="PATH", - help="CSV of firm_raw,firm,... — the REVIEWED canonical-name table that " - "fills the `firm` column. Needs --firm-map. Without it `firm` repeats " - "`firm_raw`.", - ) parser.add_argument( "--memory-limit", default=DEFAULT_MEMORY_LIMIT, @@ -666,24 +590,6 @@ def main(): project_meta = load_project_meta( args.project_meta, args.project_key or args.repo_name) - if args.firm_canonical and not args.firm_map: - parser.error("--firm-canonical needs --firm-map: there is no firm_raw to " - "canonicalise without a map to read it from") - if args.firm_map: - if not Path(args.firm_map).is_file(): - print(f"ERROR: firm-map not found: {args.firm_map}", file=sys.stderr) - sys.exit(1) - n = check_key_is_unique(args.firm_map, "domain", "the firm map") - print(f"Firm map: {args.firm_map} ({n} domains)") - if args.firm_canonical: - if not Path(args.firm_canonical).is_file(): - print(f"ERROR: firm-canonical not found: {args.firm_canonical}", - file=sys.stderr) - sys.exit(1) - n = check_key_is_unique(args.firm_canonical, "firm_raw", - "the canonical-name table") - print(f"Firm canon: {args.firm_canonical} ({n} names)") - blame_files = sorted(blame_root.rglob("*.blame")) if not blame_files: print("WARNING: no .blame files found", file=sys.stderr) @@ -805,7 +711,6 @@ def main(): con.execute(f"CALL sqlite_attach('{args.persons_db}')") meta_sql = project_meta_sql(project_meta) - firm_select, firm_join = firm_sql(args.firm_map, args.firm_canonical) query = f""" COPY ( @@ -836,7 +741,6 @@ def main(): e.emailaddr AS person_email, e.domain AS person_domain, -{firm_select} coalesce(m.repo, '') AS repo_tag, coalesce(ftr.footer_signed_off_by, []) AS footer_signed_off_by, @@ -861,7 +765,7 @@ def main(): LEFT JOIN emails e ON (c.autname = e.emailname AND c.autemail = e.emailaddr) LEFT JOIN persons p ON e.personid = p.personid -{firm_join} LEFT JOIN ( + LEFT JOIN ( SELECT f.cid, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'signed-off-by') AS footer_signed_off_by, diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 275644ab1..3aca67567 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -120,11 +120,6 @@ Output: --project-key NAME which key of the sidecar holds this project. Omit to use --repo-name. A key the sidecar does not hold fails the run. - --firm-map PATH the domain->firm CSV joined per row against person_domain - to fill firm_raw and firm_source. Omit and they are empty. - --firm-canonical PATH - the reviewed canonical-name table that fills `firm`. Needs - --firm-map. Omit and `firm` repeats `firm_raw`. Tokenizer: --mode MODE tokenizer walk mode (default: pipeline) @@ -204,8 +199,6 @@ build_dataset_argv() { [ -n "$DUCKDB_THREADS" ] && DATASET_ARGV+=(--duckdb-threads "$DUCKDB_THREADS") [ -n "$PROJECT_META" ] && DATASET_ARGV+=(--project-meta "$PROJECT_META") [ -n "$PROJECT_KEY" ] && DATASET_ARGV+=(--project-key "$PROJECT_KEY") - [ -n "$FIRM_MAP" ] && DATASET_ARGV+=(--firm-map "$FIRM_MAP") - [ -n "$FIRM_CANONICAL" ] && DATASET_ARGV+=(--firm-canonical "$FIRM_CANONICAL") return 0 # a false test above must not fail the function under `set -e` } @@ -239,8 +232,6 @@ DUCKDB_THREADS="" # on whether the caller knows about the sidecar. PROJECT_META="" PROJECT_KEY="" -FIRM_MAP="" -FIRM_CANONICAL="" FORCE_CLEAN=0 MASK_WIDENED=0 RETOKENIZE="" @@ -418,8 +409,6 @@ while [ $# -gt 0 ]; do --duckdb-threads) need_val "$@"; DUCKDB_THREADS="$2"; shift 2 ;; --project-meta) need_val "$@"; PROJECT_META="$2"; shift 2 ;; --project-key) need_val "$@"; PROJECT_KEY="$2"; shift 2 ;; - --firm-map) need_val "$@"; FIRM_MAP="$2"; shift 2 ;; - --firm-canonical) need_val "$@"; FIRM_CANONICAL="$2"; shift 2 ;; --mode) need_val "$@"; MODE="$2"; shift 2 ;; --shards) need_val "$@"; SHARDS="$2"; shift 2 ;; --jobs) need_val "$@"; JOBS="$2"; shift 2 ;; @@ -548,19 +537,6 @@ if [ -n "$PROJECT_KEY" ] && [ -z "$PROJECT_META" ]; then exit 2 fi -if [ -n "$FIRM_MAP" ] && [ ! -f "$FIRM_MAP" ]; then - echo "invalid --firm-map: '$FIRM_MAP' is not a file (build it with build_domain_map.py)" >&2 - exit 2 -fi -if [ -n "$FIRM_CANONICAL" ] && [ ! -f "$FIRM_CANONICAL" ]; then - echo "invalid --firm-canonical: '$FIRM_CANONICAL' is not a file" >&2 - exit 2 -fi -if [ -n "$FIRM_CANONICAL" ] && [ -z "$FIRM_MAP" ]; then - echo "--firm-canonical without --firm-map has no firm_raw to canonicalise" >&2 - exit 2 -fi - # Validate early: step 2 runs for hours. blobExec enforces the relationship # between the two values; this only rejects non-positive integers. for _tv in "BLOB_TIMEOUT:$BLOB_TIMEOUT:--blob-timeout" "STALL_TIMEOUT:$STALL_TIMEOUT:--stall-timeout"; do diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index fd8111fa8..d91861e07 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -15,12 +15,11 @@ import pytest -from generate_dataset import (DEFAULT_MEMORY_LIMIT, FIRM_FIELDS, - PROJECT_META_FIELDS, SourceReader, - check_key_is_unique, firm_sql, is_ws, - load_project_meta, parse_memory_limit, - process_blame_file, project_meta_sql, - repair_token_line, skip_comment, skip_literal, skip_token, +from generate_dataset import (DEFAULT_MEMORY_LIMIT, PROJECT_META_FIELDS, + SourceReader, is_ws, load_project_meta, + parse_memory_limit, process_blame_file, + project_meta_sql, repair_token_line, + skip_comment, skip_literal, skip_token, sql_literal, undo_mojibake) @@ -304,11 +303,11 @@ def test_a_key_absent_from_the_sidecar_fails_loudly(tmp_path): # --------------------------------------------------------------------------- # # end to end, against DuckDB. The unit tests above prove the SQL text; only a -# written Parquet proves the column set, and 70 columns is the contract +# written Parquet proves the column set, and 67 columns is the contract # cregit-token-pipeline/validate_schema.py gates the corpus with. # --------------------------------------------------------------------------- # -TOTAL_COLUMNS = 70 # 38 token/commit + 29 metadata + 3 firm +TOTAL_COLUMNS = 67 # 38 token/commit columns + 29 metadata SHA = "a" * 40 @@ -519,238 +518,6 @@ def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): assert list(names) == [] -# --------------------------------------------------------------------------- # -# firm attribution: a per-row LEFT JOIN, so a duplicate key in either lookup -# table silently multiplies token rows. -# --------------------------------------------------------------------------- # - -FIRM_MAP_HEADER = "domain,company,kind,source\n" - - -def write_map(tmp_path, *lines, name="firm.csv"): - path = tmp_path / name - path.write_text(FIRM_MAP_HEADER + "".join(f"{l}\n" for l in lines)) - return path - - -def write_canonical(tmp_path, *lines, name="canon.csv"): - path = tmp_path / name - path.write_text("firm_raw,firm,decision,note\n" - + "".join(f"{l}\n" for l in lines)) - return path - - -def generate_with_domain(monkeypatch, tmp_path, domain, argv_extra=()): - """The tiny project plus one person on `domain`, so person_domain is set.""" - import generate_dataset as gd - - blame, src, cregit_db, persons_db = build_tiny_project(tmp_path) - con = sqlite3.connect(persons_db) - con.execute( - "INSERT INTO emails (personid, fullemail, emailaddr, emailname, " - "lcemail, userid, domain) VALUES (?,?,?,?,?,?,?)", - ("p1", "A Dev ", "a@example.com", "A Dev", - "a@example.com", "a", domain)) - con.execute("INSERT INTO persons VALUES ('p1', 'A Dev')") - con.commit() - con.close() - - out = tmp_path / "out" / "proj-dataset.parquet" - monkeypatch.setattr(gd.sys, "argv", [ - "generate_dataset.py", - "--blame-dir", str(blame), - "--source-dir", str(src), - "--cregit-db", str(cregit_db), - "--persons-db", str(persons_db), - "--output", str(out), - "--repo-name", "proj", - *argv_extra, - ]) - gd.main() - return out - - -def firm_of(duckdb, path): - return duckdb.sql("select person_domain, firm_raw, firm, firm_source " - f"from read_parquet('{path}')").fetchall() - - -def test_the_firm_field_list_is_three_names_in_dataset_order(): - assert FIRM_FIELDS == ("firm_raw", "firm", "firm_source") - - -def test_no_firm_map_emits_no_join_and_three_empty_literals(): - """An older caller must still produce a schema-valid file.""" - select, join = firm_sql("", "") - assert join == "" - assert select.count("'' AS") == 3 - - -def test_a_map_without_a_canonical_table_makes_firm_repeat_firm_raw(): - select, join = firm_sql("/m.csv", "") - assert "read_csv_auto('/m.csv'" in join - assert "fc" not in join - assert "coalesce(fm.company, '') AS firm," in select - - -def test_the_map_path_is_escaped_like_every_other_literal(): - """The query is one f-string and the path comes from the command line.""" - _select, join = firm_sql("/o'brien/m.csv", "") - assert "'/o''brien/m.csv'" in join - - -def test_the_three_firm_columns_sit_between_person_domain_and_repo_tag( - monkeypatch, tmp_path): - """firm is resolved from person_domain, so they sit side by side.""" - duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path) - names = [n for n, _ in schema_of(duckdb, out)] - i = names.index("person_domain") - assert names[i:i + 5] == ["person_domain", "firm_raw", "firm", - "firm_source", "repo_tag"] - assert len(names) == TOTAL_COLUMNS - - -def test_a_call_with_no_firm_flags_writes_three_empty_strings( - monkeypatch, tmp_path): - duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path) - assert duckdb.sql("select firm_raw, firm, firm_source " - f"from read_parquet('{out}')").fetchone() == ("", "", "") - - -def test_a_domain_in_the_map_gets_its_firm_and_its_source(monkeypatch, tmp_path): - """`correction` is this repository's reviewed overlay.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "qti.qualcomm.com,Qualcomm,company,correction") - out = generate_with_domain(monkeypatch, tmp_path, "qti.qualcomm.com", - argv_extra=("--firm-map", str(firm_map))) - assert firm_of(duckdb, out) == [ - ("qti.qualcomm.com", "Qualcomm", "Qualcomm", "correction")] - - -def test_the_canonical_table_fills_firm_and_never_touches_firm_raw( - monkeypatch, tmp_path): - """The raw string is evidence and must survive beside the canonical name.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map( - tmp_path, "au1.ibm.com,International Business Machines,company,cncf-gitdm") - canon = write_canonical( - tmp_path, "International Business Machines,IBM,merge,one firm spelled two ways") - out = generate_with_domain(monkeypatch, tmp_path, "au1.ibm.com", - argv_extra=("--firm-map", str(firm_map), - "--firm-canonical", str(canon))) - assert firm_of(duckdb, out) == [ - ("au1.ibm.com", "International Business Machines", "IBM", "cncf-gitdm")] - - -def test_a_name_the_canonical_table_does_not_mention_passes_through( - monkeypatch, tmp_path): - """The table lists only the names that change.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") - canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") - out = generate_with_domain(monkeypatch, tmp_path, "google.com", - argv_extra=("--firm-map", str(firm_map), - "--firm-canonical", str(canon))) - assert firm_of(duckdb, out) == [ - ("google.com", "Google", "Google", "gitdm")] - - -def test_a_domain_absent_from_the_map_gets_three_empty_strings( - monkeypatch, tmp_path): - """An empty firm_source must mean exactly 'not attributed'.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") - out = generate_with_domain(monkeypatch, tmp_path, "nowhere.example", - argv_extra=("--firm-map", str(firm_map))) - assert firm_of(duckdb, out) == [("nowhere.example", "", "", "")] - - -def test_the_domain_match_ignores_case(monkeypatch, tmp_path): - """build_domain_map lower-cases domains; persons.db keeps the header's case.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "redhat.com,Red Hat,company,gitdm") - out = generate_with_domain(monkeypatch, tmp_path, "RedHat.COM", - argv_extra=("--firm-map", str(firm_map))) - assert firm_of(duckdb, out) == [("RedHat.COM", "Red Hat", "Red Hat", "gitdm")] - - -def test_a_company_spelled_like_a_number_stays_a_string(monkeypatch, tmp_path): - """Without all_varchar=true DuckDB reads `360` as a number and the schema breaks.""" - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "360.cn,360,company,gitdm") - out = generate_with_domain(monkeypatch, tmp_path, "360.cn", - argv_extra=("--firm-map", str(firm_map))) - assert firm_of(duckdb, out) == [("360.cn", "360", "360", "gitdm")] - assert dict(schema_of(duckdb, out))["firm_raw"] == "VARCHAR" - - -def test_one_token_stays_one_row_when_the_map_matches(monkeypatch, tmp_path): - duckdb = pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") - canon = write_canonical(tmp_path, "Google,Google LLC,merge,irrelevant here") - out = generate_with_domain(monkeypatch, tmp_path, "google.com", - argv_extra=("--firm-map", str(firm_map), - "--firm-canonical", str(canon))) - assert duckdb.sql(f"select count(*) from read_parquet('{out}')").fetchone() == (1,) - - -def test_a_repeated_domain_in_the_map_stops_the_run_before_phase_1( - monkeypatch, tmp_path): - """Otherwise invisible: the file still validates; only the row count grows.""" - pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "google.com,Google,company,gitdm", - "Google.com,Alphabet,company,gitdm") - with pytest.raises(SystemExit, match="multiplies token rows"): - generate_with_domain(monkeypatch, tmp_path, "google.com", - argv_extra=("--firm-map", str(firm_map))) - assert not (tmp_path / "out" / "proj-dataset.parquet").exists() - - -def test_a_repeated_name_in_the_canonical_table_stops_the_run( - monkeypatch, tmp_path): - """Two canonical names for one raw string is an unresolved review.""" - pytest.importorskip("duckdb") - firm_map = write_map(tmp_path, "google.com,Google,company,gitdm") - canon = write_canonical(tmp_path, "Google,Alphabet,merge,one reviewer", - "Google,Google LLC,merge,another reviewer") - with pytest.raises(SystemExit, match="multiplies token rows"): - generate_with_domain(monkeypatch, tmp_path, "google.com", - argv_extra=("--firm-map", str(firm_map), - "--firm-canonical", str(canon))) - - -def test_check_key_is_unique_returns_the_row_count(tmp_path): - path = write_map(tmp_path, "a.example,A,company,gitdm", - "b.example,B,company,gitdm") - assert check_key_is_unique(path, "domain", "the firm map") == 2 - - -def test_a_canonical_table_without_a_map_is_refused(monkeypatch, tmp_path): - """Without a map there is no firm_raw to canonicalise.""" - canon = write_canonical(tmp_path, "NVidia,NVIDIA,merge,case only") - with pytest.raises(SystemExit): - run_main(monkeypatch, tmp_path, - argv_extra=("--firm-canonical", str(canon))) - - -def test_a_missing_firm_map_stops_the_run_before_phase_1(monkeypatch, tmp_path): - """A typo in the path must not produce a corpus of blank firm columns.""" - with pytest.raises(SystemExit): - run_main(monkeypatch, tmp_path, - argv_extra=("--firm-map", str(tmp_path / "absent.csv"))) - - -def test_a_missing_canonical_table_stops_the_run_before_phase_1( - monkeypatch, tmp_path): - firm_map = write_map(tmp_path, "a.example,A,company,gitdm") - with pytest.raises(SystemExit): - run_main(monkeypatch, tmp_path, - argv_extra=("--firm-map", str(firm_map), - "--firm-canonical", str(tmp_path / "absent.csv"))) - - # --------------------------------------------------------------------------- # # Token-line repairs before the source walk: srcML mojibake, the Rust position # prefix and the BOM. Each test checks text and position after the defect. From 725188e718333981e8b5460c9d67ae8d91ba9ec8 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 16:54:17 +0000 Subject: [PATCH 09/15] refactor(blobexec): keep the denylist as Scala data Return to master's design: the entries live in BlobDenylistEntries.scala, grouped by defect, and BlobDenylist only looks them up. The TSV resource and its parser are gone. Only the 12 non-terminating blobs stay: 8 Java (srcML/srcML#2361) and 4 C++ headers parsed as C. The 197 crash entries are dropped, because a blob whose tokenizer crashes is already excluded by the walk, with the same tree as a result. BlobDenylistSpec now asserts the shipped keys, their citations and the reproducers; the parser tests went with it. --- .../cregit/blobexec/blob-denylist.tsv | 219 ------------------ .../scala/cregit/blobexec/BlobDenylist.scala | 71 +----- .../cregit/blobexec/BlobDenylistEntries.scala | 36 +++ .../src/main/scala/cregit/blobexec/Main.scala | 24 +- .../main/scala/cregit/blobexec/Walker.scala | 2 +- .../cregit/blobexec/BlobDenylistSpec.scala | 198 +++------------- .../blobexec/WalkerIntegrationSpec.scala | 5 +- 7 files changed, 86 insertions(+), 469 deletions(-) delete mode 100644 blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv create mode 100644 blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala diff --git a/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv b/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv deleted file mode 100644 index 5a757a58d..000000000 --- a/blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv +++ /dev/null @@ -1,219 +0,0 @@ -# Blobs blobExec never tokenizes, keyed by git blob sha so one entry covers every -# path and copy of the content. A listed blob is dropped from rewritten trees, -# counted as blobsDenylisted, and does not change the exit status. -# -# Class 1: srcML Java parser hangs on annotated array types (srcML/srcML#2361). -# Class 2: srcML C parser hangs on C++ inside a .h (no upstream issue). -# -# sha citation reason -2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65 srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar -303c1a109f1fdc4b9d84e15111c56931202844ac srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar -504aaf9109fc01d6d62d99ac529ded99a8dfa8ca srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar -c91924d1e87cb82d372a73b24579ccc34609efaa srcML/srcML#2361 srcML 1.1.0 does not terminate on this file's annotated multi-dimensional array declarations (0 bytes out, killed at every budget from 5s to 600s); OpenJDK langtools regression test @bug 8005681, written to stress this grammar -56c8adf588e697a3e42b600b2e5ec54710592c61 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 178, `Object [] @TA [] arr @TA[];` — an annotated array dimension in the declared type together with an annotated trailing C-style dimension on the declarator, needing no initializer; minimal reproducer `class C { int @A [] a @A []; }` (0 bytes out, 100% of one core, RSS flat at 10,108 kB, at 15s and 60s here and at 600s twice in the pipeline); line 179 is not a trigger; OpenJDK langtools regression test CheckErrorsForSource7, 8,388 B -5e9c9138ed9c43c20591d1814cb72ad014d1a9e5 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 181, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); a second historical version of CheckErrorsForSource7.java, 8,750 B, recorded at both of the file's two paths -6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7 srcML/srcML#2361 srcML 1.1.0 does not terminate on line 179, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); a third historical version of CheckErrorsForSource7.java, 8,645 B -afbd81e5789ba060147b0de3be4f01759963299d srcML/srcML#2361 srcML 1.1.0 does not terminate on line 178, `Object [] @TA [] arr @TA[];` — same construct and same minimal reproducer `class C { int @A [] a @A []; }` as 56c8adf5 (0 bytes out at 15s, 100% of one core, RSS flat); the fourth historical version of CheckErrorsForSource7.java, 8,617 B, after the OpenJDK path move -5460d43f1884e3be230e02215151b9b963844a48 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: srcML 1.1.0's C parser does not terminate on this C++ header, which CregitLanguages.pm maps to language C; minimal reproducer `namespace n{struct T&f(const struct S);}` — a named namespace or class block containing a function returning a reference to an elaborated struct type and taking a cv-qualified elaborated struct parameter (0 bytes out, 100% of one core, RSS flat at ~9,300 kB, at 5s/15s/60s/300s here and at 600s twice in the pipeline); the continued `#elif` conditionals are NOT the trigger and the same bytes parse as C++ in 9 ms; eden/fs/utils/StatTimes.h, 1,861 B -720914b27e170b73b38ef662fb04239273dd0a34 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); a second historical version of eden/fs/utils/StatTimes.h, 2,580 B -99c191e9a208547f5c9b1224c290f106a640de59 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); a third historical version of eden/fs/utils/StatTimes.h, 2,443 B -dbb885e0bf046755c4fcfbddfc2454f90c550c62 srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h NOT srcML/srcML#2361, and no upstream issue exists: same C-parser non-termination and same minimal reproducer `namespace n{struct T&f(const struct S);}` as 5460d43f, and the same `const struct timespec&` reference return type (0 bytes out at 15s and 60s, 100% of one core, RSS flat); the fourth historical version of eden/fs/utils/StatTimes.h, 2,442 B -# -# Class 3: srcML C/C++ parsers die on a signal under --position (no upstream issue). -046a9a1a8b09089625dc77d04185415a710cd1aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,549 B -0cdf543499d0ee5e962b37fdd574131f8b03f711 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 117,124 B -d1d06fc15b4e6abbd1dffb7c11e6d5c7bdff736e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd server/util.c, 122,431 B -1de38760dd5228b923c133c4b40002e1bde6b9aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,745 B -1e35a9a56726e5884dd3bca8ea8426308aa0061c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,903 B -414e0887fe445325673b38c8fef8d26164e69e02 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,731 B -94ffeaaf43b92a2dc665a4e2b691a3a7122a064f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 34,194 B -a090ac7b09bbc414319da6000cfe027e853b25ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. apache__httpd test/clients/h2ws.c, 33,165 B -e35e38af931a76ff0e2a9b98d8ce9cab567e03c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. aws__amazon-redshift-odbc-driver src/pgclient/src/backend/libpq/zstd-in.c, 2,057,541 B -28530d12c2ed7de827ca2e14ce099262a34492de silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 6,158 B -39a6ec8d4c8a45ae6558bbe78d9b562c33ca47ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 9,471 B -67e558666399f02c73e8606b72af5f7c9d3964c1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,226 B -69b1d0edde928e85f7afa4c8285e46eba84051dc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,249 B -715d5b85a6bf60ce63a87f9ae984dcd5f98363c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,221 B -abb9cde9969ea21171a0b1278dc3f5c6cbe7af4e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,242 B -bd5bf7a19faa968b9ae40a58b967a0f74dcc4fbd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 12,978 B -be5b2ada16d13bf6438eb7a6d3395645d7935681 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,226 B -c4a7154ba732cc19a00f8cf7d7f203c29d8b0249 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,249 B -e2b23bbeb7ee563c332baaf3e0847ab645a5d2b8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. dpdk__dpdk drivers/net/pfe/pfe_hif.c, 21,256 B -3b15148d68dc75dbe727df52fe6cd9242481512f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 26,726 B -456629abf1392f382a62c569fe99080ec8f07404 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 27,429 B -a17001dd91b1247f83213a80442800dbb2614604 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/alignment.h, 26,789 B -1e2976b83bc76371b044cbc0e680e081a1416c7f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 16,586 B -1f59b32426b3ece9b10cb947de09c52f2d0ed065 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 18,702 B -50f2a29a7237e264ea437e299147c1af3e7210c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 17,339 B -7bb26742930a9e6d494ca3693ebd3bcdb64ebf69 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. duckdb__duckdb third_party/mbedtls/library/common.h, 16,614 B -f0787fd85cd49490497fe8743d8eec107e0b2353 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. elfmz__far2l plugins/multiarc/src/formats/7z/C/Sha512.c, 19,801 B -933fcad8c46bd263441bbac4dab084c53090004b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. elfmz__far2l plugins/multiarc/src/formats/rar/unrar/blake2s_sse.cpp, 4,461 B -05ed4a412d7acff9629abd4824ca98ac22724875 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 152,327 B -0c24a13cb2c2ea0543d3f4b4ebe0fa3741285d30 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,263 B -0ec1d427e27bbc5c0ad42ced54e5afa8a19ae15a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,050 B -0f3bfe9b3b225213ce7880fb5119cf786296df86 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,673 B -0f707c0d21a74a74654ff03bc14bf616e23e0110 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,529 B -2c031624ce9da4a9f75d9dc29eb5076a002901e3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 164,630 B -2f193091254956a10f9feb4712a155a0818fa7f2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,394 B -2fb3a04df9fcd8ad3786b8494b5066bd95308eb7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,616 B -33e10e272e8de9b3d0f31e3fa6a30b8d25d8a62a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,905 B -33e33b44fa67050820759b63d1dd419a9ef45dbf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 154,285 B -383f345c57e8a718ffc6093beee12fb228b24607 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 161,635 B -3e5313015bd651ae57a1155f236d748cfce434e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,029 B -404e2daeba97c1d3cb4e6550b064ec338cbe0f6f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,497 B -4177bb55017d51c2e2e3ac997e60fcc84e1bddd6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,049 B -4509d2f318f19398da26a0f0506d8751dd26b24e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,499 B -4b30e2af77bcb5ff028f3bd947602cef36ec3ab2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,335 B -51e90452f68b0f627092148f52a87657701026a4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 161,634 B -52aa3a0bf7091d618239c73168f07820e853861e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,410 B -5e365fa0b1bc5b4f6d96a5447cfc11181b81a730 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 162,192 B -602546e19350bc001678de2e8be8a8bb70c7b18d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,337 B -6171c31762db866fbb545797ce21f269c11640dd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,544 B -65bbeae52abae4975141f4ed2a04563e361c8cec silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,036 B -6f02daee0dcb4f3107441c0009b888000c1342de silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,741 B -7e1db5cda9298e20c837e2824060559aef1aa868 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,429 B -87fd46e27098697fcabecd0a86a82c60d2bc1390 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 171,316 B -89c41a15747b2a9ce0f120c403f89a751d17c3e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,922 B -8cc2d1e47bfac43d027ef03ad7c763e07ea107e6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,863 B -916d906d99baea144dd1134c10c1c01d32fc1d15 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 175,544 B -96390ceef0abcbe821337850f0584d81de34adb4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,471 B -9a03addfdacdb46f0310952164ca451d5a223427 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,550 B -9a1ee408870ae2d572b242d006e6bd3e042489c8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,034 B -a0a74b95221d915d95678f2716ebe599f21cd11b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,035 B -ab86df42865e5aebbc3f60b547a365b3bb8b0575 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,732 B -ae43f731c3c433a608f5dc4e83c2bdfaeae9190d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 157,334 B -b5dc5f6630e608e49e7208d18cd7a371845a202c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,954 B -ba0134cc103445972e162e8e2b0e6b1ea57115b5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,055 B -bd3cf5ecdd953ee1d0b2523977f6045914ab243c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 170,826 B -c32127a08e590715f0342f223378b40a8d5882ce silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 158,389 B -cf65f8536ac904ae955c6ec9bc62c28e3f1cbf07 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,220 B -d0655644a8d0dd386ad46413efa0dd27ea8147b9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 159,474 B -d60d3dd4e2ae1b271f1f2ca66da946f4d15951f5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 153,857 B -def89df2cdd9275cbe7e5929b0521b90bae51139 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,540 B -e02931f0940695f5da27ac080af14cf9b63e0bc9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 175,420 B -e112238123a7a8f78c5ed5579dbe71fbc6bd6a19 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 165,385 B -ea6a60c02519c14cdfcd7de0feb00d3df8f25dee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,830 B -eadd203c8afa9fba2680c0f61a26305f7a086789 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 168,740 B -ef3d052b726a202edc7cf09a4a53fbd5c00ad118 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,125 B -f0b497b6014ca90691dc415830fa0a5491ad9554 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 174,642 B -f4f93443ddf9549005694ac1cf9d6d60f3248ea7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 169,221 B -f5b021c8e98afac62fe5a9dd99d2e17430d7980e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 167,397 B -fabb972a818449b3df1cec8df75f783e1dec263b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. googleapis__cpp-bigquery-odbc google/cloud/odbc/integration_tests/odbc_driver_tests/statement_test.cc, 155,033 B -1c85746e4c4bc6488f8726a1efa8a15dccdddc04 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. java-native-access__jna native/libffi/testsuite/libffi.call/many2.c, 1,070 B -98eac6013a4381071b6785fff399fe1b52cce310 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. java-native-access__jna native/libffi/testsuite/libffi.call/many2.c, 1,069 B -03283dabd1c32502905c92398e06b6f91fdacb60 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,410 B -228f8bd0b451a34040e5f759158e7968d63214e4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,393,936 B -366883c533a0b821c236c0714f3ff7ad79fce474 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,670,090 B -4042c1d6bc6610f4dd8ac85488807cd84275a6ec silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,500 B -573e70c2b27e1fab1d4fbe4c2ac2edca14df7afa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,429 B -81363cec7afdbcd0a35b9265c17c47cb73aeef87 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,574,270 B -8de338781a9ea4640d30bcf6b0fd04da0eccf1e8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,398,868 B -97bd7dcf18b7bfdd3861a03d1827554d8fb5f310 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,505 B -9e0580406539837a64a1f25868f869ff0002b2ee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,670,082 B -b5c33d555fa2b71760e771481810242bdf09096a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,683,415 B -ba4572cb83bd59a19a2745bfee680fe2bcb2ff4c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,465,940 B -ba55550ed25b41aa9e66749389f649586426c697 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,514,074 B -c3781f766fba4ec06491897a9965428ee58a6898 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,574,278 B -c6f1f3b49ab279df3fdf4909229babdb8cac318a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,900,518 B -cd2f7919024d8e863aae68ca0d7f7427e2412890 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,675,329 B -e836ef4bc55254f70a0c535bc8a6c1d0b3b9a58d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kamailio__kamailio src/modules/app_jsdt/duktape.c, 3,675,310 B -53dacfa43d8932e31c2ba1cf5dac9a09bef3237e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. kicad__kicad-source-mirror thirdparty/eigen/Eigen/src/Core/arch/MSA/Complex.h, 17,541 B -3575d3b6aa7d0ea111cd5cb80212fc670d8a10cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. nationalsecurityagency__ghidra Ghidra/Features/Decompiler/src/decompile/zlib/deflate.c, 81,787 B -30597abfa1e1320dfb5e91e7445b2b534c6672d1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,448 B -309853a1f6910e72061c215686b43aca0441f783 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,091 B -3f231fd0435ba0f4be9c9c8c3985bb7fb0bfa0cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,314 B -4ed2f82097b0a6436b610ea14f1a1fe10e5f8b41 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 299,493 B -6258b7a7135bd2beab1a8d239ea309a8c8acd21d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,058 B -6930d8f7a42292a35e76f442b1c960343e5204b4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,424 B -6c04e1c1b62f35a51bb1ddc07db3b7f7dbaad014 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,269 B -79dbbc97aac295ee5a1c44f3a2a7a7007891ece2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,813 B -8256f5a96a8f1db7629e2c5651c599f62a7db53e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,736 B -907f04f2f4b0edf4d8ea37ff0cdfde4feccc20fc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,824 B -90c4a9adacb5423ef6f2f306378377117deaf4c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,118 B -9ce560f932dc17daeda1047a3bce933e451d707b silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,277 B -aa013c1884f847ae3c6b78fb25d54bd8a01644bb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 117,839 B -af218291846c944e3fd4dd29d811c1593c57f9ef silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,253 B -c9a41217fbc425ff0c867613473ff9c177809ecc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,857 B -d3283082d7871dbce2e305a98ec165c3c9fe6ebd silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,835 B -d57032de2134b34e0e99191f9f96bc85bbb548c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,262 B -db2e30f1f0994a55231784858640bf9038c466aa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 112,733 B -dede365cbe9be3cf8410be0cd4f11def94182faf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 118,428 B -df1da8d1e931da4c081960e3839239b4a66dfbe7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 295,821 B -e3ec9472d5501c7fcd13f7528c0ca5bce8ebdcd3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,170 B -ebfa62a39ad6d1418075c18310dc4585ab139fc3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,286 B -ef36c1b86571b8faea2e8cb068a6748ae0eef9ca silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 290,094 B -f5e4d64b0c3dab24c0b776114934740e3445f6e2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here dropping --position would not recover the blob: without it srcML exits 1 with a parse diagnostic and still produces no tokenization. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. oceanbase__oceanbase deps/oblib/src/lib/charset/ob_ctype_uca.cc, 296,194 B -2ee2dd4d33960766e04e4ccf58bca5a9e8afe0ca silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,884 B -333d4687230a78f8f6b9fd50e1a4af58510a1e4d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,709 B -3a8727bd6b473ddf301a61fb2b369b888e2b5826 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,043 B -4b77786aa5b7e6090f50be39c43da7dce0c585e9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,745 B -4cd23b5415822f5c9b8b762b6ed6366fa329dca1 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,665 B -6084d12c5ff6ffc0b662649848ec7f3ea19ecc30 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,848 B -60be21e572963097c0a29a279228f51d8d922724 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,658 B -69303749878b3535bf61b40d425a3de380a74872 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,782 B -72903c04d785a966a580d4a5fb43522d194c3f32 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,519 B -76bb8392b107bf6ee1a7af6623217316f8c1bdd5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,059 B -8f569ae5f2e701e770e5612c7740a606a278b068 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,876 B -9dc3d5ec0d9cb5a0a993e830c30361538f760a1f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,756 B -a48a2ada09935e4e9b477342f4eceb478d160027 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 28,849 B -c7ef4dcd816f0c564a90d045d7a3be0140094866 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,030 B -cf3211cb9740d1fd851ebd6e2d23ab3b0f9bc344 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,677 B -dc68ca6fc776617bf5a4400c775f42022bc483df silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,364 B -e7bee4e9aa5e2f4b019ec68929c0b95b13d57d9e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,540 B -ea3aa7f42b60a23aeceea1a39f5ca7215e2603a7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,169 B -ee6c056ef7a96062cf43e4728a6f4343f3f2bc63 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,676 B -f76fe3ed5fa1279aeefa6bd090d74a86608ce87d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. openrc__openrc src/librc/librc-depend.c, 29,168 B -012ea8148e8d21cf515a9ff47d4af719d049b276 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. ossec__ossec-hids src/external/zlib-1.3.1/deflate.c, 81,731 B -32ba0a14f4cbe94f638959a8ff1ed5e19bf8814c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/a-unrar/unrar.cpp, 705,097 B -ffc37ddd5fd2895ceb6ce95744250b51ee4ba408 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/a-zlib/zlib.c, 819,904 B -675e80674311343726ca271fc2ab79ebca3a2d81 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/chmdec/chm.c, 72,897 B -cd8c0847a22e1f1dcf4e0ec44e3e3dd859318371 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/libjpeg-turbo/src/jdhuff.c, 26,826 B -4a62830c807a5a2aff73f474891c38ed913d39c4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/liblzma/check/crc32_small.c, 1,263 B -ee4ea26f67d0df5e9a39be5d60234c1393c89b3c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/liblzma/check/crc64_small.c, 1,023 B -791d224754ee54e930a43141b09a7bf459658a4c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. sumatrapdfreader__sumatrapdf ext/webview2/WebView2EnvironmentOptions.h, 19,500 B -6d2176c258b34fa814fef75f6934e110973dfa78 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGABRT, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. tencent__tendbcluster-tdbctl cmd-line-utils/libedit/history.c, 25,445 B -6917923a35b45c9f556ba06baa84262066f5625d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C++ blob under the tokenizer's own `srcml -l C++ --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. tencent__tendbcluster-tendb storage/rocksdb/rocksdb/utilities/document/json_document.cc, 16,883 B -c229eda7d995851318f82aa3fa462cf227f09884 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_feather/flexspi_nor_config.c, 1,106 B -3bab1761d7c85b9b9f5074db9f2220f9f48e94d3 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_feather/mmfeather_sdram_ini_dcd.c, 7,281 B -cac7fa33ccf1bfd40439ffb5ee0ffb5d3d081468 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/madmachine/mm_swiftio/mmswiftio_sdram_ini_dcd.c, 7,281 B -7a59acd6c54fe2497a442d7fb3505ccb1a4e2f2f silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/nxp/vmu_rt1170/flexspi_nor_config.c, 4,437 B -85ad476cf84c051d510190a73b5274fdb0496ed8 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/nxp/vmu_rt1170/flexspi_nor_config.c, 4,415 B -020bf6abd57b2676c207174943f5ed1d06130ecf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position IS the trigger: the same bytes parse cleanly (exit 0) without it. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. texasinstruments__simplelink-zephyr boards/pjrc/teensy4/flexspi_nor_config.c, 1,146 B -27aa72b69a124dae8bdcadfa8d6a497ea59efecb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,509 B -29484ffe569ec3c4ec13acb88577b8b7cf252463 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,554 B -2c56e15018af94c8aadd91ea77d08cc356eaa21d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,557 B -37cf5d5b1985a7510ad9c952897ef253231e80f7 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,560 B -6b5078677fba7ef7de770623564e68e9e38cee0a silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,549 B -8498d2ea64b268b431d3c38cde0ea6ebf27c34d2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,538 B -b804a92bc31de459b9781febb3fb87c9af54c6fb silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,501 B -ccc226a02dccbbee5306569fa945cd4b5996a2e2 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,512 B -e6ba7d66d7f12f57e87f158decc659276cb5f4cc silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux disk-utils/swaplabel.c, 4,490 B -1142febb016545cd524a49f8019e1a5086d7abee silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,670 B -447a342e7012f9cbad0c5681d8d2ce13adb1e6a4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,750 B -4d706c3a95a53e6cbdecd770238c42cf5b12a8c9 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,834 B -62ac281ce0e097b987ffad41b02a5d71325ec15c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 16,589 B -7779e10e68dcfeedcf6f55d7f91b75086f98fffe silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,068 B -80b69eea271bef4d8f39fe681b23f307284444f0 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,387 B -89f1e216984a0ffcae4b0f6762d11baedf5df6b0 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,129 B -8ae02efd5abb170fc942396588fa5e403aecaf48 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,774 B -8ca2c0aaeb1e5a0ef5de2592426befdfaf11434e silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,606 B -95ee516350cae32d784ad46c1a156fd2dc2e07c6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,784 B -a90ca6913f268161ab85d919d8cf25b24d508d43 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,028 B -a9c2022be5fc36bc4aaf1de3a9f7696ab3301eff silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,386 B -b7acae43082196238a408a9cf5a2aed169a79e06 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 6,785 B -be480dff5a7c1aead25e0e6e8ee4b8b93129cf38 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,149 B -c4e180617725397a291f948c566b799155361d55 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 8,502 B -c600708556fc3f92056b87d73ab7e94168eb89f5 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 7,412 B -d607f13c33fa52261dba8d81315fbb39af719c87 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 16,601 B -dd7c5ffc1f8662932b8f799576554f730923d393 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,323 B -dfce9c5d7046f70a9f8f0af22ecba28ee9a9c4f6 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGSEGV (shell status 139) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,147 B -e405850bf7f659dc01dc53dcba9571b21227bb7d silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,040 B -eba3aba4c5b8ed7528eb97f5c2a9aed3f7488ce4 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 14,884 B -f605b07bdd5490314575bca73fe9d17d61a3acfa silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 11,092 B -fb88a5b70235c9e9cd87239b98590ccdf46e704c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 9,103 B -ffdf47ff4d20bbf5c96196b27fe346f0d86a8336 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/fileutils.c, 10,808 B -32b67a3b19775cc4a84a8818811ec9feb84dba85 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,315 B -46b6463387b685600af0e28dddf14022d1e4990c silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,961 B -895f6e75239f112a3496156b5b0b4e278a8248cf silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,312 B -a837359201821dff2b906194381b228dd1a4a926 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,980 B -aeda953c0d26024896b54451fa0aa339f6e30233 silent-empty-sweep.md#the-probe-i-had-to-redo--and-why-it-matters srcML 1.1.0 is killed by SIGABRT (shell status 134) parsing this C blob under the tokenizer's own `srcml -l C --position` invocation, so the parse yields 0 bytes and tokenizeSrcMl.pl exits 33; --position cannot be dropped because the token format carries line:col positions, and here --position is NOT the trigger: srcML also dies, on SIGSEGV, without it, so the flag only changes which signal. A CRASH, not the non-termination of srcML/srcML#2361 and not srcML 1.1.0's C-parser hang on C++ in a .h; measured 2026-09-20 against the pinned srcml 1.1.0. util-linux__util-linux lib/pidfd-utils.c, 2,978 B diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala index 3a4806ebb..d9fcba16d 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala @@ -1,77 +1,28 @@ package cregit.blobexec -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Path} - -import scala.jdk.CollectionConverters._ -import scala.util.Using - -/** Blobs never handed to the tokenizer, keyed by git blob id. The list is a data - * file ([[BlobDenylist.ResourcePath]]) so the dataset can cite it. */ +/** Blobs never handed to the tokenizer, keyed by git blob id. Excluded like an + * oversized blob, and deliberately not gating the exit status. */ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.Entry]) { - def entryFor(sha: String): Option[BlobDenylist.Entry] = - if (bySha.isEmpty) None else bySha.get(sha.toLowerCase) + def entryFor(sha: String): Option[BlobDenylist.Entry] = bySha.get(sha.toLowerCase) def size: Int = bySha.size def isEmpty: Boolean = bySha.isEmpty - def shas: Set[String] = bySha.keySet - def entries: Vector[BlobDenylist.Entry] = bySha.values.toVector.sortBy(_.sha) } object BlobDenylist { - final case class Entry(sha: String, citation: String, reason: String) { - def describe: String = s"$reason [$citation]" - } + /** Both fields are required: an exclusion nobody can trace back to a defect is + * indistinguishable from data loss. */ + final case class Entry(citation: String, reason: String) - val ResourcePath = "/cregit/blobexec/blob-denylist.tsv" + /** Where a reader finds the list, quoted in the excluded-blob report. */ + val EntriesSource = "BlobDenylistEntries.scala" val empty: BlobDenylist = new BlobDenylist(Map.empty) - private val ShaPattern = "^[0-9a-f]{40}$".r - - /** Throws on malformed input: a dropped line would silently re-admit a blob. */ - def parse(lines: IterableOnce[String], where: String): BlobDenylist = { - val acc = scala.collection.mutable.LinkedHashMap.empty[String, Entry] - lines.iterator.zipWithIndex.foreach { case (raw, i) => - val line = raw.trim - if (line.nonEmpty && !line.startsWith("#")) { - // Trailing tabs are tolerated; an empty field in the middle is not. - val fields = line.split("\t", -1).map(_.trim).reverse.dropWhile(_.isEmpty).reverse - if (fields.length != 3 || fields.exists(_.isEmpty)) - throw new IllegalArgumentException( - s"$where:${i + 1}: expected 3 non-empty tab-separated fields " + - s"(sha, citation, reason), got ${fields.length}: [$line]") - val Array(sha, citation, reason) = fields - val key = sha.toLowerCase - if (ShaPattern.findFirstIn(key).isEmpty) - throw new IllegalArgumentException( - s"$where:${i + 1}: [$sha] is not a 40-character hex git blob sha. The key is the " + - "git blob id, which is what blobExec logs, not the memo's content hash.") - if (acc.contains(key)) - throw new IllegalArgumentException( - s"$where:${i + 1}: [$key] is listed twice. Two reasons for one blob means one of " + - "them is wrong, and the list is what the paper cites.") - acc.update(key, Entry(key, citation, reason)) - } - } - new BlobDenylist(acc.toMap) - } - - def fromFile(path: Path): BlobDenylist = - parse(Files.readAllLines(path, UTF_8).asScala, path.toString) - - def fromResource(resource: String): BlobDenylist = { - val stream = Option(getClass.getResourceAsStream(resource)).getOrElse( - throw new IllegalStateException( - s"blobExec resource [$resource] is missing from the classpath. The blob denylist " + - "ships inside the jar; a jar built without it would hand four known " + - "non-terminating blobs to srcml again.")) - Using.resource(scala.io.Source.fromInputStream(stream, UTF_8.name))(src => - parse(src.getLines().toVector, resource)) - } + def apply(bySha: Map[String, Entry]): BlobDenylist = + new BlobDenylist(bySha.map { case (sha, e) => sha.toLowerCase -> e }) - /** Parsed on first use; a parse failure is fatal by design. */ - lazy val shipped: BlobDenylist = fromResource(ResourcePath) + val shipped: BlobDenylist = apply(BlobDenylistEntries.NonTerminating) } diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala new file mode 100644 index 000000000..236feae7e --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala @@ -0,0 +1,36 @@ +package cregit.blobexec + +/** Blobs on which srcML 1.1.0 does not terminate, keyed by git blob id and grouped + * by defect. A tokenizer crash needs no entry: the walk already excludes that blob. */ +private[blobexec] object BlobDenylistEntries { + + import BlobDenylist.Entry + + private val JavaAnnotatedArrays = Entry( + "srcML/srcML#2361", + "srcML 1.1.0 does not terminate on annotated array dimensions, minimal reproducer " + + "`class C { int @A [] a @A []; }`; OpenJDK langtools regression tests " + + "TestNewCastArray (@bug 8005681) and CheckErrorsForSource7" + ) + + private val CppInCHeader = Entry( + "srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h", + "srcML 1.1.0's C parser does not terminate on C++ in a .h file (no upstream issue), " + + "minimal reproducer `namespace n{struct T&f(const struct S);}`; eden/fs/utils/StatTimes.h" + ) + + val NonTerminating: Map[String, Entry] = Map( + "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65" -> JavaAnnotatedArrays, + "303c1a109f1fdc4b9d84e15111c56931202844ac" -> JavaAnnotatedArrays, + "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca" -> JavaAnnotatedArrays, + "c91924d1e87cb82d372a73b24579ccc34609efaa" -> JavaAnnotatedArrays, + "56c8adf588e697a3e42b600b2e5ec54710592c61" -> JavaAnnotatedArrays, + "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5" -> JavaAnnotatedArrays, + "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7" -> JavaAnnotatedArrays, + "afbd81e5789ba060147b0de3be4f01759963299d" -> JavaAnnotatedArrays, + "5460d43f1884e3be230e02215151b9b963844a48" -> CppInCHeader, + "720914b27e170b73b38ef662fb04239273dd0a34" -> CppInCHeader, + "99c191e9a208547f5c9b1224c290f106a640de59" -> CppInCHeader, + "dbb885e0bf046755c4fcfbddfc2454f90c550c62" -> CppInCHeader + ) +} diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index d8c55df67..ee618ca70 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -196,12 +196,11 @@ object Main { | process is killed with status ${Walker.StalledExitStatus}. The memo is | durable, so re-running resumes. | - | Blobs on the shipped denylist (${BlobDenylist.ResourcePath} inside this - | jar) are never handed to : they are dropped from the rewritten - | trees, counted as blobsDenylisted, named with their reason and citation on - | an EXCLUDED line, and they do NOT change the exit status. srcML 1.1.0 does - | not terminate on the listed blobs, and the defect is diagnosed and cited - | upstream, so the list saves the --blob-timeout each run would spend. + | Blobs on the shipped denylist (${BlobDenylist.EntriesSource}) are never + | handed to : they are dropped from the rewritten trees, counted as + | blobsDenylisted, named with their reason and citation on an EXCLUDED line, + | and they do NOT change the exit status. srcML 1.1.0 does not terminate on + | the listed blobs, so the list saves the --blob-timeout each run would spend. | | A blob whose tokenizer times out or reports a parser crash is excluded | the same way: dropped from the rewritten trees, counted as blobsTimedOut @@ -453,14 +452,7 @@ object Main { val dbParent = dbPath.getParent if (dbParent != null && !Files.isDirectory(dbParent)) Files.createDirectories(dbParent) - // Eager, so a missing or malformed denylist stops the run now, not mid-walk. - val denylist = - try BlobDenylist.shipped - catch { - case e: Exception => - System.err.println(s"Error: cannot read the blob denylist: ${e.getMessage}") - sys.exit(1) - } + val denylist = BlobDenylist.shipped val incremental = Files.isDirectory(dstPath) val shardStr = shard.map { case (k, n) => s"$k/$n" }.getOrElse("none") @@ -596,10 +588,10 @@ object Main { if (stats.blobsDenylisted > 0) { System.err.println( s"blobExec: ${stats.blobsDenylisted} blob(s) were excluded by the blob denylist " + - s"(${BlobDenylist.ResourcePath} in this jar, ${denylist.size} entr" + + s"(${BlobDenylist.EntriesSource}, ${denylist.size} entr" + s"${if (denylist.size == 1) "y" else "ies"}). Each one is named with its sha, path, " + "reason and upstream citation on an 'EXCLUDED denylisted blob' line above; those " + - "lines and that file are the record of what this project's dataset does not contain. " + + "lines and that list are the record of what this project's dataset does not contain. " + "The files are absent from the tokenized repository, not present as raw source, so " + "they produce no blame and no dataset row. This is not a failure and does not affect " + "the exit status." diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index f686f8953..a9de86d4b 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -1014,7 +1014,7 @@ final class Walker( "row, and the file is absent from the tokenized repository rather than present as raw " + "source. The exclusion is deterministic and cited; the walk carries on and the run " + "still exits 0. See " + - s"${BlobDenylist.ResourcePath} in the blobExec jar for the list itself." + s"${BlobDenylist.EntriesSource} in blobExec for the list itself." ) } } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala index 66091588c..dc033b8bf 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala @@ -3,26 +3,23 @@ package cregit.blobexec import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import java.nio.file.Files - -/** The shipped file's contents are part of the contract, so they are asserted, not only parsed. */ +/** What a published dataset does NOT contain is part of its contract, so the shipped + * entries are asserted rather than trusted. */ class BlobDenylistSpec extends AnyFunSuite with Matchers { - // Class 1: the Java parser hang, srcML/srcML#2361. + // TestNewCastArray.java (4) and CheckErrorsForSource7.java (4): srcML/srcML#2361. private val JavaAnnotationShas = Set( - // TestNewCastArray.java "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65", "303c1a109f1fdc4b9d84e15111c56931202844ac", "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca", "c91924d1e87cb82d372a73b24579ccc34609efaa", - // CheckErrorsForSource7.java "56c8adf588e697a3e42b600b2e5ec54710592c61", "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5", "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7", "afbd81e5789ba060147b0de3be4f01759963299d" ) - // Class 2: the C parser hang on C++ in a `.h` (StatTimes.h). No upstream issue. + // StatTimes.h: the C parser on C++ in a `.h`. No upstream issue. private val CParserShas = Set( "5460d43f1884e3be230e02215151b9b963844a48", "720914b27e170b73b38ef662fb04239273dd0a34", @@ -30,109 +27,35 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { "dbb885e0bf046755c4fcfbddfc2454f90c550c62" ) - // Class 3: C/C++ parser crashes under `--position`. Checked by count and spot - // checks, not by duplicating 197 literals. - private val ParserCrashCount = 197 - - // crc32_small.c (position-only), a SIGABRT, and one blob vendored into two projects. - private val CrashPositionOnlySha = "4a62830c807a5a2aff73f474891c38ed913d39c4" - private val CrashAbortSha = "37cf5d5b1985a7510ad9c952897ef253231e80f7" - private val CrashSharedBlobSha = "012ea8148e8d21cf515a9ff47d4af719d049b276" - - private val KnownHangShas = JavaAnnotationShas ++ CParserShas - - test("the shipped list is on the classpath and holds the twelve hangs plus 197 crashes") { - BlobDenylist.shipped.size shouldEqual 12 + ParserCrashCount - JavaAnnotationShas.size shouldEqual 8 - CParserShas.size shouldEqual 4 - KnownHangShas.foreach { sha => - withClue(s"$sha is missing from the shipped denylist: ") { - BlobDenylist.shipped.entryFor(sha) should not be empty - } - } - } + private val shipped = BlobDenylistEntries.NonTerminating - test("class 3 holds exactly the 197 signal-death blobs, and no hang is counted among them") { - val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) - crashes.size shouldEqual ParserCrashCount - crashes.map(_.sha).toSet.size shouldEqual ParserCrashCount // no duplicates - crashes.map(_.sha).toSet intersect KnownHangShas shouldBe empty + test("the shipped list holds exactly the twelve known hangs, so an addition is deliberate") { + // Also catches a sha written twice: a Map literal silently keeps the last. + shipped.keySet shouldEqual JavaAnnotationShas ++ CParserShas + BlobDenylist.shipped.size shouldEqual 12 } - test("a class 3 entry says CRASH, names its signal, and disclaims #2361") { - val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) - crashes.foreach { e => - withClue(s"${e.sha}: ") { - e.citation should not include "2361" - e.reason should include("srcML 1.1.0") - e.reason should include("--position") - e.reason.toUpperCase should include("CRASH") - e.reason should (include("SIGSEGV (shell status 139)") or include("SIGABRT (shell status 134)")) - e.reason should include("token format carries line:col positions") - e.reason should include("not the non-termination") - } + test("every key is a 40-character lowercase hex git blob id") { + shipped.keys.foreach { sha => + withClue(s"$sha: ") { sha should fullyMatch regex "[0-9a-f]{40}" } } } - test("each class 3 entry records whether --position is the trigger, measured per blob") { - // silent-empty-sweep.md says --position is the trigger for all; it is not for 65. - val crashes = BlobDenylist.shipped.entries.filter(_.citation.contains("silent-empty-sweep.md")) - val positionOnly = crashes.count(_.reason.contains("--position IS the trigger")) - val eitherWay = crashes.count(_.reason.contains("--position is NOT the trigger")) - val noHelp = crashes.count(_.reason.contains("dropping --position would not recover")) - - positionOnly shouldEqual 132 - eitherWay shouldEqual 49 - noHelp shouldEqual 16 - positionOnly + eitherWay + noHelp shouldEqual ParserCrashCount - } - - test("the spot-checked class 3 blobs are present, with the property each was chosen for") { - // Source of the reproducer tests/t/fixtures/srcml-position-crash.c. - val posOnly = BlobDenylist.shipped.entryFor(CrashPositionOnlySha).getOrElse( - fail(s"$CrashPositionOnlySha (crc32_small.c) is missing from the shipped denylist")) - posOnly.reason should include("--position IS the trigger") - posOnly.reason should include("SIGSEGV (shell status 139)") - - val abort = BlobDenylist.shipped.entryFor(CrashAbortSha).getOrElse( - fail(s"$CrashAbortSha (util-linux swaplabel.c) is missing from the shipped denylist")) - abort.reason should include("SIGABRT (shell status 134)") - - BlobDenylist.shipped.entryFor(CrashSharedBlobSha) should not be empty - } - - test("every entry carries a one-line reason and a citation for its own defect") { - // Per class: citing #2361 for class 2 or 3 would be a false attribution. - BlobDenylist.shipped.entries.foreach { e => - withClue(s"${e.sha}: ") { - if (JavaAnnotationShas.contains(e.sha)) e.citation shouldEqual "srcML/srcML#2361" - else if (CParserShas.contains(e.sha)) e.citation should include("srcml-nontermination-analysis.md") - else e.citation should include("silent-empty-sweep.md") - e.reason should not be empty + test("every entry carries a one-line reason and the citation of its own defect") { + shipped.foreach { case (sha, e) => + withClue(s"$sha: ") { + if (JavaAnnotationShas.contains(sha)) e.citation shouldEqual "srcML/srcML#2361" + else e.citation should include("srcml-nontermination-analysis.md") e.reason should not include "\n" - e.reason.toLowerCase should include("srcml") + e.reason should include("srcML 1.1.0") } } } - test("every citation belongs to exactly one of the three classes, so none is unattributed") { - val byClass = BlobDenylist.shipped.entries.groupBy { e => - if (e.citation == "srcML/srcML#2361") "class1" - else if (e.citation.contains("srcml-nontermination-analysis.md")) "class2" - else if (e.citation.contains("silent-empty-sweep.md")) "class3" - else "UNATTRIBUTED: " + e.citation - } - byClass.keys.filter(_.startsWith("UNATTRIBUTED")) shouldBe empty - byClass("class1").size shouldEqual 8 - byClass("class2").size shouldEqual 4 - byClass("class3").size shouldEqual ParserCrashCount - } - test("the C parser entries do not claim to be srcML#2361, and carry their reproducer") { // With no upstream issue, the reproducer in the reason is the evidence. CParserShas.foreach { sha => - val e = BlobDenylist.shipped.entryFor(sha).getOrElse( - fail(s"the C parser blob $sha is missing from the shipped denylist")) + val e = BlobDenylist.shipped.entryFor(sha).getOrElse(fail(s"$sha is not denylisted")) withClue(s"$sha: ") { e.citation should not include "2361" e.reason should include("namespace n{struct T&f(const struct S);}") @@ -141,85 +64,20 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("lookup is by git blob id, in either case, and misses everything else") { - val sha = KnownHangShas.head - BlobDenylist.shipped.entryFor(sha).map(_.sha) shouldEqual Some(sha) - BlobDenylist.shipped.entryFor(sha.toUpperCase).map(_.sha) shouldEqual Some(sha) + val sha = JavaAnnotationShas.head + BlobDenylist.shipped.entryFor(sha) should not be empty + BlobDenylist.shipped.entryFor(sha.toUpperCase) should not be empty BlobDenylist.shipped.entryFor("0" * 40) shouldEqual None BlobDenylist.shipped.entryFor("") shouldEqual None } - test("an empty list matches nothing, so a test fixture cannot exclude by accident") { - BlobDenylist.empty.isEmpty shouldBe true - BlobDenylist.empty.entryFor(KnownHangShas.head) shouldEqual None - } - - // -- the parser -------------------------------------------------------------- - - private val sha1 = "a" * 40 - private val sha2 = "b" * 40 - - test("comments, blank lines and surrounding whitespace are ignored") { - val list = BlobDenylist.parse( - Vector( - "# a comment", - "", - " ", - s" $sha1\tsrcML/srcML#2361\tnon-termination ", - s"$sha2\tsrcML/srcML#2361\tnon-termination\t" - ), - "fixture") - list.size shouldEqual 2 - list.entryFor(sha1).map(_.reason) shouldEqual Some("non-termination") - } - - test("a line that is not three fields is refused, naming the line") { - val bad = intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector(s"$sha1\tsrcML/srcML#2361"), "fixture") - } - bad.getMessage should include("fixture:1") - intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector(s"$sha1\t\treason"), "fixture") // empty citation - } - intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector(s"$sha1\tcitation\t"), "fixture") // empty reason - } + test("a fixture list built from uppercase keys still matches lowercase ids") { + val entry = BlobDenylist.Entry("c", "r") + BlobDenylist(Map("AB" * 20 -> entry)).entryFor("ab" * 20) shouldEqual Some(entry) } - test("a key that is not a 40-hex git blob sha is refused") { - intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector(s"${"a" * 39}\tc\tr"), "fixture") - } - intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector(s"${"z" * 40}\tc\tr"), "fixture") - } - intercept[IllegalArgumentException] { - BlobDenylist.parse(Vector("TestNewCastArray.java\tc\tr"), "fixture") - } - } - - test("a duplicated sha is refused: two reasons for one blob means one is wrong") { - val bad = intercept[IllegalArgumentException] { - BlobDenylist.parse( - Vector(s"$sha1\tc\tfirst reason", s"${sha1.toUpperCase}\tc\tsecond reason"), - "fixture") - } - bad.getMessage should include("twice") - } - - test("the same parser reads a file, which is how a fixture list is built") { - val f = Files.createTempFile("denylist-", ".tsv") - try { - Files.writeString(f, s"# header\n$sha1\tsrcML/srcML#2361\tnon-termination\n") - val list = BlobDenylist.fromFile(f) - list.shas shouldEqual Set(sha1) - list.entryFor(sha1).map(_.describe) shouldEqual - Some("non-termination [srcML/srcML#2361]") - } finally Files.deleteIfExists(f) - } - - test("a missing resource is fatal rather than an empty list") { - intercept[IllegalStateException] { - BlobDenylist.fromResource("/cregit/blobexec/no-such-denylist.tsv") - } + test("an empty list matches nothing, so a test fixture cannot exclude by accident") { + BlobDenylist.empty.isEmpty shouldBe true + BlobDenylist.empty.entryFor(JavaAnnotationShas.head) shouldEqual None } } diff --git a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala index 293d25409..7b1ab37fe 100644 --- a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala @@ -826,9 +826,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } finally git.close() val denied = blobIdOf(hangs) - val denylist = BlobDenylist.parse( - Vector(s"${denied.name}\tsrcML/srcML#2361\tsrcML 1.1.0 does not terminate on it"), - "fixture") + val denylist = BlobDenylist(Map( + denied.name -> BlobDenylist.Entry("srcML/srcML#2361", "srcML 1.1.0 does not terminate on it"))) // Every invocation of the tokenizer records the blob it was given. "Excluded in // milliseconds, never handed to the tokenizer" is only checkable from the From becc72bc248ef68cea8f05432d38fc9fdfd3aba8 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 16:57:43 +0000 Subject: [PATCH 10/15] refactor(blobexec): run each blob in ChildRunner The merge replaced master's ChildRunner with deploy's GNU `timeout` wrapper. Bring ChildRunner back: the JVM holds the wall-clock budget and kills the whole process tree, so no PATH lookup, no warning when `timeout` is missing, and no 124/137 status set. #100's outcomes are unchanged. A killed child (budget spent, or a pipe still open after exit) calls onTimeout and is excluded with reason=timeout. Exit 33, which the tokenizer also reports for a srcML signal death, and exit 0 with no output call onParserCrash. The failed-blob set in Walker is untouched. TokenizerWorkerPool now returns ChildRunner.Outcome, waits ChildRunner.maxLifetimeSeconds for a wedged worker and kills it with ChildRunner.killTree. The stall floor of 3fbcb84 is based on ChildRunner.maxLifetimeSeconds again, as in 634be5f and 1e22e9d; the refusal and the 600/1800 defaults stay. BlobExecSpec gets back master's three ChildRunner tests. --- .../main/scala/cregit/blobexec/BlobExec.scala | 228 ++++-------------- .../scala/cregit/blobexec/ChildRunner.scala | 114 +++++++++ .../src/main/scala/cregit/blobexec/Main.scala | 4 +- .../scala/cregit/blobexec/Saturating.scala | 11 + .../cregit/blobexec/TokenizerWorkerPool.scala | 36 +-- .../main/scala/cregit/blobexec/Walker.scala | 2 +- .../scala/cregit/blobexec/BlobExecSpec.scala | 66 +++-- .../cregit/blobexec/MainOptionsSpec.scala | 6 +- .../blobexec/TokenizerWorkerPoolSpec.scala | 13 +- 9 files changed, 246 insertions(+), 234 deletions(-) create mode 100644 blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala create mode 100644 blobExec/src/main/scala/cregit/blobexec/Saturating.scala diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala index 39962351c..44bd809ed 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala @@ -3,11 +3,7 @@ package cregit.blobexec import org.eclipse.jgit.lib.Constants.OBJ_BLOB import org.eclipse.jgit.lib.{ObjectId, ObjectInserter} -import java.io.{InputStream, OutputStream} -import java.nio.charset.StandardCharsets import java.util.{Arrays => JavaArrays} -import scala.io.Source -import scala.sys.process.{Process, ProcessIO} /** Runs a per-blob command: original bytes on stdin, replacement on stdout, env * `BFG_BLOB`, `BFG_PATH` and `BFG_FILENAME` (the basename, which `tokenBySha.pl` @@ -17,48 +13,9 @@ object BlobExec { /** Per-blob wall-clock budget for the external command, in seconds. */ val DefaultTimeoutSeconds: Int = 600 - /** Reported for a killed child; negative, so no real exit status can equal it. */ - val TimeoutExitCode: Int = -1 - /** Must match `$PARSER_CRASH_EXIT` in `tokenize/tokenizeSrcMl.pl`. */ val ParserCrashExitCode: Int = 33 - private val NotExitedSentinel: Int = Int.MinValue - - /** Seconds GNU `timeout` waits between its SIGTERM and its SIGKILL (`-k`). */ - private val KillGraceSeconds: Int = 5 - - /** Backstop for the JVM latch, which fires only if `timeout` is missing or wedged. */ - private val BackstopSlackSeconds: Int = 25 - - /** Longest one [[invoke]] can take: the budget, the kill grace and the backstop. */ - private[blobexec] def maxChildLifetimeSeconds(timeoutSeconds: Int): Int = - math.min(math.max(1, timeoutSeconds).toLong + KillGraceSeconds + BackstopSlackSeconds, Int.MaxValue.toLong).toInt - - /** GNU `timeout`: 124 = budget expired, 137 = SIGKILL after `-k`. */ - private val TimeoutStatuses: Set[Int] = Set(124, 137) - - /** GNU `timeout` kills the child's whole process group; `Process.destroy()` would - * kill only `tokenBySha.pl` and leave `srcml` holding our pipes open. */ - private lazy val gnuTimeout: Option[String] = { - val found = sys.env - .getOrElse("PATH", "") - .split(java.io.File.pathSeparatorChar) - .iterator - .filter(_.nonEmpty) - .map(dir => java.nio.file.Paths.get(dir, "timeout")) - .find(java.nio.file.Files.isExecutable) - .map(_.toString) - if (found.isEmpty) { - System.err.println( - "blobExec: warning: GNU `timeout` was not found on PATH. Per-blob budgets " + - "still apply, but a killed child's grandchildren can survive and hold the " + - "pipe open. Install coreutils to get process-group kills." - ) - } - found - } - sealed trait Outcome object Outcome { case object Skip extends Outcome @@ -80,150 +37,54 @@ object BlobExec { onParserCrash: () => Unit = () => (), workerPool: Option[TokenizerWorkerPool] = None ): Outcome = { - val (exitCode, stdout, stderr) = workerPool match { + val env = Seq("BFG_BLOB" -> origSha, "BFG_FILENAME" -> filename, "BFG_PATH" -> fullPath) + val ran = workerPool match { case Some(pool) => pool.invoke(bytes, filename, timeoutSeconds) - case None => invoke(bytes, origSha, filename, fullPath, command, timeoutSeconds) + case None => new ChildRunner(timeoutSeconds).run(command, bytes, env) } - if (exitCode == TimeoutExitCode) { - // Not routed through `abortOnError`: one wedged blob must not abort a long run. - System.err.println( - s"Warning: command [$command] timed out after ${timeoutSeconds}s on blob $origSha " + - s"at path [$fullPath]: child killed, blob excluded" - ) - onTimeout() - Outcome.Skip - } else if (exitCode == ParserCrashExitCode) { - // Skip like a timeout, but counted apart: more time never fixes a crash. - System.err.println( - s"Warning: command [$command] reported a parser crash (exit $exitCode) on blob $origSha " + - s"at path [$fullPath]: srcML died or produced no tokens, so this blob is excluded " + - "rather than written as an empty tokenization" - ) - if (stderr.nonEmpty) { - System.err.println(s"--- stderr from $command on $origSha ($fullPath) ---") - System.err.print(stderr) - if (!stderr.endsWith("\n")) System.err.println() - System.err.println("--- end stderr ---") - } - onParserCrash() - Outcome.Skip - } else if (exitCode != 0) { - logError(command, origSha, fullPath, exitCode, stderr) - if (abortOnError) Outcome.Abort(stderr, exitCode) else Outcome.Skip - } else if (stdout.isEmpty && bytes.nonEmpty) { - // Backstop for any tokenizer that exits 0 with no output: never insert it. - System.err.println( - s"Warning: command [$command] exited 0 but produced no output for the ${bytes.length}-byte " + - s"blob $origSha at path [$fullPath]. Refusing to write an empty tokenization: counting " + - "this as a parser crash and excluding the blob." - ) - onParserCrash() - Outcome.Skip - } else if (JavaArrays.equals(bytes, stdout)) { - Outcome.Skip - } else { - Outcome.Replace(inserter.insert(OBJ_BLOB, stdout)) + ran match { + case ChildRunner.Outcome.Killed(why) => + // Not routed through `abortOnError`: one wedged blob must not abort a long run. + System.err.println( + s"Warning: command [$command] timed out on blob $origSha at path [$fullPath] ($why): " + + "blob excluded" + ) + onTimeout() + Outcome.Skip + + case ChildRunner.Outcome.Exited(ParserCrashExitCode, _, stderr) => + // Skip like a timeout, but counted apart: more time never fixes a crash. + System.err.println( + s"Warning: command [$command] reported a parser crash (exit $ParserCrashExitCode) on blob " + + s"$origSha at path [$fullPath]: srcML died or produced no tokens, so this blob is excluded " + + "rather than written as an empty tokenization" + ) + printStderr(command, origSha, fullPath, stderr) + onParserCrash() + Outcome.Skip + + case ChildRunner.Outcome.Exited(status, _, stderr) if status != 0 => + logError(command, origSha, fullPath, status, stderr) + if (abortOnError) Outcome.Abort(stderr, status) else Outcome.Skip + + case ChildRunner.Outcome.Exited(_, stdout, _) if stdout.isEmpty && bytes.nonEmpty => + System.err.println( + s"Warning: command [$command] exited 0 but produced no output for the ${bytes.length}-byte " + + s"blob $origSha at path [$fullPath]. Refusing to write an empty tokenization: counting " + + "this as a parser crash and excluding the blob." + ) + onParserCrash() + Outcome.Skip + + case ChildRunner.Outcome.Exited(_, stdout, _) if JavaArrays.equals(bytes, stdout) => + Outcome.Skip + + case ChildRunner.Outcome.Exited(_, stdout, _) => + Outcome.Replace(inserter.insert(OBJ_BLOB, stdout)) } } - /** Returns (exit, stdout, stderr), or [[TimeoutExitCode]] and no output if killed. */ - private[blobexec] def invoke( - bytes: Array[Byte], - origSha: String, - filename: String, - fullPath: String, - command: String, - timeoutSeconds: Int = DefaultTimeoutSeconds - ): (Int, Array[Byte], String) = { - val stdoutBuilder = new java.io.ByteArrayOutputStream(math.max(bytes.length, 1024)) - val stderrBuilder = new StringBuilder - - val readStdout: InputStream => Unit = in => { - try transfer(in, stdoutBuilder) finally in.close() - } - val writeStdin: OutputStream => Unit = out => { - try { out.write(bytes); out.flush() } finally out.close() - } - val readStderr: InputStream => Unit = err => { - val src = Source.fromInputStream(err, StandardCharsets.UTF_8.name) - try stderrBuilder.append(src.mkString) finally src.close() - } - - // Daemon readers: on timeout they are abandoned while blocked and must not - // keep the JVM alive. - val io = new ProcessIO(writeStdin, readStdout, readStderr, daemonizeThreads = true) - - val secs = math.max(1, timeoutSeconds) - val argv = gnuTimeout match { - case Some(bin) => Seq(bin, "-k", KillGraceSeconds.toString, secs.toString, command) - case None => Seq(command) - } - val groupKilled = gnuTimeout.isDefined - val pb = Process( - argv, - None, - "BFG_BLOB" -> origSha, - "BFG_FILENAME" -> filename, - "BFG_PATH" -> fullPath - ) - val proc = pb.run(io) - - // `exitValue()` joins the io threads, which a wedged child can block forever, - // so it runs on a daemon thread and the caller waits on a latch. - val finished = new java.util.concurrent.CountDownLatch(1) - val exitHolder = new java.util.concurrent.atomic.AtomicInteger(NotExitedSentinel) - val waiter = new Thread( - () => { - try exitHolder.set(proc.exitValue()) - catch { case _: InterruptedException => Thread.currentThread().interrupt() } - finally finished.countDown() - }, - s"blobexec-wait-$origSha" - ) - waiter.setDaemon(true) - waiter.start() - - val latchBudget = maxChildLifetimeSeconds(secs).toLong - - // On the timeout paths the io threads are abandoned, so the builders are racy: - // do not read them there. - if (!finished.await(latchBudget, java.util.concurrent.TimeUnit.SECONDS)) { - System.err.println( - s"blobExec: no exit from [$command] ${latchBudget}s after start on blob $origSha " + - s"($fullPath); destroying the direct child" + - (if (groupKilled) "" else " (no GNU timeout: grandchildren may survive)") - ) - proc.destroy() - (TimeoutExitCode, Array.emptyByteArray, "") - } else - exitHolder.get() match { - case NotExitedSentinel => - // Interrupted before any status: the result is unknown, so Skip. - System.err.println( - s"blobExec: wait for blob $origSha ($fullPath) was interrupted before the child " + - "reported a status; treating it as a timeout and discarding its output" - ) - (TimeoutExitCode, Array.emptyByteArray, "") - case code if groupKilled && TimeoutStatuses.contains(code) => - System.err.println( - s"blobExec: timeout after ${secs}s on blob $origSha ($fullPath); " + - s"child process group killed by `timeout` (status $code)" - ) - (TimeoutExitCode, Array.emptyByteArray, "") - case code => - (code, stdoutBuilder.toByteArray, stderrBuilder.toString) - } - } - - private def transfer(in: InputStream, out: java.io.OutputStream): Unit = { - val buf = new Array[Byte](8192) - Iterator - .continually(in.read(buf)) - .takeWhile(_ != -1) - .foreach(n => out.write(buf, 0, n)) - } - private def logError( command: String, origSha: String, @@ -234,11 +95,14 @@ object BlobExec { System.err.println( s"Warning: error executing command [$command] on blob $origSha at path [$fullPath]: exit code $exitCode" ) + printStderr(command, origSha, fullPath, stderr) + } + + private def printStderr(command: String, origSha: String, fullPath: String, stderr: String): Unit = if (stderr.nonEmpty) { System.err.println(s"--- stderr from $command on $origSha ($fullPath) ---") System.err.print(stderr) if (!stderr.endsWith("\n")) System.err.println() System.err.println("--- end stderr ---") } - } } diff --git a/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala b/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala new file mode 100644 index 000000000..ec96ff8be --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/ChildRunner.scala @@ -0,0 +1,114 @@ +package cregit.blobexec + +import java.io.{InputStream, OutputStream} +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.concurrent.{CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters._ + +/** One run of an external command under a wall-clock budget. Owns the child's + * environment, its pipes, its budget and its kill. */ +final class ChildRunner(budgetSeconds: Int, drainGraceSeconds: Int = ChildRunner.DrainGraceSeconds) { + + import ChildRunner._ + + private val budget: Int = math.max(1, budgetSeconds) + + def run(command: String, stdin: Array[Byte], env: Seq[(String, String)]): Outcome = { + val builder = new ProcessBuilder(command) + env.foreach { case (name, value) => builder.environment.put(name, value) } + val child = builder.start() + + val stdout = new java.io.ByteArrayOutputStream(math.max(stdin.length, StdoutBufferBytes)) + val stderr = new StringBuilder + val readers = new CountDownLatch(2) + + pump("stdin", () => writeThenClose(child.getOutputStream, stdin)) + pump("stdout", () => try copyBytes(child.getInputStream, stdout) finally readers.countDown()) + pump("stderr", () => try copyText(child.getErrorStream, stderr) finally readers.countDown()) + + val exited = + try child.waitFor(budget.toLong, TimeUnit.SECONDS) + catch { + case _: InterruptedException => + Thread.currentThread().interrupt() + false + } + + if (!exited) kill(child, s"no exit within ${budget}s") + else if (!readers.await(drainGraceSeconds.toLong, TimeUnit.SECONDS)) + // A surviving grandchild still holds a pipe, so the output is incomplete + // and the buffers are not safe to read. + kill(child, s"output still open ${drainGraceSeconds}s after exit") + else Outcome.Exited(child.exitValue, stdout.toByteArray, stderr.toString) + } + + private def kill(child: Process, why: String): Outcome = { + val killed = killTree(child) + Outcome.Killed(if (killed > 0) s"$why; killed $killed process(es)" else why) + } +} + +object ChildRunner { + + sealed trait Outcome + object Outcome { + final case class Exited(status: Int, stdout: Array[Byte], stderr: String) extends Outcome + + /** No result that can be trusted. Never carries output. */ + final case class Killed(why: String) extends Outcome + } + + /** Seconds an open pipe is waited for after the child itself has exited. */ + private[blobexec] val DrainGraceSeconds: Int = 5 + + /** Seconds a forced kill needs to take effect. */ + private val KillSettleSeconds: Int = 5 + + private val StdoutBufferBytes: Int = 1024 + + private[blobexec] def maxLifetimeSeconds(budgetSeconds: Int): Int = + Saturating.sum(math.max(1, budgetSeconds), DrainGraceSeconds + KillSettleSeconds) + + /** Kill the child and every process beneath it, and report how many were + * signalled. Signalling the direct child alone leaves grandchildren holding + * this JVM's pipes. */ + private[blobexec] def killTree(child: Process): Int = { + val handle = child.toHandle + // Snapshot before killing anything: a dead parent's children are reparented + // to init and leave the tree, but a handle taken now stays valid. + val tree = handle.descendants().iterator().asScala.toVector + val parent = if (handle.destroyForcibly()) 1 else 0 + val known = tree.count(_.destroyForcibly()) + val late = tree.flatMap(_.descendants().iterator().asScala).count(_.destroyForcibly()) + parent + known + late + } + + private def pump(name: String, body: () => Unit): Unit = { + val t = new Thread(() => body(), s"blobexec-$name") + t.setDaemon(true) // an abandoned reader must never keep the JVM alive + t.start() + } + + private def writeThenClose(out: OutputStream, bytes: Array[Byte]): Unit = + // A command that ignores its input breaks this pipe; that is not a failure. + try { out.write(bytes); out.flush() } + catch { case _: java.io.IOException => () } + finally closeQuietly(out) + + private def copyBytes(in: InputStream, out: java.io.ByteArrayOutputStream): Unit = + try { + val buf = new Array[Byte](8192) + Iterator.continually(in.read(buf)).takeWhile(_ != -1).foreach(n => out.write(buf, 0, n)) + } catch { case _: java.io.IOException => () } finally closeQuietly(in) + + private def copyText(in: InputStream, out: StringBuilder): Unit = + try { + val reader = new java.io.InputStreamReader(in, UTF_8) + val buf = new Array[Char](4096) + Iterator.continually(reader.read(buf)).takeWhile(_ != -1).foreach(n => out.appendAll(buf, 0, n)) + } catch { case _: java.io.IOException => () } finally closeQuietly(in) + + private def closeQuietly(c: java.io.Closeable): Unit = + try c.close() + catch { case _: java.io.IOException => () } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index ee618ca70..ad544cbd3 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -70,7 +70,7 @@ object Main { private[blobexec] def widenedStall(blobTimeoutSeconds: Int): Int = math.max( - math.min(math.max(1, blobTimeoutSeconds).toLong * StallTimeoutMultiple, Int.MaxValue.toLong).toInt, + Saturating.product(math.max(1, blobTimeoutSeconds), StallTimeoutMultiple), Walker.stallFloorFor(blobTimeoutSeconds) ) @@ -181,7 +181,7 @@ object Main { | --blob-timeout= | wall-clock budget for one invocation | (default ${BlobExec.DefaultTimeoutSeconds}). A child that exceeds it is - | killed (whole process group) and that blob is + | killed (whole process tree) and that blob is | excluded as failed (see below); the run continues. | --stall-timeout= | watchdog window (default ${Walker.DefaultStallTimeoutSeconds}); must clear one diff --git a/blobExec/src/main/scala/cregit/blobexec/Saturating.scala b/blobExec/src/main/scala/cregit/blobexec/Saturating.scala new file mode 100644 index 000000000..97fdd918e --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/Saturating.scala @@ -0,0 +1,11 @@ +package cregit.blobexec + +private[blobexec] object Saturating { + + def sum(a: Int, b: Int): Int = clampToInt(a.toLong + b.toLong) + + def product(a: Int, b: Int): Int = clampToInt(a.toLong * b.toLong) + + private def clampToInt(value: Long): Int = + math.min(math.max(value, 0L), Int.MaxValue.toLong).toInt +} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala index 47e178c86..ca705b786 100644 --- a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala @@ -23,27 +23,19 @@ final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoClose try (0 until size).foreach(addWorker) catch { case failure: Throwable => close(); throw failure } - /** Same result as [[BlobExec.invoke]]: (exit, stdout, stderr), and - * [[BlobExec.TimeoutExitCode]] for a blob that ran out of time. */ - def invoke( - bytes: Array[Byte], - filename: String, - timeoutSeconds: Int - ): (Int, Array[Byte], String) = { - val worker = idle.take() - val backstop = timeoutSeconds + BackstopSlackSeconds - within(backstop)(request(worker, bytes, filename, timeoutSeconds)) match { + def invoke(bytes: Array[Byte], filename: String, timeoutSeconds: Int): ChildRunner.Outcome = { + val worker = idle.take() + val backstop = ChildRunner.maxLifetimeSeconds(timeoutSeconds) + within(backstop.toLong)(request(worker, bytes, filename, timeoutSeconds)) match { case None => - System.err.println( - s"blobExec: tokenizer worker ${worker.index} gave no response within ${backstop}s; replacing it") replace(worker) - Timeout + ChildRunner.Outcome.Killed(s"tokenizer worker ${worker.index} gave no response within ${backstop}s; replaced it") case Some(Left(failure)) => replace(worker) - (1, Array.emptyByteArray, s"tokenizer worker ${worker.index} failed: $failure\n") + ChildRunner.Outcome.Exited(1, Array.emptyByteArray, s"tokenizer worker ${worker.index} failed: $failure\n") case Some(Right(response)) => if (worker.process.isAlive) idle.put(worker) else replace(worker) - if (response._1 == WorkerTimeoutExitCode) Timeout else response + response } } @@ -85,7 +77,9 @@ final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoClose readLine(worker.stdout).split(' ') match { case Array("RES", exit, outLength, errLength) => val out = readBytes(worker.stdout, outLength.toInt) - (exit.toInt, out, new String(readBytes(worker.stdout, errLength.toInt), UTF_8)) + val err = new String(readBytes(worker.stdout, errLength.toInt), UTF_8) + if (exit.toInt == WorkerTimeoutExitCode) ChildRunner.Outcome.Killed(s"no exit within ${timeoutSeconds}s") + else ChildRunner.Outcome.Exited(exit.toInt, out, err) case header => throw new IOException(s"malformed response header [${header.mkString(" ")}]") } } @@ -94,19 +88,11 @@ final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoClose object TokenizerWorkerPool { /** What a worker answers when it killed the tokenizer for running out of time. */ val WorkerTimeoutExitCode = 124 - private val Timeout = (BlobExec.TimeoutExitCode, Array.emptyByteArray, "") - /** The worker kills its own tokenizer at the budget. This only catches a wedged worker. */ - private val BackstopSlackSeconds = 5L private val ReadySeconds = 30L private val CloseSeconds = 2L private val MaxHeaderBytes = 8192 - /** Kill the children first: once the worker is gone they are no longer its descendants. */ - private def kill(process: Process): Unit = { - process.descendants().forEach(child => { child.destroyForcibly(); () }) - process.destroyForcibly() - () - } + private def kill(process: Process): Unit = { ChildRunner.killTree(process); () } /** Runs `read` on a daemon thread, so that a worker which never answers cannot block the caller. */ private def within[A](seconds: Long)(read: => A): Option[Either[Throwable, A]] = { diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index a9de86d4b..343abd825 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -1459,7 +1459,7 @@ object Walker { /** Smallest stall window that cannot fire while one blob is still inside its own * lifetime: a commit whose last blob is slow completes nothing until it ends. */ private[blobexec] def stallFloorFor(blobTimeoutSeconds: Int): Int = - math.min(BlobExec.maxChildLifetimeSeconds(blobTimeoutSeconds).toLong + StallFloorMarginSeconds, Int.MaxValue.toLong).toInt + Saturating.sum(ChildRunner.maxLifetimeSeconds(blobTimeoutSeconds), StallFloorMarginSeconds) /** Pure form of the watchdog's decision, so it can be tested without halting * a JVM: has more than `stallTimeoutSeconds` passed with no progress? */ diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala index 51c6606cb..dbef5ecb6 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala @@ -160,22 +160,15 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { timeouts.get shouldEqual 1 } - test("a child that never exits is killed at the timeout, not awaited forever") { + test("a child that never exits is killed at its budget, not awaited forever") { // `sleep 30` is the wedged srcml chain: it reads nothing, writes nothing, and // holds its stdout open so the reader parks in pipe_read. val cmd = shellScript("sleep 30") val started = System.currentTimeMillis() - val (exit, _, _) = BlobExec.invoke( - "irrelevant".getBytes(UTF_8), - "0" * 40, - "input.java", - "input.java", - cmd, - timeoutSeconds = 1 - ) + val outcome = new ChildRunner(1).run(cmd, "irrelevant".getBytes(UTF_8), Nil) val elapsed = System.currentTimeMillis() - started - assert(elapsed < 10000, s"invoke took ${elapsed}ms; the timeout did not fire") - exit shouldEqual BlobExec.TimeoutExitCode + assert(elapsed < 10000, s"run took ${elapsed}ms; the budget did not fire") + outcome shouldBe a[ChildRunner.Outcome.Killed] } test("a timed-out blob is skipped, never tokenized and never an abort") { @@ -188,16 +181,57 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { out shouldBe BlobExec.Outcome.Skip } - test("an orphan grandchild holding the pipes cannot outlive the timeout") { + test("an orphan grandchild holding the pipes cannot outlive the budget") { // The real process shape, tokenBySha.pl -> sh -> srcml: killing only the // direct child leaves the background `sleep` holding this JVM's pipes. val cmd = shellScript("sleep 30 & sleep 30") val started = System.currentTimeMillis() - val (exit, _, _) = BlobExec.invoke( - "irrelevant".getBytes(UTF_8), "0" * 40, "input.java", "input.java", cmd, timeoutSeconds = 1) + val outcome = new ChildRunner(1).run(cmd, "irrelevant".getBytes(UTF_8), Nil) val elapsed = System.currentTimeMillis() - started - assert(elapsed < 10000, s"invoke took ${elapsed}ms; the process group was not killed") - exit shouldEqual BlobExec.TimeoutExitCode + assert(elapsed < 10000, s"run took ${elapsed}ms; the descendant tree was not killed") + outcome shouldBe a[ChildRunner.Outcome.Killed] + } + + test("a descendant holding the pipe cannot park the call, nor forge output") { + // perl's fork() guarantees the child inherits fd 1. A shell's `&` does not + // reliably, which is why this does not use one. + val cmd = shellScript( + """exec perl -e 'if (fork() == 0) { sleep 10; exit 0 } print "partial"; exit 0'""") + val started = System.currentTimeMillis() + val outcome = new ChildRunner(30, drainGraceSeconds = 1).run(cmd, Array.emptyByteArray, Nil) + val elapsed = System.currentTimeMillis() - started + + // The holder lives 10s. Either branch below is safe, and neither may wait for it. + assert(elapsed < 5000, s"run took ${elapsed}ms: it waited for the descendant") + outcome match { + // EOF was reached, so the buffer holds everything the child wrote. + case ChildRunner.Outcome.Exited(status, stdout, _) => + status shouldEqual 0 + new String(stdout, UTF_8) shouldEqual "partial" + // The reader was still blocked, so no output is offered at all. + case ChildRunner.Outcome.Killed(why) => + why should include("still open") + } + } + + test("a blob whose command leaves a descendant on the pipe is never an abort") { + val cmd = shellScript( + """exec perl -e 'if (fork() == 0) { sleep 10; exit 0 } print "partial"; exit 0'""") + val started = System.currentTimeMillis() + val outcome = BlobExec.run("original".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, + abortOnError = true, inserter, timeoutSeconds = 30) + assert(System.currentTimeMillis() - started < 20000, "the walk waited for the descendant") + outcome should not be a[BlobExec.Outcome.Abort] + } + + test("the environment reaches the child, and the exit status comes back") { + val cmd = shellScript("""printf '%s' "$BFG_PATH"; exit 3""") + inside(new ChildRunner(30).run(cmd, Array.emptyByteArray, Seq("BFG_PATH" -> "src/x.c"))) { + case ChildRunner.Outcome.Exited(status, stdout, _) => + status shouldEqual 3 + new String(stdout, UTF_8) shouldEqual "src/x.c" + case other => fail(s"expected Exited, got $other") + } } test("a timeout is reported to the caller exactly once, and only on a timeout") { diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index c23ad4615..60ac75ae9 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -54,7 +54,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("the floor is the child's whole lifetime, not just the budget") { - Walker.stallFloorFor(600) should be > BlobExec.maxChildLifetimeSeconds(600) + Walker.stallFloorFor(600) should be > ChildRunner.maxLifetimeSeconds(600) Main.resolveStallTimeout(600, 601, stallExplicit = true).isLeft shouldBe true Main.resolveStallTimeout(600, 630, stallExplicit = true).isLeft shouldBe true } @@ -67,7 +67,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("a small --blob-timeout no longer lets the watchdog stop a healthy run") { - val oneBlobLifetimeNanos = BlobExec.maxChildLifetimeSeconds(5).toLong * 1000000000L + val oneBlobLifetimeNanos = ChildRunner.maxLifetimeSeconds(5).toLong * 1000000000L Main.resolveStallTimeout(5, 15, stallExplicit = true).isLeft shouldBe true val window = Main.resolveStallTimeout(5, 3, stallExplicit = false).getOrElse(0) Walker.isStalled(oneBlobLifetimeNanos, 0L, window) shouldBe false @@ -76,7 +76,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { test("the floor cannot overflow into a guard that always passes") { Walker.stallFloorFor(Int.MaxValue) should be > 0 - BlobExec.maxChildLifetimeSeconds(Int.MaxValue) should be > 0 + ChildRunner.maxLifetimeSeconds(Int.MaxValue) should be > 0 } test("the defaults satisfy the relationship") { diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala index 4921b3fbb..9a35e4201 100644 --- a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala @@ -59,9 +59,10 @@ class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAf } private def stdout(pool: TokenizerWorkerPool, filename: String, body: String, timeoutSeconds: Int = 2) = { - val (exit, out, _) = pool.invoke(body.getBytes(UTF_8), filename, timeoutSeconds) - exit shouldEqual 0 - new String(out, UTF_8) + pool.invoke(body.getBytes(UTF_8), filename, timeoutSeconds) match { + case ChildRunner.Outcome.Exited(0, out, _) => new String(out, UTF_8) + case other => fail(s"expected exit 0, got $other") + } } test("returns the worker's output and sends each request's own budget") { @@ -93,7 +94,7 @@ class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAf test("kills a wedged worker with its children, replaces it, and serves the next request") { withPool(1) { pool => val started = System.nanoTime() - pool.invoke("x".getBytes(UTF_8), "hang.c", 1)._1 shouldEqual BlobExec.TimeoutExitCode + pool.invoke("x".getBytes(UTF_8), "hang.c", 1) shouldBe a[ChildRunner.Outcome.Killed] (System.nanoTime() - started).nanos should be < 15.seconds eventuallyStopped(Seq(Files.readString(grandchildPidFile).trim.toLong)) shouldBe true stdout(pool, "next.c", "next") shouldEqual "NEXT" @@ -102,7 +103,9 @@ class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAf test("a worker that dies in a request fails that blob only") { withPool(1) { pool => - pool.invoke("x".getBytes(UTF_8), "die.c", 2)._1 should not equal 0 + pool.invoke("x".getBytes(UTF_8), "die.c", 2) should not matchPattern { + case ChildRunner.Outcome.Exited(0, _, _) => + } stdout(pool, "ok.c", "ok") shouldEqual "OK" } } From ca3888b6938fcd42bde02ca27c5a227ea48c1696 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 17:08:07 +0000 Subject: [PATCH 11/15] refactor(blobexec): parse options into one value Review of #102 asked for a Main that reads in one pass. main() was one 390-line method with 13 vars and a flag loop that called sys.exit from inside each case. - parse() returns Either[String, Options]: every flag is a `case Flag(value)` through one flagValue helper, the combination rules are a table checked in the old order, and main() exits once on a Left. - The stderr texts live in a Messages object; their meaning is kept. - openMapping() turns Mapping.open's refusals into one Left, so src and dst are closed and the status is chosen in a single place. - run(), walk(), startLine(), doneLine() and reportExclusions() replace the inline steps. The start and done lines are byte-identical. - The usage text goes from 163 lines to 35: each option is described in at most 3 lines, most in 1. A test pins that. --retokenize, --mask-widened and every other flag behave as before. --- .../src/main/scala/cregit/blobexec/Main.scala | 963 ++++++++---------- .../cregit/blobexec/MainOptionsSpec.scala | 77 +- 2 files changed, 476 insertions(+), 564 deletions(-) diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index ad544cbd3..96fe44408 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -1,51 +1,31 @@ package cregit.blobexec import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.{Constants, ObjectId} import org.eclipse.jgit.storage.file.FileRepositoryBuilder -import java.nio.file.{Files, Paths} - -/** - * From-scratch rewriter for the cregit pipeline (step 2). Replaces the - * previous in-place BFG-based rewrite with a jgit walk that builds a new - * destination repo and persists `(orig → new)` mappings to SQLite, so a - * subsequent invocation can resume incrementally. - * - * blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] - * - * --abort-on-error exit immediately (status 2) on the first non-zero - * exit from , instead of skipping that blob - * path to the bare source repo (read-only) - * path to the bare destination repo (created if missing) - * path to the SQLite mapping file (created if missing) - * absolute path to the per-blob script to run - * regex matched against each blob's filename - * - * For each blob whose filename matches `fileMaskRegex`, `command` is invoked - * with the blob bytes on stdin and env vars `BFG_BLOB` (orig sha) + - * `BFG_FILENAME`. Its stdout becomes the new blob. Non-zero exit or - * identical-output leaves the blob unchanged. The new commit message gets - * `Former-commit-id: ` appended (BFG-compatible). - */ +import java.nio.file.{Files, Path, Paths} +import scala.util.Try + +/** Step 2 of the cregit pipeline: rewrites into through the + * per-blob , and records (orig -> new) ids in SQLite so a rerun resumes. */ object Main { - /** Exit status when `--retokenize` would invalidate nothing. Not 3 (memo mismatch): - * a no-op request must be told apart from a refusal, and never read as success. */ + /** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */ private[blobexec] val RetokenizeIneffectiveExitStatus = 7 + private val UsageExitStatus = 1 + private val RefusedExitStatus = 3 + /** 2 on an abort, else 0: excluded blobs are named on EXCLUDED lines, not failed. */ private[blobexec] def exitStatus(stats: WalkStats): Int = if (stats.aborted) 2 else 0 - /** Value parser for the `--blob-timeout=` / `--stall-timeout=` seconds: a - * positive whole number, else None (which the caller reports and exits 1 on). - * Zero and negatives are rejected rather than read as "no limit" — an - * unbounded blob is the defect this whole change exists to remove. */ + /** Zero and negatives are refused rather than read as "no limit". */ private[blobexec] def parsePositiveSeconds(spec: String): Option[Int] = spec.toIntOption.filter(_ > 0) - /** Multiple of `--blob-timeout` used when the stall window has to be widened - * for it, matching the ratio of the two defaults (600 and 1800). */ + /** Ratio of the two defaults (600 and 1800). */ private[blobexec] val StallTimeoutMultiple = 3 /** An explicit window below the floor is refused; a defaulted one is widened to @@ -74,565 +54,432 @@ object Main { Walker.stallFloorFor(blobTimeoutSeconds) ) - // `raw` (not `s`): the mask example below contains a regex backslash, which a - // processed-escape interpolator rejects. `$$` therefore renders a literal `$`. - private val Usage = - raw"""Usage: blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] [--mask-widened] [--tokenizer-identity==,...] [--retokenize=,...] [--memo-dir=] [--tokenizer-worker=] [--blob-timeout=] [--stall-timeout=] - | - | --abort-on-error exit immediately (status 2) on the first non-zero - | exit from , instead of skipping that blob - | --mask-widened resume against a blob map recorded under a DIFFERENT - | , keeping the tokenizations already in - | blob_map. Without it a mask change is refused (status - | 3), which is the right default and stays the default. - | Valid only because the mask decides WHICH blobs are - | tokenized, never HOW: the language comes from the - | file's extension, per file, so the same (blob, path) - | yields the same tokens under any mask that selects it. - | - | It is not taken on trust. Two checks run first, and - | NOTHING is written until both pass: - | 1. every path of a tokenized row (orig <> new) must - | still match the new mask. One regex test per row, - | against the data, so a narrowing mislabelled as a - | widening is caught and named. Regexes are never - | compared to each other. - | 2. a spread sample of the retained new_blob ids must - | resolve in . Those ids exist only there, - | so the map is reusable only alongside it. - | - | Then: tree_map, commit_map and ref_map are emptied - | (trees gain entries, commits name trees, refs name - | commits), and so are blob_map's IDENTITY rows - | (orig == new). The identity rows are the subtle part: - | they record "this path was not selected, its bytes pass - | through", and under a wider mask some of those paths - | ARE selected. Keeping them would serve RAW SOURCE as a - | cache hit for precisely the files the widening exists - | to tokenize. - | - | This is a step-2 resume that KEEPS the work directory, - | never a fresh run — a fresh run deletes dst.git, and - | check 2 then refuses. - | --tokenizer-identity==[,=...] - | which tokenizer produced the tokens for each file - | extension, as an opaque value per extension (a digest - | of the parser toolchain; tokenize/tokenizerIdentity.pl - | computes it). Recorded in the blob map's meta table on - | first sight and COMPARED on every later run. - | - | This closes a hole that could not. - | is the constant path tokenizeByBlobId/tokenBySha.pl, so - | when a tokenizer behind it is corrected, nothing the - | reuse decision looks at changes: every cached row for - | that language stays a cache hit and the run reproduces - | the OLD tokenizer's output with no error anywhere. That - | happened — a rustTokenizer binary 16 days older than - | its source kept emitting a `line:col` prefix, and - | 741,869 .rs entries in 45 projects carried it. - | - | A mismatch REFUSES the run (status 3) and names the - | flag to fix it. It never invalidates anything by - | itself, which is why comparing is safe as a default: - | the 186 published projects are not touched. An - | extension with nothing recorded is not a mismatch — - | that is what every blob map built before this flag - | looks like — so it is simply recorded. - | --retokenize=[,...] - | the opt-in past that refusal: invalidate the cached - | tokenizations of these extensions and nothing else. - | Requires --tokenizer-identity and --memo-dir, and a - | step-2 resume (the ids live in ). - | - | SELECTIVE on purpose. Re-tokenizing is 88% of total - | pipeline time, so a defect in one language's tokenizer - | must cost one language's entries. Deliberately NOT - | --drop-memo, which destroys 2.6 M memoized - | tokenizations wholesale, and deliberately not - | --mask-widened, which answers a different question. - | - | BOTH cache layers go, together, and neither can be - | done without the other: - | - blob_map's tokenized rows on those extensions, and - | - their entries in the memo under --memo-dir. The - | memo is keyed on sha1(contents) ALONE - | (tokenBySha.pl:76) with no tokenizer in the key, so - | dropping only the blob_map row just moves the stale - | answer one layer down. - | tree_map, commit_map and ref_map go too: a tree names - | its blobs, and a retained tree_map row short-circuits - | the re-walk of the subtree holding the file. + // `raw`: the mask example holds a regex backslash; `$$` renders a literal `$`. + private[blobexec] val Usage = + raw"""Usage: blobExec [options] | - | IT CANNOT QUIETLY DO NOTHING. It refuses, before - | changing anything, when: another extension's tokenizer - | also changed and was not named; no tokenized row - | carries any named extension (status ${RetokenizeIneffectiveExitStatus}); the memo held - | none of the affected blobs and --memo-dir is not - | $$BFG_MEMO_DIR, the memo tokenBySha.pl reads (status ${RetokenizeIneffectiveExitStatus}); - | or a RETAINED new_blob id does not resolve in . - | --memo-dir= the memo directory ($$BFG_MEMO_DIR) whose entries - | --retokenize must purge. Only read with --retokenize; - | passing it alone is an error rather than a no-op. - | --tokenizer-worker= - | send each blob to one of a pool of persistent - | tokenizer processes (tokenizeByBlobId/tokenWorker.pl) - | instead of running once per blob. Its - | output is the same. Needs --pipeline or --pipeline-trees. - | --blob-timeout= - | wall-clock budget for one invocation - | (default ${BlobExec.DefaultTimeoutSeconds}). A child that exceeds it is - | killed (whole process tree) and that blob is - | excluded as failed (see below); the run continues. - | --stall-timeout= - | watchdog window (default ${Walker.DefaultStallTimeoutSeconds}); must clear one - | blob's whole lifetime (--blob-timeout plus the kill - | path), since a pure-blob commit's only progress is a - | blob finishing or being killed. If it is not, an - | explicit value is refused and a defaulted one is raised - | to ${StallTimeoutMultiple}x --blob-timeout or that floor. If no blob, tree, - | commit or blob copy completes anywhere in this window, - | the run is stuck in a way the per-blob kill did not - | cover: it is reported with the work in flight and the - | process is killed with status ${Walker.StalledExitStatus}. The memo is - | durable, so re-running resumes. + | --abort-on-error exit 2 on the first error from instead of skipping the blob + | --pipeline look-ahead pipelined walker; same output as the serial walker + | --pipeline-trees as --pipeline, and also assemble trees on the worker pool + | --shard=K/N tree-only run of shard K (0-based) of N commit ranges, no commits or + | refs (merge with shard_merge.py); not with --pipeline(-trees) + | --warm= read-only fallback mapping DB, consulted on a blob_map/tree_map miss + | --mask-widened resume under a wider , keeping blob_map's tokens; + | refused (status 3) unless every tokenized path still matches the + | new mask and holds the kept ids + | --tokenizer-identity==[,...] + | tokenizer digest per extension (tokenize/tokenizerIdentity.pl); a + | change since the cache was built refuses the run (status 3) + | --retokenize=[,...] drop these extensions' cached tokens (blob_map and memo); needs + | --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus}) + | --memo-dir= the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges + | --tokenizer-worker= tokenize through a pool of persistent workers; needs --pipeline(-trees) + | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process + | tree is killed and the blob excluded + | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); + | must cover one blob's lifetime: below it is refused, a default raised + | bare source repository (read-only) + | bare destination repository (created if missing, reused on resume) + | SQLite mapping file (created if missing, reused on resume) + | absolute path of the per-blob command + | regex matched against each blob's filename (e.g. '\.[ch]$$') | - | Blobs on the shipped denylist (${BlobDenylist.EntriesSource}) are never - | handed to : they are dropped from the rewritten trees, counted as - | blobsDenylisted, named with their reason and citation on an EXCLUDED line, - | and they do NOT change the exit status. srcML 1.1.0 does not terminate on - | the listed blobs, so the list saves the --blob-timeout each run would spend. + | Denylisted (${BlobDenylist.EntriesSource}) and oversized blobs, and blobs whose tokenizer + | times out or crashes, are left out of the trees and named on an EXCLUDED line; they do not + | change the exit status. | - | A blob whose tokenizer times out or reports a parser crash is excluded - | the same way: dropped from the rewritten trees, counted as blobsTimedOut - | or blobsParserCrashed, named on an 'EXCLUDED failed blob' line, and tried - | only once per run. It does NOT change the exit status. - | - | Exit status: 0 = clean, 1 = usage, 2 = aborted on a command error, - | 3 = memo meta mismatch (including a changed tokenizer with - | no --retokenize), ${Walker.StalledExitStatus} = killed by the stall watchdog, - | ${RetokenizeIneffectiveExitStatus} = --retokenize would have invalidated nothing. - | --pipeline use the look-ahead pipelined walker (producer runs - | ahead so the blob-command pool stays saturated); - | output is identical to the default serial walker - | --pipeline-trees as --pipeline, but also assembles rewritten trees on - | the worker pool (Design B: only commit construction - | and the ordered DB write stay on the consumer); - | output is identical to the default serial walker - | --shard=K/N TREE-ONLY history shard: partition the commits (in - | canonical TOPO+REVERSE order) into N contiguous - | ranges and process only shard K (0-based). Tokenizes - | blobs and builds+persists trees (blob_map + tree_map - | incl. subtree ids) for the slice, but does NOT fold - | commits (no commit_map, no refs). Run N shards to - | their own dst.git + db.sqlite, then merge + serial - | re-fold (shard_merge.py) for byte-identical output. - | Uses the flat-memory serial walker; mutually - | exclusive with --pipeline / --pipeline-trees. - | --warm= optional read-only fallback DB (a frozen prior memo, - | e.g. the paused whole-kernel run). On a blob_map / - | tree_map miss the lookup falls through to this DB, so - | a shard skips re-tokenizing content already done. - | Never written; commit_map is never consulted. - | bare source repo (read-only) - | bare destination repo (created on first run, reused on incremental) - | SQLite mapping file (created on first run, reused on incremental) - | absolute path to the per-blob script to run - | regex matched against each blob's filename (e.g. '\.[ch]$$') + | Exit status: 0 clean, 1 usage, 2 aborted on a command error, 3 cache mismatch, + | ${Walker.StalledExitStatus} stalled, ${RetokenizeIneffectiveExitStatus} --retokenize would invalidate nothing. |""".stripMargin - def main(args: Array[String]): Unit = { - val (flags, positional) = args.partition(_.startsWith("-")) + private[blobexec] final case class Options( + abortOnError: Boolean = false, + pipeline: Boolean = false, + pipelineTrees: Boolean = false, + shard: Option[(Int, Int)] = None, + warm: Option[Path] = None, + maskWidened: Boolean = false, + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenize: Set[String] = Set.empty, + memoDir: Option[Path] = None, + tokenizerWorker: Option[Path] = None, + blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, + stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, + stallExplicit: Boolean = false, + positional: Vector[String] = Vector.empty + ) { + def src: Path = Paths.get(positional(0)) + def dst: Path = Paths.get(positional(1)) + def db: Path = Paths.get(positional(2)) + def command: String = positional(3) + def mask: String = positional(4) + + def stallWindow: Either[String, Int] = + resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) + + def anyPipeline: Boolean = pipeline || pipelineTrees + } - var abortOnError = false - var pipeline = false - var pipelineTrees = false - var shard: Option[(Int, Int)] = None - var warmPath: Option[java.nio.file.Path] = None - var blobTimeoutSeconds = BlobExec.DefaultTimeoutSeconds - var stallTimeoutSeconds = Walker.DefaultStallTimeoutSeconds - var stallExplicit = false - var maskWidened = false - var tokenizerIdentity = TokenizerIdentity.empty - var retokenizeExtensions: Set[String] = Set.empty - var memoDir: Option[java.nio.file.Path] = None - var tokenizerWorker: Option[java.nio.file.Path] = None - flags.foreach { - case "--abort-on-error" => abortOnError = true - case "--pipeline" => pipeline = true - case "--pipeline-trees" => pipelineTrees = true - case "--mask-widened" => maskWidened = true - case t if t.startsWith("--tokenizer-identity=") => - TokenizerIdentity.parse(t.stripPrefix("--tokenizer-identity=")) match { - case Right(id) => tokenizerIdentity = id - case Left(why) => - System.err.println(s"Error: --tokenizer-identity: $why") - sys.exit(1) - } - case t if t.startsWith("--retokenize=") => - TokenizerIdentity.parseExtensions(t.stripPrefix("--retokenize=")) match { - case Right(exts) => retokenizeExtensions = exts - case Left(why) => - System.err.println(s"Error: --retokenize: $why") - sys.exit(1) - } - case t if t.startsWith("--memo-dir=") => - val p = Paths.get(t.stripPrefix("--memo-dir=")) - if (!Files.isDirectory(p)) { - System.err.println( - s"Error: --memo-dir [$p] is not a directory. It must be the SAME memo this project's " + - "tokenizations were written into, or the invalidation would leave the real memo's " + - "stale entries in place.") - sys.exit(1) - } - memoDir = Some(p) - case s if s.startsWith("--shard=") => - val spec = s.stripPrefix("--shard=") - spec.split("/", -1) match { - case Array(kStr, nStr) => - val k = kStr.toIntOption.getOrElse(-1) - val n = nStr.toIntOption.getOrElse(-1) - if (n < 1 || k < 0 || k >= n) { - System.err.println(s"Error: --shard must be K/N with 0 <= K < N and N >= 1 [$spec]") - sys.exit(1) - } - shard = Some((k, n)) - case _ => - System.err.println(s"Error: --shard must be of the form K/N [$spec]") - sys.exit(1) - } - case w if w.startsWith("--warm=") => - val p = Paths.get(w.stripPrefix("--warm=")) - if (!Files.isRegularFile(p)) { - System.err.println(s"Error: --warm db [$p] is not a file") - sys.exit(1) - } - warmPath = Some(p) - case t if t.startsWith("--tokenizer-worker=") => - tokenizerWorker = Some(Paths.get(t.stripPrefix("--tokenizer-worker="))) - case t if t.startsWith("--blob-timeout=") => - val spec = t.stripPrefix("--blob-timeout=") - parsePositiveSeconds(spec) match { - case Some(secs) => blobTimeoutSeconds = secs - case None => - System.err.println(s"Error: --blob-timeout must be a positive whole number of seconds [$spec]") - sys.exit(1) - } - case t if t.startsWith("--stall-timeout=") => - val spec = t.stripPrefix("--stall-timeout=") - parsePositiveSeconds(spec) match { - case Some(secs) => stallTimeoutSeconds = secs; stallExplicit = true - case None => - System.err.println(s"Error: --stall-timeout must be a positive whole number of seconds [$spec]") - sys.exit(1) - } - case other => - System.err.println(s"Error: unknown flag [$other]") - System.err.println(Usage) - sys.exit(1) - } + private object Messages { + def withUsage(error: String): String = s"$error\n$Usage" + + def unknownFlag(flag: String): String = withUsage(s"Error: unknown flag [$flag]") + def badFlagValue(flag: String, why: String): String = s"Error: $flag: $why" + + def secondsWanted(spec: String): String = s"must be a positive whole number of seconds [$spec]" + def shardRange(spec: String): String = s"must be K/N with 0 <= K < N and N >= 1 [$spec]" + def shardForm(spec: String): String = s"must be of the form K/N [$spec]" + + val NotAFile = "is not a file" + val NotTheMemo = + "is not a directory. It must be the SAME memo this project's tokenizations were written " + + "into, or the invalidation would leave the real memo's stale entries in place." + + val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive") + val ShardWithPipeline = withUsage( + "Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") + val MaskWidenedUnderShard = withUsage( + "Error: --mask-widened has nothing to do under --shard: each shard builds a fresh dst.git and " + + "blobmap.db, so no mask is recorded to widen. Reuse a prior run's tokenizations with --warm= instead.") + + val RetokenizeNeedsIdentity = withUsage( + "Error: --retokenize needs --tokenizer-identity. Invalidating the entries without recording which " + + "tokenizer replaces them leaves nothing for the next run to detect a change against, so the next " + + "tokenizer defect would be just as silent as this one.") + val RetokenizeNeedsMemoDir = withUsage( + "Error: --retokenize needs --memo-dir. There are two caches, not one: dropping a blob_map row makes " + + "the walker re-run , and is tokenizeByBlobId/tokenBySha.pl, which answers from " + + "$BFG_MEMO_DIR keyed on sha1(contents) with no tokenizer in the key. Invalidating one layer " + + "without the other invalidates nothing at all.") + def retokenizeUncovered(unknown: Set[String], known: Set[String]): String = + s"Error: --retokenize names extension(s) --tokenizer-identity does not cover: " + + s"${unknown.toVector.sorted.mkString(", ")}. Known: ${known.toVector.sorted.mkString(", ")}." + val RetokenizeWithMaskWidened = + "Error: --mask-widened and --retokenize cannot be combined. Each verifies its own precondition " + + "against the rows, and together each would verify against a state the other is about to " + + "change. Widen first, then resume with --retokenize." + val RetokenizeUnderShard = + "Error: --retokenize has nothing to do under --shard: each shard builds a fresh dst.git and " + + "blobmap.db, so there are no cached tokenizations to invalidate. Retokenize the merged result, " + + "or drop the shards' --warm=." + val MemoDirWithoutRetokenize = + "Error: --memo-dir has no effect without --retokenize. Nothing else in blobExec reads the memo " + + "— tokenizeByBlobId/tokenBySha.pl takes it from $BFG_MEMO_DIR in the environment." + + def workerNotExecutable(path: Path): String = s"Error: --tokenizer-worker [$path] is not an executable file" + val WorkerNeedsPipeline = "Error: --tokenizer-worker needs --pipeline or --pipeline-trees" + + def srcNotADirectory(path: Path): String = s"Error: src repo [$path] is not a directory" + def commandMissing(command: String): String = s"Error: command [$command] does not exist" + val EmptyMask = "Error: fileMaskRegex must be non-empty" + + def stallRaised(from: Int, to: Int, blobTimeoutSeconds: Int): String = + s"blobExec: raising the stall window from ${from}s to ${to}s, because " + + s"--blob-timeout=${blobTimeoutSeconds}s needs a window that clears one blob's whole lifetime. " + + "Pass --stall-timeout explicitly to choose your own." + + def error(e: Throwable): String = s"Error: ${e.getMessage}" + val MaskWidenedHint = + "Hint: if the new mask is a strict superset of the recorded one, --mask-widened reuses the " + + "tokenizations already in blob_map instead of redoing them. It verifies that against the rows " + + "themselves and refuses if it is not true. It requires the work directory to be intact (resume " + + "at step 2), because blob_map's ids live in dst." + + def denylisted(count: Long, entries: Int): String = + s"blobExec: $count blob(s) were excluded by the blob denylist " + + s"(${BlobDenylist.EntriesSource}, $entries entr${if (entries == 1) "y" else "ies"}). Each one is " + + "named with its sha, path, reason and upstream citation on an 'EXCLUDED denylisted blob' line " + + "above; those lines and that list are the record of what this project's dataset does not " + + "contain. The files are absent from the tokenized repository, not present as raw source, so " + + "they produce no blame and no dataset row. This is not a failure and does not affect the exit status." + + def oversized(count: Long): String = + s"blobExec: $count blob(s) were excluded as oversized (>= ${Walker.MaxBlobBytes} bytes, JGit's " + + "stream-file threshold). Each one is named with its sha, path and size on an 'EXCLUDED " + + "oversized blob' line above; those lines are the record of what this project's dataset does " + + "not contain. The files are absent from the tokenized repository, not present as raw source, " + + "so they produce no blame and no dataset row. This is not a failure and does not affect the exit status." + + def failed(timedOut: Long, crashed: Long, blobTimeoutSeconds: Int): String = + s"blobExec: ${timedOut + crashed} blob(s) were excluded because their tokenizer failed " + + s"($timedOut timed out after ${blobTimeoutSeconds}s, $crashed reported a parser crash). Each " + + "one is named on an 'EXCLUDED failed blob' line above; those lines are the record of what " + + "this project's dataset does not contain. This does not affect the exit status." + } + + /** `--name=` as a pattern: `case Flag(value) =>`. */ + private final class ValuedFlag(val name: String) { + private val prefix = name + "=" + def unapply(arg: String): Option[String] = Option.when(arg.startsWith(prefix))(arg.substring(prefix.length)) + } - resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) match { - case Right(secs) => - if (secs != stallTimeoutSeconds) { - System.err.println( - s"blobExec: raising the stall window from ${stallTimeoutSeconds}s to ${secs}s, because " + - s"--blob-timeout=${blobTimeoutSeconds}s needs a window that clears one blob's whole lifetime. " + - "Pass --stall-timeout explicitly to choose your own." - ) - stallTimeoutSeconds = secs + private val TokenizerIdentityFlag = new ValuedFlag("--tokenizer-identity") + private val RetokenizeFlag = new ValuedFlag("--retokenize") + private val MemoDirFlag = new ValuedFlag("--memo-dir") + private val ShardFlag = new ValuedFlag("--shard") + private val WarmFlag = new ValuedFlag("--warm") + private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker") + private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout") + private val StallTimeoutFlag = new ValuedFlag("--stall-timeout") + + private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] = + parse(spec).left.map(Messages.badFlagValue(flag.name, _)) + + private def seconds(spec: String): Either[String, Int] = + parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec)) + + private def shardSpec(spec: String): Either[String, (Int, Int)] = + spec.split("/", -1) match { + case Array(k, n) => + (k.toIntOption, n.toIntOption) match { + case (Some(k), Some(n)) if n >= 1 && k >= 0 && k < n => Right((k, n)) + case _ => Left(Messages.shardRange(spec)) } - case Left(why) => - System.err.println(s"Error: $why") - sys.exit(1) + case _ => Left(Messages.shardForm(spec)) } - if (pipeline && pipelineTrees) { - System.err.println("Error: --pipeline and --pipeline-trees are mutually exclusive") - System.err.println(Usage) - sys.exit(1) - } + private def existingPath(isValid: Path => Boolean, otherwise: String)(spec: String): Either[String, Path] = { + val path = Paths.get(spec) + if (isValid(path)) Right(path) else Left(s"[$path] $otherwise") + } - if (shard.isDefined && maskWidened) { - System.err.println("Error: --mask-widened has nothing to do under --shard: each shard builds a " + - "fresh dst.git and blobmap.db, so no mask is recorded to widen. Reuse a prior run's " + - "tokenizations with --warm= instead.") - System.err.println(Usage) - sys.exit(1) - } + private def parseFlag(o: Options, flag: String): Either[String, Options] = flag match { + case "--abort-on-error" => Right(o.copy(abortOnError = true)) + case "--pipeline" => Right(o.copy(pipeline = true)) + case "--pipeline-trees" => Right(o.copy(pipelineTrees = true)) + case "--mask-widened" => Right(o.copy(maskWidened = true)) + case TokenizerIdentityFlag(spec) => + flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id)) + case RetokenizeFlag(spec) => + flagValue(RetokenizeFlag, spec)(TokenizerIdentity.parseExtensions).map(exts => o.copy(retokenize = exts)) + case MemoDirFlag(spec) => + flagValue(MemoDirFlag, spec)(existingPath(Files.isDirectory(_), Messages.NotTheMemo)) + .map(dir => o.copy(memoDir = Some(dir))) + case ShardFlag(spec) => + flagValue(ShardFlag, spec)(shardSpec).map(kn => o.copy(shard = Some(kn))) + case WarmFlag(spec) => + flagValue(WarmFlag, spec)(existingPath(Files.isRegularFile(_), Messages.NotAFile)) + .map(db => o.copy(warm = Some(db))) + case TokenizerWorkerFlag(spec) => + Right(o.copy(tokenizerWorker = Some(Paths.get(spec)))) + case BlobTimeoutFlag(spec) => + flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs)) + case StallTimeoutFlag(spec) => + flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true)) + case other => Left(Messages.unknownFlag(other)) + } - if (retokenizeExtensions.nonEmpty) { - if (tokenizerIdentity.isEmpty) { - System.err.println( - "Error: --retokenize needs --tokenizer-identity. Invalidating the entries without " + - "recording which tokenizer replaces them leaves nothing for the next run to detect a " + - "change against, so the next tokenizer defect would be just as silent as this one.") - System.err.println(Usage) - sys.exit(1) - } - if (memoDir.isEmpty) { - System.err.println( - "Error: --retokenize needs --memo-dir. There are two caches, not one: dropping a blob_map " + - "row makes the walker re-run , and is tokenizeByBlobId/tokenBySha.pl, " + - "which answers from $BFG_MEMO_DIR keyed on sha1(contents) with no tokenizer in the key. " + - "Invalidating one layer without the other invalidates nothing at all.") - System.err.println(Usage) - sys.exit(1) - } - val unknown = retokenizeExtensions -- tokenizerIdentity.extensions - if (unknown.nonEmpty) { - System.err.println( - s"Error: --retokenize names extension(s) --tokenizer-identity does not cover: " + - s"${unknown.toVector.sorted.mkString(", ")}. Known: " + - s"${tokenizerIdentity.extensions.toVector.sorted.mkString(", ")}.") - sys.exit(1) - } - if (maskWidened) { - System.err.println( - "Error: --mask-widened and --retokenize cannot be combined. Each verifies its own " + - "precondition against the rows, and together each would verify against a state the " + - "other is about to change. Widen first, then resume with --retokenize.") - sys.exit(1) - } - if (shard.isDefined) { - System.err.println( - "Error: --retokenize has nothing to do under --shard: each shard builds a fresh dst.git " + - "and blobmap.db, so there are no cached tokenizations to invalidate. Retokenize the " + - "merged result, or drop the shards' --warm=.") - sys.exit(1) - } - } else if (memoDir.isDefined) { - System.err.println( - "Error: --memo-dir has no effect without --retokenize. Nothing else in blobExec reads the " + - "memo — tokenizeByBlobId/tokenBySha.pl takes it from $BFG_MEMO_DIR in the environment.") - sys.exit(1) - } + /** The first rule a combination of flags breaks, in the order they are checked. */ + private def combinationError(o: Options): Option[String] = { + val retokenizing = o.retokenize.nonEmpty + val uncovered = o.retokenize -- o.tokenizerIdentity.extensions + val rules: Seq[(Boolean, () => String)] = Seq( + (o.pipeline && o.pipelineTrees) -> (() => Messages.PipelinesExclusive), + (o.shard.isDefined && o.maskWidened) -> (() => Messages.MaskWidenedUnderShard), + (retokenizing && o.tokenizerIdentity.isEmpty) -> (() => Messages.RetokenizeNeedsIdentity), + (retokenizing && o.memoDir.isEmpty) -> (() => Messages.RetokenizeNeedsMemoDir), + (retokenizing && uncovered.nonEmpty) -> + (() => Messages.retokenizeUncovered(uncovered, o.tokenizerIdentity.extensions)), + (retokenizing && o.maskWidened) -> (() => Messages.RetokenizeWithMaskWidened), + (retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard), + (!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize), + (o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline), + o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) -> + (() => Messages.workerNotExecutable(o.tokenizerWorker.get)), + (o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline) + ) + rules.collectFirst { case (true, message) => message() } + } - if (shard.isDefined && (pipeline || pipelineTrees)) { - System.err.println("Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") - System.err.println(Usage) - sys.exit(1) - } + private def positionalError(o: Options): Option[String] = + if (!Files.isDirectory(o.src)) Some(Messages.srcNotADirectory(o.src)) + else if (!Files.exists(Paths.get(o.command))) Some(Messages.commandMissing(o.command)) + else if (o.mask.isEmpty) Some(Messages.EmptyMask) + else None - tokenizerWorker.foreach { path => - if (!Files.isRegularFile(path) || !Files.isExecutable(path)) { - System.err.println(s"Error: --tokenizer-worker [$path] is not an executable file") - sys.exit(1) - } - if (!(pipeline || pipelineTrees)) { - System.err.println("Error: --tokenizer-worker needs --pipeline or --pipeline-trees") - sys.exit(1) - } - } + /** Left is the whole text to print before exiting with the usage status. */ + private[blobexec] def parse(args: Seq[String]): Either[String, Options] = { + val (flags, positional) = args.partition(_.startsWith("-")) + for { + parsed <- flags.foldLeft[Either[String, Options]](Right(Options()))((acc, f) => acc.flatMap(parseFlag(_, f))) + _ <- parsed.stallWindow.left.map(why => s"Error: $why") + _ <- combinationError(parsed).toLeft(()) + _ <- Either.cond(positional.length == 5, (), Usage) + full = parsed.copy(positional = positional.toVector) + _ <- positionalError(full).toLeft(()) + } yield full + } - if (positional.length != 5) { - System.err.println(Usage) - sys.exit(1) - } - val srcPath = Paths.get(positional(0)) - val dstPath = Paths.get(positional(1)) - val dbPath = Paths.get(positional(2)) - val command = positional(3) - val mask = positional(4) - - if (!Files.isDirectory(srcPath)) { - System.err.println(s"Error: src repo [$srcPath] is not a directory") - sys.exit(1) - } - if (!Files.exists(Paths.get(command))) { - System.err.println(s"Error: command [$command] does not exist") - sys.exit(1) - } - if (mask.isEmpty) { - System.err.println("Error: fileMaskRegex must be non-empty") - sys.exit(1) + def main(args: Array[String]): Unit = { + val options = parse(args.toVector) match { + case Right(o) => o + case Left(why) => exit(why, UsageExitStatus) } - // Files.createDirectories throws FileAlreadyExistsException on macOS - // when the target is a symlink (e.g. /tmp -> /private/tmp). Guard - // against that with an explicit isDirectory check. - val dbParent = dbPath.getParent - if (dbParent != null && !Files.isDirectory(dbParent)) Files.createDirectories(dbParent) - - val denylist = BlobDenylist.shipped - - val incremental = Files.isDirectory(dstPath) - val shardStr = shard.map { case (k, n) => s"$k/$n" }.getOrElse("none") - val warmStr = warmPath.map(_.toString).getOrElse("none") - println( - s"blobExec: src=$srcPath dst=$dstPath db=$dbPath command=$command mask=$mask " + - s"abortOnError=$abortOnError pipeline=$pipeline pipelineTrees=$pipelineTrees " + - s"shard=$shardStr warm=$warmStr blobTimeout=${blobTimeoutSeconds}s " + - s"stallTimeout=${stallTimeoutSeconds}s incremental=$incremental " + - s"maskWidened=$maskWidened denylistEntries=${denylist.size} " + - s"tokenizerIdentity=${if (tokenizerIdentity.isEmpty) "none" else tokenizerIdentity.render} " + - s"retokenize=${if (retokenizeExtensions.isEmpty) "none" else retokenizeExtensions.toVector.sorted.mkString(",")} " + - s"memoDir=${memoDir.map(_.toString).getOrElse("none")} " + - s"tokenizerWorker=${tokenizerWorker.map(_.toString).getOrElse("none")}" - ) + val stats = run(options, stallSeconds(options)) + // Exit explicitly: a timed-out blob's daemon reader threads can keep the JVM alive. + sys.exit(exitStatus(stats)) + } - val src: FileRepository = openSrc(srcPath) - val dst: FileRepository = openOrInitDst(dstPath) - - val widening = - if (!maskWidened) None - else Some(Mapping.MaskWidening( - newBlobResolves = id => { - val reader = dst.newObjectReader() - try reader.has(org.eclipse.jgit.lib.ObjectId.fromString(id)) - catch { case _: IllegalArgumentException => false } - finally reader.close() - }, - report = msg => println(s"blobExec: $msg") - )) - - // The memo is keyed on content sha1 alone (tokenizeByBlobId/tokenBySha.pl:76), - // so the purge re-reads the original blobs from src. - val retokenize = - if (retokenizeExtensions.isEmpty) None - else Some(Mapping.Retokenize( - extensions = retokenizeExtensions, - newBlobResolves = id => { - val reader = dst.newObjectReader() - try reader.has(org.eclipse.jgit.lib.ObjectId.fromString(id)) - catch { case _: IllegalArgumentException => false } - finally reader.close() - }, - purgeMemo = blobs => TokenizerMemo.purge(memoDir.get, blobs, sha => { - val reader = src.newObjectReader() - try Some(reader.open(org.eclipse.jgit.lib.ObjectId.fromString(sha), - org.eclipse.jgit.lib.Constants.OBJ_BLOB).getBytes) - catch { case _: Exception => None } - finally reader.close() - }), - report = msg => println(s"blobExec: $msg"), - memoDirIsTokenizerMemo = - memoDir.exists(d => TokenizerMemo.isTokenizerMemo(d, sys.env.get("BFG_MEMO_DIR"))) - )) - - val mapping = try Mapping.open(dbPath, command, mask, warmPath, widening, - tokenizerIdentity, retokenize) catch { - case m: Mapping.MetaMismatchException => - System.err.println(s"Error: ${m.getMessage}") - if (!maskWidened && m.getMessage.contains("mask")) - System.err.println( - "Hint: if the new mask is a strict superset of the recorded one, --mask-widened reuses " + - "the tokenizations already in blob_map instead of redoing them. It verifies that " + - "against the rows themselves and refuses if it is not true. It requires the work " + - "directory to be intact (resume at step 2), because blob_map's ids live in dst." - ) - src.close(); dst.close() - sys.exit(3) - case n: Mapping.MaskNarrowedException => - System.err.println(s"Error: ${n.getMessage}") - src.close(); dst.close() - sys.exit(3) - case d: Mapping.DanglingNewBlobException => - System.err.println(s"Error: ${d.getMessage}") - src.close(); dst.close() - sys.exit(3) - case t: Mapping.TokenizerChangedException => - System.err.println(s"Error: ${t.getMessage}") - src.close(); dst.close() - sys.exit(3) - case n: Mapping.NothingInvalidatedException => - // Not 0: a corpus script would publish entries that were never invalidated. - System.err.println(s"Error: ${n.getMessage}") - src.close(); dst.close() - sys.exit(RetokenizeIneffectiveExitStatus) - case i: IllegalArgumentException => - // Mapping.open re-checks main()'s flag rules, for library callers. - System.err.println(s"Error: ${i.getMessage}") + private def exit(message: String, status: Int): Nothing = { + System.err.println(message) + sys.exit(status) + } + + private def stallSeconds(o: Options): Int = { + val secs = o.stallWindow.getOrElse(o.stallTimeoutSeconds) + if (secs != o.stallTimeoutSeconds) + System.err.println(Messages.stallRaised(o.stallTimeoutSeconds, secs, o.blobTimeoutSeconds)) + secs + } + + private def run(o: Options, stallSeconds: Int): WalkStats = { + createParentDirectory(o.db) + val incremental = Files.isDirectory(o.dst) + println(startLine(o, stallSeconds, incremental)) + + val src = openSrc(o.src) + val dst = openOrInitDst(o.dst) + val mapping = openMapping(o, src, dst) match { + case Right(m) => m + case Left(refusal) => src.close(); dst.close() - sys.exit(1) + exit(refusal.message, refusal.status) } - val stats = try { - val parallelism = math.max(1, Runtime.getRuntime.availableProcessors) - val workerPool = tokenizerWorker.map(path => new TokenizerWorkerPool(path.toString, parallelism)) - val walker = new Walker( - src, dst, mapping, mask.r, command, abortOnError, parallelism, - pipeline, pipelineTrees, shard, - destinationMayContainObjects = incremental, - blobTimeoutSeconds = blobTimeoutSeconds, - stallTimeoutSeconds = stallTimeoutSeconds, - workerPool = workerPool, - denylist = denylist - ) - try walker.run() - finally workerPool.foreach(_.close()) - } finally { - mapping.close() - dst.close() - src.close() - } + val stats = + try walk(o, stallSeconds, src, dst, mapping, incremental) + finally { mapping.close(); dst.close(); src.close() } + println(doneLine(stats)) + reportExclusions(stats, o.blobTimeoutSeconds) + stats + } - println( - s"blobExec done: commitsProcessed=${stats.commitsProcessed} " + - s"blobsRunThroughCommand=${stats.blobsRunThroughCommand} " + - s"blobCommandExecutions=${stats.blobCommandExecutions} " + - s"blobsCacheHit=${stats.blobsCacheHit} " + - s"originalBlobCopyRequests=${stats.originalBlobCopyRequests} " + - s"originalBlobCopies=${stats.originalBlobCopies} " + - s"originalBlobAlreadyPresent=${stats.originalBlobAlreadyPresent} " + - s"originalBlobCacheHits=${stats.originalBlobCacheHits} " + - s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + - s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + - s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + - s"refsProjected=${stats.refsProjected} " + - s"blobsTimedOut=${stats.blobsTimedOut} " + - s"blobsOversized=${stats.blobsOversized} " + - s"blobsDenylisted=${stats.blobsDenylisted} " + - s"blobsParserCrashed=${stats.blobsParserCrashed} " + - s"aborted=${stats.aborted}" - ) + // Not Files.createDirectories alone: it throws on macOS when the target is a symlink (/tmp). + private def createParentDirectory(path: Path): Unit = + Option(path.getParent).filterNot(Files.isDirectory(_)).foreach(Files.createDirectories(_)) + + private def startLine(o: Options, stallSeconds: Int, incremental: Boolean): String = { + def orNone(value: Option[Any]): String = value.map(_.toString).getOrElse("none") + s"blobExec: src=${o.src} dst=${o.dst} db=${o.db} command=${o.command} mask=${o.mask} " + + s"abortOnError=${o.abortOnError} pipeline=${o.pipeline} pipelineTrees=${o.pipelineTrees} " + + s"shard=${orNone(o.shard.map { case (k, n) => s"$k/$n" })} warm=${orNone(o.warm)} " + + s"blobTimeout=${o.blobTimeoutSeconds}s stallTimeout=${stallSeconds}s incremental=$incremental " + + s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " + + s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " + + s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " + + s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}" + } - if (stats.blobsDenylisted > 0) { - System.err.println( - s"blobExec: ${stats.blobsDenylisted} blob(s) were excluded by the blob denylist " + - s"(${BlobDenylist.EntriesSource}, ${denylist.size} entr" + - s"${if (denylist.size == 1) "y" else "ies"}). Each one is named with its sha, path, " + - "reason and upstream citation on an 'EXCLUDED denylisted blob' line above; those " + - "lines and that list are the record of what this project's dataset does not contain. " + - "The files are absent from the tokenized repository, not present as raw source, so " + - "they produce no blame and no dataset row. This is not a failure and does not affect " + - "the exit status." - ) - } + private final case class Refusal(status: Int, message: String) + + private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] = + Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst))) + .toEither + .left.map(e => refusalFor(e, o).getOrElse(throw e)) + + private def refusalFor(e: Throwable, o: Options): Option[Refusal] = e match { + case _: Mapping.MetaMismatchException if !o.maskWidened && e.getMessage.contains("mask") => + Some(Refusal(RefusedExitStatus, s"${Messages.error(e)}\n${Messages.MaskWidenedHint}")) + case _: Mapping.MetaMismatchException | _: Mapping.MaskNarrowedException | + _: Mapping.DanglingNewBlobException | _: Mapping.TokenizerChangedException => + Some(Refusal(RefusedExitStatus, Messages.error(e))) + // Not 0: a corpus script would publish entries that were never invalidated. + case _: Mapping.NothingInvalidatedException => Some(Refusal(RetokenizeIneffectiveExitStatus, Messages.error(e))) + // Mapping.open re-checks the flag rules above, for library callers. + case _: IllegalArgumentException => Some(Refusal(UsageExitStatus, Messages.error(e))) + case _ => None + } - if (stats.blobsOversized > 0) { - // Not fatal: an oversized blob is deterministic and named on its EXCLUDED line. - System.err.println( - s"blobExec: ${stats.blobsOversized} blob(s) were excluded as oversized (>= " + - s"${Walker.MaxBlobBytes} bytes, JGit's stream-file threshold). Each one is named with " + - "its sha, path and size on an 'EXCLUDED oversized blob' line above; those lines are the " + - "record of what this project's dataset does not contain. The files are absent from the " + - "tokenized repository, not present as raw source, so they produce no blame and no " + - "dataset row. This is not a failure and does not affect the exit status." - ) - } + private val report: String => Unit = msg => println(s"blobExec: $msg") + + private def widening(o: Options, dst: FileRepository): Option[Mapping.MaskWidening] = + Option.when(o.maskWidened)(Mapping.MaskWidening(newBlobResolves = hasObject(dst), report = report)) + + // The memo is keyed on content sha1 alone (tokenizeByBlobId/tokenBySha.pl:76), + // so the purge re-reads the original blobs from src. + private def retokenize(o: Options, src: FileRepository, dst: FileRepository): Option[Mapping.Retokenize] = + Option.when(o.retokenize.nonEmpty)(Mapping.Retokenize( + extensions = o.retokenize, + newBlobResolves = hasObject(dst), + purgeMemo = blobs => TokenizerMemo.purge(o.memoDir.get, blobs, blobBytes(src)), + report = report, + memoDirIsTokenizerMemo = o.memoDir.exists(d => TokenizerMemo.isTokenizerMemo(d, sys.env.get("BFG_MEMO_DIR"))) + )) + + private def hasObject(repo: FileRepository)(id: String): Boolean = { + val reader = repo.newObjectReader() + try reader.has(ObjectId.fromString(id)) + catch { case _: IllegalArgumentException => false } + finally reader.close() + } - if (stats.blobsTimedOut + stats.blobsParserCrashed > 0) { - System.err.println( - s"blobExec: ${stats.blobsTimedOut + stats.blobsParserCrashed} blob(s) were excluded because " + - s"their tokenizer failed (${stats.blobsTimedOut} timed out after ${blobTimeoutSeconds}s, " + - s"${stats.blobsParserCrashed} reported a parser crash). Each one is named on an " + - "'EXCLUDED failed blob' line above; those lines are the record of what this project's " + - "dataset does not contain. This does not affect the exit status." - ) - } + private def blobBytes(repo: FileRepository)(sha: String): Option[Array[Byte]] = { + val reader = repo.newObjectReader() + try Some(reader.open(ObjectId.fromString(sha), Constants.OBJ_BLOB).getBytes) + catch { case _: Exception => None } + finally reader.close() + } - // Exit explicitly: a timed-out blob's daemon reader threads can keep the JVM alive. - sys.exit(exitStatus(stats)) + private def walk( + o: Options, + stallSeconds: Int, + src: FileRepository, + dst: FileRepository, + mapping: Mapping, + incremental: Boolean + ): WalkStats = { + val parallelism = math.max(1, Runtime.getRuntime.availableProcessors) + val workerPool = o.tokenizerWorker.map(path => new TokenizerWorkerPool(path.toString, parallelism)) + try + new Walker( + src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism, + o.pipeline, o.pipelineTrees, o.shard, + destinationMayContainObjects = incremental, + blobTimeoutSeconds = o.blobTimeoutSeconds, + stallTimeoutSeconds = stallSeconds, + workerPool = workerPool, + denylist = BlobDenylist.shipped + ).run() + finally workerPool.foreach(_.close()) + } + + private def doneLine(stats: WalkStats): String = + s"blobExec done: commitsProcessed=${stats.commitsProcessed} " + + s"blobsRunThroughCommand=${stats.blobsRunThroughCommand} " + + s"blobCommandExecutions=${stats.blobCommandExecutions} " + + s"blobsCacheHit=${stats.blobsCacheHit} " + + s"originalBlobCopyRequests=${stats.originalBlobCopyRequests} " + + s"originalBlobCopies=${stats.originalBlobCopies} " + + s"originalBlobAlreadyPresent=${stats.originalBlobAlreadyPresent} " + + s"originalBlobCacheHits=${stats.originalBlobCacheHits} " + + s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + + s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + + s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + + s"refsProjected=${stats.refsProjected} " + + s"blobsTimedOut=${stats.blobsTimedOut} " + + s"blobsOversized=${stats.blobsOversized} " + + s"blobsDenylisted=${stats.blobsDenylisted} " + + s"blobsParserCrashed=${stats.blobsParserCrashed} " + + s"aborted=${stats.aborted}" + + private def reportExclusions(stats: WalkStats, blobTimeoutSeconds: Int): Unit = { + if (stats.blobsDenylisted > 0) + System.err.println(Messages.denylisted(stats.blobsDenylisted, BlobDenylist.shipped.size)) + if (stats.blobsOversized > 0) + System.err.println(Messages.oversized(stats.blobsOversized)) + if (stats.blobsTimedOut + stats.blobsParserCrashed > 0) + System.err.println(Messages.failed(stats.blobsTimedOut, stats.blobsParserCrashed, blobTimeoutSeconds)) } - private def openSrc(path: java.nio.file.Path): FileRepository = { + private def openSrc(path: Path): FileRepository = { val gitDir = if (Files.isDirectory(path.resolve(".git"))) path.resolve(".git").toFile else path.toFile FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository] } - private def openOrInitDst(path: java.nio.file.Path): FileRepository = { + private def openOrInitDst(path: Path): FileRepository = { val exists = Files.isDirectory(path) val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] - if (!exists) repo.create(true) // bare init + if (!exists) repo.create(true) repo } } diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index 60ac75ae9..d5637fb2b 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -3,12 +3,10 @@ package cregit.blobexec import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -/** `--blob-timeout` and `--stall-timeout` are binding parts of the spec: 600 and - * 1800 by default, and values that are not a positive whole number of seconds - * must be rejected rather than silently read as "no limit". The shared value - * parser is tested here; `main` itself cannot be, because it answers bad input - * with `sys.exit`. The stall watchdog's decision is tested here too, because - * the watchdog itself halts the JVM and so cannot be exercised in-process. */ +import java.nio.file.Files + +/** The option parser, the 600/1800 budget defaults and the stall watchdog's decision. + * `main` itself answers with `sys.exit`, so its parts are tested instead. */ class MainOptionsSpec extends AnyFunSuite with Matchers { test("the default per-blob budget is 600 seconds") { @@ -201,4 +199,71 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { test("nanoTime is monotonic but not epoch-based, so negative elapsed is never a stall") { Walker.isStalled(nowNanos = -5 * second, lastProgressNanos = -3 * second, 30) shouldBe false } + // -- the option parser -------------------------------------------------------- + + private lazy val srcDir = Files.createTempDirectory("main-options-src-") + private def positional = Seq(srcDir.toString, "/tmp/dst.git", "/tmp/db.sqlite", "/bin/sh", "\\.c$") + + test("parse reads every flag into one options value") { + val parsed = Main.parse(Seq("--abort-on-error", "--pipeline", "--mask-widened", + "--blob-timeout=60", "--stall-timeout=900") ++ positional) + inside(parsed) { case Right(o) => + (o.abortOnError, o.pipeline, o.maskWidened) shouldEqual ((true, true, true)) + (o.blobTimeoutSeconds, o.stallTimeoutSeconds, o.stallExplicit) shouldEqual ((60, 900, true)) + (o.command, o.mask) shouldEqual (("/bin/sh", "\\.c$")) + } + } + + test("a shard is read as K and N") { + inside(Main.parse("--shard=1/4" +: positional)) { case Right(o) => o.shard shouldEqual Some((1, 4)) } + } + + test("parse leaves the defaults when no flag is given") { + inside(Main.parse(positional)) { case Right(o) => + o shouldEqual Main.Options(positional = positional.toVector) + } + } + + test("an unknown flag is refused with the usage text") { + inside(Main.parse("--frobnicate" +: positional)) { case Left(why) => + why should include("unknown flag [--frobnicate]") + why should include(Main.Usage) + } + } + + test("a bad flag value is refused, naming the flag") { + Seq( + "--blob-timeout=0" -> "--blob-timeout", + "--stall-timeout=x" -> "--stall-timeout", + "--shard=4/4" -> "--shard", + "--shard=1" -> "--shard", + "--warm=/nonexistent/db" -> "--warm", + "--tokenizer-identity=" -> "--tokenizer-identity", + "--memo-dir=/nonexistent/" -> "--memo-dir" + ).foreach { case (flag, name) => + withClue(s"$flag: ") { + inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") } + } + } + } + + test("a wrong number of positional arguments prints the usage text alone") { + Main.parse(positional.dropRight(1)) shouldEqual Left(Main.Usage) + } + + test("every option description in the usage text is at most three lines, and most are one") { + val continuation = "^ {29}\\S".r + val entries = Main.Usage.linesIterator.foldLeft(Vector.empty[Vector[String]]) { + case (acc, line) if line.startsWith(" --") || line.startsWith(" <") => acc :+ Vector(line) + case (acc, line) if acc.nonEmpty && continuation.findFirstIn(line).isDefined => acc.init :+ (acc.last :+ line) + case (acc, _) => acc + } + val lengths = entries.map(e => e.size - (if (e.head.trim.contains(' ')) 0 else 1)) + entries should not be empty + all(lengths) should be <= 3 + lengths.count(_ == 1) should be > entries.size / 2 + } + + private def inside[T](v: T)(pf: PartialFunction[T, Unit]): Unit = + if (pf.isDefinedAt(v)) pf(v) else fail(s"value did not match: $v") } From 051d71eef49c431bca400b440b5cae54b9eb7aa7 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 16:53:47 +0000 Subject: [PATCH 12/15] docs(pipeline): shorten the new option help Review of #102: each option the PR adds or changes gets at most three lines of help, most one. Same for their README sections. Every option stays documented; the detail lives in the refusal messages. --- README.md | 45 +++++------------------------------------ run_pipeline_process.sh | 44 ++++++++++++---------------------------- 2 files changed, 18 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index be918806b..c26a275a6 100644 --- a/README.md +++ b/README.md @@ -172,10 +172,8 @@ Flags (see `./run_pipeline_process.sh --help` for the full list): token moved between files keeps its original author. It changes up to 25% of a project's authors, so blame from before 2026-09-22 is not comparable with later blame. -**Pass `--reblame` whenever the blame itself changed**, not the file list. Without it -step 7 skips every file whose `.blame` exists, exits 0, and step 10 rebuilds from the -old blame. A real re-blame shows `Already done [0]` in step 7's summary. Do **not** -pass the flag to resume an interrupted run. +**Pass `--reblame` when the blame itself changed**: without it step 7 keeps every +existing `.blame` file. Do **not** pass it to resume an interrupted run. A full run starts by **deleting the work directory** — to keep several target repositories side by side, give each its own `--work`. To resume a failed run @@ -207,42 +205,9 @@ p.s.: long pauses are trimmed. ### When a tokenizer is corrected: `--retokenize` -A rebuilt tokenizer does **not** by itself change cached tokens: blobExec's cache -key (`command`, `mask`) stays the same, so a resume reuses the old output and exits 0. - -1. **Every run records a tokenizer identity** (`tokenize/tokenizerIdentity.pl`), one - digest per extension over its parser toolchain, in the blob map's `meta` table. - A mismatch on an extension with cached rows **refuses the run** (exit 3). - Nothing is invalidated automatically. -2. **`--retokenize EXTS` is the opt-in past that refusal**, for those extensions - only (re-tokenizing is 88% of pipeline time). - -To re-tokenize only the `.rs` entries of one project, after rebuilding the -tokenizer: - -```sh -# 1. rebuild whatever is stale -./run_pipeline_process.sh --ensure-artifacts - -# 2. resume at step 2 — never step 1, which deletes the work -./run_pipeline_process.sh \ - --repo-url --repo-name \ - --work ../cregit-files- \ - --retokenize rs \ - 2 -``` - -`--retokenize` needs `FROM_STEP=2` exactly: step 1 deletes the cache and step 3 or -later skips the invalidation. It is not available with `--mode sharded`. - -In one transaction it drops those extensions' `blob_map` rows and **their memo -entries** (the memo is keyed on content alone), plus `tree_map`, `commit_map` and -`ref_map`. Other extensions' tokenizations survive. - -blobExec refuses **before changing anything** if no cached row carries a named -extension, the memo held none of the affected blobs (wrong `--memo-dir`), another -extension's tokenizer changed and was not named, or a retained `new_blob` id is -missing from the cregit repository. An ineffective `--retokenize` exits 7, never 0. +Each run records a tokenizer identity per extension; a changed one with cached rows refuses +the run (exit 3). After `./run_pipeline_process.sh --ensure-artifacts`, resume at step 2 with +`--retokenize rs` to redo only those cached tokens; one that would change nothing exits 7. ### Outputs diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 3aca67567..d5180c3c3 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -13,12 +13,7 @@ artifacts (jars, tokenizers) are built automatically first — run inside Build: --build-only build all pipeline artifacts and exit, running nothing --ensure-artifacts - run the pre-run build guard and exit. That guard builds any - MISSING artifact, and also rebuilds the Rust tokenizer when its - sources are newer than the binary — it is the pipeline's only - compiled tokenizer, and a stale one silently shifts every - Rust token column by a field. The four jars and srcml2token - are still only checked for existence. + build missing artifacts and a stale Rust tokenizer, then exit Target repository: --repo-url URL git URL (or local path) of the repository to process (REQUIRED) @@ -27,19 +22,13 @@ Target repository: --commit-url URL base URL for the commit links in the generated HTML (default: derived from --repo-url as /commit/, which is correct for GitHub/GitLab-style hosts) - --mask REGEX regex selecting the files to tokenize; quote it - (default: `perl tokenize/fileMask.pl`, every extension - cregit can parse). A different mask forces a full rebuild, - unless --mask-widened applies. - --mask-widened resume across a wider mask, keeping blob_map's - tokenizations. Needs FROM_STEP>=2, not with --mode sharded. - blobExec verifies the rows and refuses with exit 3 if the - new mask drops a tokenized path or a new_blob is missing. - --retokenize EXTS comma-separated extensions (as in CregitLanguages.pm, e.g. - rs or c,h) whose cached tokenizations and memo entries are - redone, after their tokenizer identity changed. Needs - FROM_STEP=2 exactly, not with --mode sharded. blobExec - refuses (exit 7) a --retokenize that would change nothing. + --mask REGEX regex selecting the files to tokenize; quote it (default: + `perl tokenize/fileMask.pl`, every extension cregit parses). + A different mask forces a full rebuild, unless --mask-widened. + --mask-widened resume across a wider mask, keeping blob_map's tokenizations. + Needs FROM_STEP>=2, not with --mode sharded. + --retokenize EXTS redo the cached tokens of these extensions (e.g. rs or c,h) + after their tokenizer changed. Needs FROM_STEP=2, not sharded. --work DIR working/output directory (default: ../cregit-files). NOTE: a full run (FROM_STEP=1) starts by deleting this directory; use one directory per target repository. @@ -63,11 +52,8 @@ Target repository: A step-1 wipe that would delete a memo holding 10,000 entries or more (MEMO_KEEP_THRESHOLD, overridable as CREGIT_MEMO_KEEP_THRESHOLD) is refused; see --force-clean. - --blob-timeout N wall-clock budget in seconds for one blob's tokenizer - (blobExec default: 600). A child that exceeds it is killed - and that blob is excluded, as an EXCLUDED failed blob line - says. Also settable as CREGIT_BLOB_TIMEOUT in the - environment, which is how to reach it through ctp.py. + --blob-timeout N seconds before one blob's tokenizer is killed and the blob + excluded (blobExec default: 600; env CREGIT_BLOB_TIMEOUT). --stall-timeout N watchdog window in seconds (blobExec default: 1800). If no blob, tree, commit or blob copy completes anywhere in this window the run is killed with exit 5. Must be larger than @@ -88,9 +74,7 @@ Output: should skip them. NOTE: the HTML views are the fallback output when python3+duckdb is missing, so --skip-html without duckdb leaves the run with no final artifact. - --reblame re-blame every file in step 7, replacing existing .blame - output. Pass it when the blame itself changed; a resume - without it skips every file already blamed. + --reblame re-blame every file in step 7, overwriting its .blame output --gc MODE how to pack the generated cregit repo after tokenizing (default: plain) none do not pack at all. Fastest, but every later @@ -131,10 +115,8 @@ Tokenizer: delegates to blobExec/shard_build.sh --shards N shard count for --mode sharded (default: 4) --tokenizer-worker - send each blob to a pool of persistent tokenizer processes - (tokenizeByBlobId/tokenWorker.pl) instead of starting the - tokenizer once per blob. Same output. Needs --mode pipeline or - pipeline-trees (default: off) + tokenize through a pool of persistent tokenizer processes; same + output. Needs --mode pipeline or pipeline-trees (default: off) --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not From c1e69c0fe639b4cb399e2e510d43f7bb19148c21 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 16:55:39 +0000 Subject: [PATCH 13/15] refactor(pipeline): cut new refusals to two lines Review of #102: the step and mode refusals, the needs_build check, the tokenizer identity errors and the re-fold log each print at most two lines now: what is wrong and the command that fixes it. The reasoning moves out of the output. Exit statuses and every phrase the shell tests match are unchanged. --- run_pipeline_process.sh | 75 ++++++++++++----------------------------- 1 file changed, 22 insertions(+), 53 deletions(-) diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index d5180c3c3..7ac200e42 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -352,8 +352,7 @@ tokenize_gate() { summary="$stage hit a parser crash: deterministic, so --blob-timeout will not help. Fix srcML, or denylist the diagnosed blob." ;; 7) die "$stage refused --retokenize, which would have invalidated nothing (exit 7). - Check the extension spelling (as in tokenize/CregitLanguages.pm) and that - --memo-dir names this project's memo ($MEMO_DIR). Nothing in $WORK changed." ;; + Check the extension spelling and that --memo-dir is this project's memo ($MEMO_DIR)." ;; *) die "$stage failed (exit $status)" ;; esac @@ -414,59 +413,37 @@ if [ -z "$MASK" ]; then fi if [ "$MASK_WIDENED" = 1 ] && [ "$FROM_STEP" = "1" ]; then - echo "--mask-widened needs FROM_STEP>=2. A step-1 run starts by deleting $WORK, so the - blob map it would reuse and the cregit.git its new_blob ids live in are both - gone before blobExec starts, and the flag would preserve nothing. - Resume instead: - runner: $0 --repo-url --work $WORK --mask-widened [same flags] 2 - ctp.py: python3 ./ctp.py run [same flags] --mask-widened --from-step 2" >&2 + echo "--mask-widened needs FROM_STEP>=2: step 1 deletes $WORK and the blob map it reuses. + Resume with: $0 [same flags] --mask-widened 2 (ctp.py: --mask-widened --from-step 2)" >&2 exit 2 fi if [ "$MASK_WIDENED" = 1 ] && [ "$MODE" = "sharded" ]; then - echo "--mask-widened is not available with --mode sharded: every shard builds a fresh - blob map, so no recorded mask exists to widen. Reuse a prior run's - tokenizations with shard_build.sh --warm-db instead." >&2 + echo "--mask-widened is not available with --mode sharded: shards keep no mask to widen. + Reuse a prior run's tokenizations with shard_build.sh --warm-db instead." >&2 exit 2 fi if [ -n "$RETOKENIZE" ] && [ "$FROM_STEP" != "2" ]; then - echo "--retokenize needs FROM_STEP=2 exactly (got $FROM_STEP). - Step 1 deletes $WORK, so the blob map and memo whose poisoned entries this flag - removes are gone before blobExec starts — and a from-scratch run re-tokenizes - everything anyway, with the current tokenizer, which is the same outcome at - full cost. - Step 3 or later never reaches the invalidation at all: it lives in step 2, so - the flag would be skipped, steps 3-10 would run over the poisoned tokens, and - the run would exit 0. - Resume at step 2: - runner: $0 --repo-url --work $WORK --retokenize $RETOKENIZE [same flags] 2 - ctp.py: python3 ./ctp.py run [same flags] --from-step 2" >&2 + echo "--retokenize needs FROM_STEP=2 exactly (got $FROM_STEP): step 1 deletes the cache, 3+ skip it. + Resume with: $0 [same flags] --retokenize $RETOKENIZE 2 (ctp.py: --from-step 2)" >&2 exit 2 fi if [ "$REBLAME" = 1 ] && [ "$FROM_STEP" -gt 7 ]; then - echo "--reblame needs FROM_STEP<=7 (got $FROM_STEP). The re-blame happens inside - step 7. From step 8 onward the flag is skipped, step 10 rebuilds the Parquet from - the blame files already on disk, and the run exits 0 having changed nothing. - Resume at step 7: - runner: $0 --repo-url --work $WORK --reblame [same flags] 7 - ctp.py: python3 ./ctp.py run [same flags] --reblame --from-step 7" >&2 + echo "--reblame needs FROM_STEP<=7 (got $FROM_STEP): later steps skip the re-blame. + Resume with: $0 [same flags] --reblame 7 (ctp.py: --reblame --from-step 7)" >&2 exit 2 fi if [ -n "$RETOKENIZE" ] && [ "$MODE" = "sharded" ]; then - echo "--retokenize is not available with --mode sharded: every shard builds a fresh - blob map, so there are no cached tokenizations for it to invalidate. Retokenize - the merged result in a non-sharded step-2 resume, or drop the shards' warm db." >&2 + echo "--retokenize is not available with --mode sharded: shards keep no cached tokens." >&2 exit 2 fi if [ -n "$RETOKENIZE" ] && [ "$MASK_WIDENED" = 1 ]; then - echo "--mask-widened and --retokenize cannot be used in the same run. Each verifies its - own precondition against the blob map's rows, and together each would verify - against a state the other is about to change. - Do them one at a time: widen first, then resume again with --retokenize." >&2 + echo "--mask-widened and --retokenize cannot be used in the same run. + Widen first, then resume again with --retokenize." >&2 exit 2 fi @@ -474,9 +451,8 @@ if [ -n "$RETOKENIZE" ]; then for _ext in ${RETOKENIZE//,/ }; do case "$_ext" in ''|*[!a-z0-9+]*) - echo "invalid --retokenize: '$_ext' is not an extension. Want lowercase names - without a leading dot, comma-separated, as spelled in tokenize/CregitLanguages.pm: - --retokenize rs --retokenize c,h" >&2 + echo "invalid --retokenize: '$_ext'. Want lowercase extensions without a dot, + as in tokenize/CregitLanguages.pm: --retokenize rs or --retokenize c,h" >&2 exit 2 ;; esac done @@ -666,13 +642,10 @@ needs_build() { NEEDS_BUILD_REASON="" [ $# -gt 0 ] || die "needs_build: no sources declared for $artifact (a source list is missing)" local p + # A missing source path would silently stop guarding the artifact. for p in "$@"; do - [ -e "$p" ] || die \ -"declared source path does not exist: $p - (it guards $artifact). Either the checkout is incomplete or the source list - in this script is wrong. Refusing to guess: an unreadable source list - silently stops guarding that artifact, which is how a 16-day-old - rustTokenizer binary shipped a corpus of shifted token columns." + [ -e "$p" ] || die "declared source path does not exist: $p (it guards $artifact). + The checkout is incomplete or the source list in this script is wrong." done if [ ! -e "$artifact" ]; then NEEDS_BUILD_REASON="missing" @@ -813,10 +786,8 @@ echo "" # dir and step 7 would keep blame of the old shas. Keeps the bare repos, blob map # and memo. Named paths only, never a glob or $WORK itself. drop_refold_derived_artifacts() { # $1 = which flag is asking, for the log - log "$1: dropping the artifacts derived from the tokenized repo," - log " because the re-fold gives every cregit commit a new sha." - log " KEEPING: $REPO_PATH_ORIGINAL_BARE, $REPO_PATH_CREGIT_BARE," - log " $DB_PATH_BLOBMAP, $MEMO_DIR" + log "$1: dropping what derives from the tokenized repo (the re-fold changes every sha)." + log " KEEPING: $REPO_PATH_ORIGINAL_BARE, $REPO_PATH_CREGIT_BARE, $DB_PATH_BLOBMAP, $MEMO_DIR" local _stale for _stale in \ "$WORK/blame" \ @@ -876,8 +847,7 @@ TOKENIZER_IDENTITY=$(perl "${CREGIT}/tokenize/tokenizerIdentity.pl" \ --srcml2token="${SRCML2TOKEN}" \ --ctags="$(which ctags)") \ || die "cannot compute the tokenizer identity (tokenize/tokenizerIdentity.pl). - blobExec needs it to tell a cache built by this tokenizer from one built by a - different one. Build the artifacts first: $0 --ensure-artifacts" + Build the artifacts first: $0 --ensure-artifacts" [ -n "$TOKENIZER_IDENTITY" ] \ || die "tokenize/tokenizerIdentity.pl printed nothing; refusing to run with no tokenizer identity" log "tokenizer identity: $TOKENIZER_IDENTITY" @@ -897,9 +867,8 @@ if [ "$MODE" = "sharded" ]; then ${BLOB_TIMEOUT:+--blob-timeout "$BLOB_TIMEOUT"} \ ${STALL_TIMEOUT:+--stall-timeout "$STALL_TIMEOUT"} || TOKENIZE_RC=$? tokenize_gate "$TOKENIZE_RC" "sharded tokenize (shard_build.sh)" - log "note: --mode sharded does not record a tokenizer identity in the merged blob map." - log " A later resume cannot detect a tokenizer change against it. Non-sharded" - log " modes record and check it; see --retokenize." + log "note: --mode sharded records no tokenizer identity in the merged blob map," + log " so a later resume cannot detect a tokenizer change against it." else MODE_FLAG="" [ "$MODE" = "pipeline" ] && MODE_FLAG="--pipeline" From 994326f53bebc9386a552ecd22b233c008d54130 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 17:37:51 +0000 Subject: [PATCH 14/15] docs: point the denylist docs at the Scala list --- README.md | 10 +++------- generate_dataset/DATASET.md | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c26a275a6..252577053 100644 --- a/README.md +++ b/README.md @@ -186,13 +186,9 @@ returns without invoking `srcml` at all. Put it out of reach with `--memo-dir`, names neither repository nor extension). The runner refuses a step-1 wipe that would delete a memo of 10,000 entries or more; `--force-clean` overrides it. -209 blobs are on a **blob denylist** -(`blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv`): 12 on which -srcML 1.1.0 does not terminate (8 Java, srcML/srcML#2361; 4 C), and 197 C and C++ -blobs on which it crashes under `--position`. The file header cites each defect. -Denylisted blobs never reach the tokenizer, are dropped from the rewritten trees, -are counted as `blobsDenylisted`, and do **not** change the exit status. Adding an -entry means editing that file and rebuilding the jar. +12 blobs on which srcML 1.1.0 does not terminate (8 Java, srcML/srcML#2361; 4 C) are +on a **blob denylist** (`BlobDenylistEntries.scala`). They never reach the tokenizer, +are dropped from the rewritten trees, and are counted as `blobsDenylisted`. A blob whose tokenizer times out or reports a parser crash is excluded the same way without an entry: it is named on an `EXCLUDED failed blob` line, tried once diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index f08e53dd2..6fdaddcb3 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -300,7 +300,7 @@ produces no blame and no dataset row rather than rows of unparsed text: | Mechanism | Recorded as | Effect here | |---|---|---| -| the **blob denylist**, `blobExec/src/main/resources/cregit/blobexec/blob-denylist.tsv` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list ships in the jar; nothing at run time can extend or override it | +| the **blob denylist**, `BlobDenylistEntries.scala` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list is compiled into the jar | | **oversized** blobs (at or above JGit's stream-file threshold) | `blobsOversized`, plus one `EXCLUDED oversized blob` line each | no rows for those blobs | | a blob the tokenizer **timed out** or **crashed** on | `blobsTimedOut` or `blobsParserCrashed`, plus one `EXCLUDED failed blob` line each with `reason=timeout` or `reason=parser-crash` | no rows for those blobs. The run still exits 0 | From aeccf6ae6e64ed4eba2f841d545aac4184c158b0 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Mon, 5 Oct 2026 17:59:37 +0000 Subject: [PATCH 15/15] fix(blobexec): exclude a blob whose tokenizer dies GNU timeout reported a tokenizer killed from outside (for example by the OOM killer) as 137, so the blob was excluded as a timeout. With ChildRunner that status fell into the generic non-zero branch, which keeps the original blob: the raw source went into the token tree and into blob_map, with no EXCLUDED line. A death by signal (JVM exit status 128 + N) now takes the parser-crash path, as tokenizeSrcMl.pl already does for srcML. A tokenizer worker that dies in a request (EOF on its stdout) is now reported as Killed, as a worker that does not answer already was, so that blob takes the timeout path. Both are excluded and counted; the walk carries on. --- .../src/main/scala/cregit/blobexec/BlobExec.scala | 13 +++++++++++++ .../scala/cregit/blobexec/TokenizerWorkerPool.scala | 2 +- .../test/scala/cregit/blobexec/BlobExecSpec.scala | 10 ++++++++++ .../cregit/blobexec/TokenizerWorkerPoolSpec.scala | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala index 44bd809ed..dde250270 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala @@ -16,6 +16,9 @@ object BlobExec { /** Must match `$PARSER_CRASH_EXIT` in `tokenize/tokenizeSrcMl.pl`. */ val ParserCrashExitCode: Int = 33 + /** The JVM reports a death by signal N as exit status 128 + N. */ + private val SignalDeathStatus: Int = 128 + sealed trait Outcome object Outcome { case object Skip extends Outcome @@ -64,6 +67,16 @@ object BlobExec { onParserCrash() Outcome.Skip + case ChildRunner.Outcome.Exited(status, _, stderr) if status > SignalDeathStatus => + // A kill from outside (e.g. the OOM killer) must not keep the source as tokens. + System.err.println( + s"Warning: command [$command] was killed by signal ${status - SignalDeathStatus} on blob " + + s"$origSha at path [$fullPath]: blob excluded" + ) + printStderr(command, origSha, fullPath, stderr) + onParserCrash() + Outcome.Skip + case ChildRunner.Outcome.Exited(status, _, stderr) if status != 0 => logError(command, origSha, fullPath, status, stderr) if (abortOnError) Outcome.Abort(stderr, status) else Outcome.Skip diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala index ca705b786..b0bc6161c 100644 --- a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala @@ -32,7 +32,7 @@ final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoClose ChildRunner.Outcome.Killed(s"tokenizer worker ${worker.index} gave no response within ${backstop}s; replaced it") case Some(Left(failure)) => replace(worker) - ChildRunner.Outcome.Exited(1, Array.emptyByteArray, s"tokenizer worker ${worker.index} failed: $failure\n") + ChildRunner.Outcome.Killed(s"tokenizer worker ${worker.index} died ($failure); replaced it") case Some(Right(response)) => if (worker.process.isAlive) idle.put(worker) else replace(worker) response diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala index dbef5ecb6..d973c6139 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala @@ -131,6 +131,16 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { crashes.get shouldEqual 1 } + test("a tokenizer killed by a signal is excluded like a parser crash, never kept as source") { + val crashes = new java.util.concurrent.atomic.AtomicInteger(0) + val cmd = shellScript("cat > /dev/null; kill -9 $$") + BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, + abortOnError = false, inserter, + onParserCrash = () => { crashes.incrementAndGet(); () } + ) shouldBe BlobExec.Outcome.Skip + crashes.get shouldEqual 1 + } + test("a parser crash skips one blob even with abortOnError, like a timeout") { val cmd = shellScript(s"exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala index 9a35e4201..e9cfd78bf 100644 --- a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala @@ -87,6 +87,7 @@ class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAf } run("timeout.c") shouldEqual ((BlobExec.Outcome.Skip, "timeout")) run("crash.c") shouldEqual ((BlobExec.Outcome.Skip, "parser-crash")) + run("die.c") shouldEqual ((BlobExec.Outcome.Skip, "timeout")) run("ok.c")._1 shouldBe a[BlobExec.Outcome.Replace] } }