diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 161affb5f..d0f4b9ee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,14 +76,47 @@ jobs: - name: Build & test Rust tokenizer (rust_tokenizer) run: devenv shell -- bash -c 'cd tokenize/rustTokenizer && make && make test' + - name: "Tokenizer worker: same output as tokenBySha.pl" + run: devenv shell -- bash tests/test_token_worker.sh + - name: Perl tests (prove) run: devenv shell -- bash -c 'prove tests/t' - name: Pipeline tests (shell) - run: devenv shell -- bash -c 'tests/tokenize_gate.sh && tests/pipeline_workdir_guard.sh' + run: | + devenv shell -- bash -c 'set -e + for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + bash "tests/$t.sh" + done' - name: Dataset tests (pytest) - run: devenv shell -- bash -c 'pytest -q' + run: devenv shell -- pytest -q --junitxml=junit-generate-dataset.xml + + # `devenv shell -- pytest` does not flush its summary line; read the JUnit XML. + - name: Re-check the pytest JUnit counts + if: always() + shell: bash + run: | + f=junit-generate-dataset.xml + [ -f "$f" ] || { echo "no $f: pytest did not get far enough to write one"; exit 1; } + python3 - "$f" <<'PY' + import sys, xml.etree.ElementTree as ET + root = ET.parse(sys.argv[1]).getroot() + s = root.find("testsuite") if root.tag == "testsuites" else root + a = s.attrib + print(f"tests={a['tests']} failures={a['failures']} " + f"errors={a['errors']} skipped={a['skipped']}") + if int(a["failures"]) or int(a["errors"]): + sys.exit("failures or errors in the suite") + PY + + - name: Upload the pytest JUnit report + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-generate-dataset + path: junit-generate-dataset.xml + if-no-files-found: warn - name: Build & test legacy onejar modules (sbt test one-jar, Scala 2.10 / JDK 8) run: | diff --git a/README.md b/README.md index d3a6c95fe..252577053 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ HTML views land in the sibling directory `../cregit-files/html`. See ## Preliminaries - Code is written in Scala, C++, Rust and Perl. -- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the pinned `srcml` parser from source for those platforms (`nix/srcml.nix`). The 1.1.0 release binary segfaults under `--position`, which the pipeline always passes, and the fix is upstream but unreleased, so we pin a revision rather than a release. `srcml --version` still prints `1.1.0`. +- Platform: Linux x86_64 or macOS arm64 — `devenv` builds the pinned `srcml` + 1.1.0 parser from its release tag (`nix/srcml.nix`). ## Prerequisites @@ -87,7 +88,7 @@ This builds, in dependency order: | artifact | module | toolchain | | ------------------------------------------------------------- | ------------------- | ------------------------- | -| `tokenize/srcMLtoken/srcml2token` | C++ transcoder | gcc + xerces-c | +| `tokenize/srcMLtoken/srcml2token` | srcML tokenizer | gcc + xerces-c + libsrcml | | `tokenize/rustTokenizer` binary | Rust tokenizer | cargo | | `blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar` | tokenization driver | sbt, JDK 21 | | `{slickGitLog,persons,remapCommits}/target/scala-2.10/*-one-jar.jar` | history / persons / remap tools | sbt 0.13, JDK 8 | @@ -111,9 +112,12 @@ cd ../tokenize/srcMLtoken && make && make test cd ../rustTokenizer && make && make test cd ../.. +bash tests/test_token_worker.sh prove tests/t pytest -q -tests/tokenize_gate.sh && tests/pipeline_workdir_guard.sh +for t in tokenize_gate pipeline_workdir_guard ensure_artifacts retokenize_passthrough reblame_passthrough; do + bash "tests/$t.sh" +done for module in slickGitLog persons remapCommits; do (cd "$module" && sbt --java-home "$LEGACY_JAVA_HOME" -batch test one-jar) @@ -162,6 +166,14 @@ Flags (see `./run_pipeline_process.sh --help` for the full list): | `--memo-dir` | where to memoize tokenized blobs; outside `--work` it survives the full-run wipe | `/memo` | | `--mode` / `--shards` | tokenizer walk mode / shard count for `sharded` | `pipeline` / `4` | | `--jobs` | concurrent blame/HTML processes | `CREGIT_JOBS` or up to `4` CPUs | +| `--reblame` | re-blame every file in step 7, replacing existing `.blame` output | off — a resume skips files already blamed | + +**`git blame` runs with `-C100` copy detection** (`blameRepo/formatBlame.pl`), so a +token moved between files keeps its original author. It changes up to 25% of a +project's authors, so blame from before 2026-09-22 is not comparable with later blame. + +**Pass `--reblame` when the blame itself changed**: without it step 7 keeps every +existing `.blame` file. Do **not** pass it to resume an interrupted run. A full run starts by **deleting the work directory** — to keep several target repositories side by side, give each its own `--work`. To resume a failed run @@ -174,19 +186,25 @@ returns without invoking `srcml` at all. Put it out of reach with `--memo-dir`, names neither repository nor extension). The runner refuses a step-1 wipe that would delete a memo of 10,000 entries or more; `--force-clean` overrides it. -Four blobs in one corpus project are on a **blob denylist** -(`blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala`): srcML 1.1.0 does -not terminate on them (upstream srcML/srcML#2361, which the revision we now pin closes, -so the entries should be inert). They are never handed to -the tokenizer, are dropped from the rewritten trees rather than kept as raw -source, are counted as `blobsDenylisted` and named with a reason and a citation — -and, unlike a tokenizer timeout, they do **not** change the exit status. Adding an -entry means editing that file and rebuilding the jar. +12 blobs on which srcML 1.1.0 does not terminate (8 Java, srcML/srcML#2361; 4 C) are +on a **blob denylist** (`BlobDenylistEntries.scala`). They never reach the tokenizer, +are dropped from the rewritten trees, and are counted as `blobsDenylisted`. + +A blob whose tokenizer times out or reports a parser crash is excluded the same +way without an entry: it is named on an `EXCLUDED failed blob` line, tried once +per run, and the run exits 0. An entry still saves the `--blob-timeout` (600 s by +default) that a known hang costs on each run. Example run (cregit run on itself): ![Example cregit run](cregit.gif) p.s.: long pauses are trimmed. +### When a tokenizer is corrected: `--retokenize` + +Each run records a tokenizer identity per extension; a changed one with cached rows refuses +the run (exit 3). After `./run_pipeline_process.sh --ensure-artifacts`, resume at step 2 with +`--retokenize rs` to redo only those cached tokens; one that would change nothing exits 7. + ### Outputs Everything lands in the work directory (default: `../cregit-files`, a sibling diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index eb86b9891..7036ef5ab 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -207,7 +207,7 @@ =head1 SYNOPSIS blameRepoFiles.pl [options] Options: - --override overwrite existing files + --overwrite re-blame files whose .blame output already exists --help brief help message --man full documentation --jobs=N maximum concurrent blame processes (default 1) @@ -228,9 +228,9 @@ =head1 OPTIONS Prints the manual page and exits. -=item B<--override> +=item B<--overwrite> - By default, if an output file exists it is skipped. This changes that behaviour. + Re-blame files whose output already exists (by default they are skipped). =back diff --git a/blobExec/shard_build.sh b/blobExec/shard_build.sh index 035b5dcb9..a1393e818 100755 --- a/blobExec/shard_build.sh +++ b/blobExec/shard_build.sh @@ -80,9 +80,9 @@ if [ -z "$THREADS" ]; then fi # Default tokenizer dispatcher (srcML for .c/.h, rustTokenizer for .rs, …); the -# --srcml/--ctags paths are forwarded only to the srcML parser. +# --srcml2token/--ctags paths are forwarded only to the srcML parser. if [ -z "$TOK_CMD" ]; then - TOK_CMD="$CREGIT/tokenize/tokenize.pl --srcml2token=$CREGIT/tokenize/srcMLtoken/srcml2token --srcml=$(command -v srcml) --ctags=$(command -v ctags)" + TOK_CMD="$CREGIT/tokenize/tokenize.pl --srcml2token=$CREGIT/tokenize/srcMLtoken/srcml2token --ctags=$(command -v ctags)" fi export BFG_TOKENIZE_CMD="$TOK_CMD" JAVA="$(command -v java)" diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala index c9032c00c..d9fcba16d 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylist.scala @@ -1,8 +1,7 @@ package cregit.blobexec -/** Blobs never handed to the tokenizer, keyed by git blob id: srcML 1.1.0 does not - * terminate on them. Excluded like an oversized blob, and deliberately not gating - * the exit status. The entries live in [[BlobDenylistEntries]]. */ +/** Blobs never handed to the tokenizer, keyed by git blob id. Excluded like an + * oversized blob, and deliberately not gating the exit status. */ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.Entry]) { def entryFor(sha: String): Option[BlobDenylist.Entry] = bySha.get(sha.toLowerCase) @@ -13,8 +12,8 @@ final class BlobDenylist private (private val bySha: Map[String, BlobDenylist.En object BlobDenylist { - /** Why one blob is excluded. Both fields are required: an exclusion nobody can - * trace back to a defect is indistinguishable from data loss. */ + /** Both fields are required: an exclusion nobody can trace back to a defect is + * indistinguishable from data loss. */ final case class Entry(citation: String, reason: String) /** Where a reader finds the list, quoted in the excluded-blob report. */ diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala index 6fb83a03f..236feae7e 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala @@ -1,27 +1,36 @@ package cregit.blobexec -/** The denylisted blobs themselves, keyed by git blob id. Data only: [[BlobDenylist]] - * holds the lookup, and Walker drops these paths from every rewritten tree. - * - * Keyed on content, not path, so one file's history across a move is several - * entries sharing a reason. Each defect class is named once below, which is why the - * shas are grouped rather than each carrying its own prose. */ +/** Blobs on which srcML 1.1.0 does not terminate, keyed by git blob id and grouped + * by defect. A tokenizer crash needs no entry: the walk already excludes that blob. */ private[blobexec] object BlobDenylistEntries { import BlobDenylist.Entry - /** srcML 1.1.0 loops forever on `String @A [] [] s = new String @A [2] [2];`. - * Open upstream against the latest release. */ private val JavaAnnotatedArrays = Entry( "srcML/srcML#2361", - "srcML 1.1.0 does not terminate on this file's annotated multi-dimensional " + - "array declarations; OpenJDK langtools regression test @bug 8005681" + "srcML 1.1.0 does not terminate on annotated array dimensions, minimal reproducer " + + "`class C { int @A [] a @A []; }`; OpenJDK langtools regression tests " + + "TestNewCastArray (@bug 8005681) and CheckErrorsForSource7" + ) + + private val CppInCHeader = Entry( + "srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h", + "srcML 1.1.0's C parser does not terminate on C++ in a .h file (no upstream issue), " + + "minimal reproducer `namespace n{struct T&f(const struct S);}`; eden/fs/utils/StatTimes.h" ) val NonTerminating: Map[String, Entry] = Map( "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65" -> JavaAnnotatedArrays, "303c1a109f1fdc4b9d84e15111c56931202844ac" -> JavaAnnotatedArrays, "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca" -> JavaAnnotatedArrays, - "c91924d1e87cb82d372a73b24579ccc34609efaa" -> JavaAnnotatedArrays + "c91924d1e87cb82d372a73b24579ccc34609efaa" -> JavaAnnotatedArrays, + "56c8adf588e697a3e42b600b2e5ec54710592c61" -> JavaAnnotatedArrays, + "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5" -> JavaAnnotatedArrays, + "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7" -> JavaAnnotatedArrays, + "afbd81e5789ba060147b0de3be4f01759963299d" -> JavaAnnotatedArrays, + "5460d43f1884e3be230e02215151b9b963844a48" -> CppInCHeader, + "720914b27e170b73b38ef662fb04239273dd0a34" -> CppInCHeader, + "99c191e9a208547f5c9b1224c290f106a640de59" -> CppInCHeader, + "dbb885e0bf046755c4fcfbddfc2454f90c550c62" -> CppInCHeader ) } diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala index b71e7f2e5..dde250270 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobExec.scala @@ -5,37 +5,28 @@ import org.eclipse.jgit.lib.{ObjectId, ObjectInserter} import java.util.{Arrays => JavaArrays} -/** - * Run the per-blob command and map its result onto a git object. - * - * The command reads the original blob on stdin and writes the replacement on - * stdout. `BFG_FILENAME` is the basename rather than the path because cregit's - * `tokenBySha.pl` keys its on-disk memo by it. - * - * - env `BFG_BLOB` / `BFG_FILENAME` / `BFG_PATH` - * - exit != 0 → `Skip` (or `Abort` if `abortOnError`) - * - killed → `Skip`, never `Abort`, never `Replace` - * - stdout == stdin → `Skip` - * - otherwise → `Replace(newBlob)` - */ +/** Runs a per-blob command: original bytes on stdin, replacement on stdout, env + * `BFG_BLOB`, `BFG_PATH` and `BFG_FILENAME` (the basename, which `tokenBySha.pl` + * keys its memo by). */ object BlobExec { /** Per-blob wall-clock budget for the external command, in seconds. */ val DefaultTimeoutSeconds: Int = 600 + /** Must match `$PARSER_CRASH_EXIT` in `tokenize/tokenizeSrcMl.pl`. */ val ParserCrashExitCode: Int = 33 + /** The JVM reports a death by signal N as exit status 128 + N. */ + private val SignalDeathStatus: Int = 128 + sealed trait Outcome object Outcome { - case object Skip extends Outcome - final case class Replace(newBlob: ObjectId) extends Outcome + case object Skip extends Outcome + final case class Replace(newBlob: ObjectId) extends Outcome final case class Abort(stderr: String, exitCode: Int) extends Outcome } - /** - * Run `command` against `bytes`. Pure aside from the child process and the - * jgit inserter, which must be confined to the calling thread. - */ + /** Thread-safe if `inserter` is confined to the calling thread. */ def run( bytes: Array[Byte], origSha: String, @@ -46,27 +37,43 @@ object BlobExec { inserter: ObjectInserter, timeoutSeconds: Int = DefaultTimeoutSeconds, onTimeout: () => Unit = () => (), - onParserCrash: () => Unit = () => () + onParserCrash: () => Unit = () => (), + workerPool: Option[TokenizerWorkerPool] = None ): Outcome = { val env = Seq("BFG_BLOB" -> origSha, "BFG_FILENAME" -> filename, "BFG_PATH" -> fullPath) + val ran = workerPool match { + case Some(pool) => pool.invoke(bytes, filename, timeoutSeconds) + case None => new ChildRunner(timeoutSeconds).run(command, bytes, env) + } - new ChildRunner(timeoutSeconds).run(command, bytes, env) match { + ran match { case ChildRunner.Outcome.Killed(why) => + // Not routed through `abortOnError`: one wedged blob must not abort a long run. System.err.println( - s"Warning: command [$command] on blob $origSha at path [$fullPath] gave no usable " + - s"result ($why): blob left untokenized" + s"Warning: command [$command] timed out on blob $origSha at path [$fullPath] ($why): " + + "blob excluded" ) onTimeout() Outcome.Skip case ChildRunner.Outcome.Exited(ParserCrashExitCode, _, stderr) => - reportParserCrash(command, origSha, fullPath, s"reported a parser crash (exit $ParserCrashExitCode)", stderr) + // Skip like a timeout, but counted apart: more time never fixes a crash. + System.err.println( + s"Warning: command [$command] reported a parser crash (exit $ParserCrashExitCode) on blob " + + s"$origSha at path [$fullPath]: srcML died or produced no tokens, so this blob is excluded " + + "rather than written as an empty tokenization" + ) + printStderr(command, origSha, fullPath, stderr) onParserCrash() Outcome.Skip - case ChildRunner.Outcome.Exited(0, stdout, stderr) if stdout.isEmpty && bytes.nonEmpty => - reportParserCrash(command, origSha, fullPath, - s"reported a parser crash: exited 0 with no output for a ${bytes.length}-byte blob", stderr) + case ChildRunner.Outcome.Exited(status, _, stderr) if status > SignalDeathStatus => + // A kill from outside (e.g. the OOM killer) must not keep the source as tokens. + System.err.println( + s"Warning: command [$command] was killed by signal ${status - SignalDeathStatus} on blob " + + s"$origSha at path [$fullPath]: blob excluded" + ) + printStderr(command, origSha, fullPath, stderr) onParserCrash() Outcome.Skip @@ -74,6 +81,15 @@ object BlobExec { logError(command, origSha, fullPath, status, stderr) if (abortOnError) Outcome.Abort(stderr, status) else Outcome.Skip + case ChildRunner.Outcome.Exited(_, stdout, _) if stdout.isEmpty && bytes.nonEmpty => + System.err.println( + s"Warning: command [$command] exited 0 but produced no output for the ${bytes.length}-byte " + + s"blob $origSha at path [$fullPath]. Refusing to write an empty tokenization: counting " + + "this as a parser crash and excluding the blob." + ) + onParserCrash() + Outcome.Skip + case ChildRunner.Outcome.Exited(_, stdout, _) if JavaArrays.equals(bytes, stdout) => Outcome.Skip @@ -82,20 +98,6 @@ object BlobExec { } } - private def reportParserCrash( - command: String, - origSha: String, - fullPath: String, - what: String, - stderr: String - ): Unit = { - System.err.println( - s"Warning: command [$command] $what on blob $origSha at path [$fullPath]: " + - "blob left untokenized rather than written as an empty tokenization" - ) - printStderr(command, origSha, fullPath, stderr) - } - private def logError( command: String, origSha: String, diff --git a/blobExec/src/main/scala/cregit/blobexec/Main.scala b/blobExec/src/main/scala/cregit/blobexec/Main.scala index c608e0358..96fe44408 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Main.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Main.scala @@ -1,376 +1,485 @@ package cregit.blobexec import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.{Constants, ObjectId} import org.eclipse.jgit.storage.file.FileRepositoryBuilder -import java.nio.file.{Files, Paths} - -/** - * From-scratch rewriter for the cregit pipeline (step 2). Replaces the - * previous in-place BFG-based rewrite with a jgit walk that builds a new - * destination repo and persists `(orig → new)` mappings to SQLite, so a - * subsequent invocation can resume incrementally. - * - * blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] - * - * --abort-on-error exit immediately (status 2) on the first non-zero - * exit from , instead of skipping that blob - * path to the bare source repo (read-only) - * path to the bare destination repo (created if missing) - * path to the SQLite mapping file (created if missing) - * absolute path to the per-blob script to run - * regex matched against each blob's filename - * - * For each blob whose filename matches `fileMaskRegex`, `command` is invoked - * with the blob bytes on stdin and env vars `BFG_BLOB` (orig sha) + - * `BFG_FILENAME`. Its stdout becomes the new blob. Non-zero exit or - * identical-output leaves the blob unchanged. The new commit message gets - * `Former-commit-id: ` appended (BFG-compatible). - */ -object Main { - - /** Memo `meta` key holding the cumulative number of blobs whose command was - * killed for exceeding its budget. Durable because the skip is durable. */ - private[blobexec] val BlobsTimedOutMetaKey = "blobs_timed_out" - - /** Exit status when any blob in this memo has ever timed out: the walk itself - * succeeded, but the output is incomplete and must not be validated. */ - private[blobexec] val TimedOutExitStatus = 4 - - private[blobexec] val ParserCrashedExitStatus = 6 +import java.nio.file.{Files, Path, Paths} +import scala.util.Try - /** Upper bound on either timeout flag. A week is already far past any real - * budget, and bounding the input here is what keeps every window derived from - * it inside `Int`. */ - private[blobexec] val MaxTimeoutSeconds: Int = 7 * 86400 +/** Step 2 of the cregit pipeline: rewrites into through the + * per-blob , and records (orig -> new) ids in SQLite so a rerun resumes. */ +object Main { - /** Seconds for `--blob-timeout=` / `--stall-timeout=`: a positive whole number - * within [[MaxTimeoutSeconds]], else None. Zero is rejected rather than read - * as "no limit". */ + /** Not 3 (memo mismatch): a no-op request must be told apart from a refusal. */ + private[blobexec] val RetokenizeIneffectiveExitStatus = 7 - private[blobexec] val ToleratedCrashRate = 0.01 + private val UsageExitStatus = 1 + private val RefusedExitStatus = 3 - private[blobexec] def toleratedCrashes(blobsTokenized: Long): Long = - math.max(1L, (blobsTokenized * ToleratedCrashRate).toLong) + /** 2 on an abort, else 0: excluded blobs are named on EXCLUDED lines, not failed. */ + private[blobexec] def exitStatus(stats: WalkStats): Int = + if (stats.aborted) 2 else 0 - private[blobexec] def exitStatus(stats: WalkStats): Int = { - val tokenized = stats.blobsRunThroughCommand.toLong + stats.blobsCacheHit.toLong - if (stats.aborted) 2 - else if (stats.blobsTimedOut > 0) TimedOutExitStatus - else if (stats.blobsParserCrashed > toleratedCrashes(tokenized)) ParserCrashedExitStatus - else 0 + /** Zero and negatives are refused rather than read as "no limit". */ + private[blobexec] def parsePositiveSeconds(spec: String): Option[Int] = + spec.toIntOption.filter(_ > 0) + + /** Ratio of the two defaults (600 and 1800). */ + private[blobexec] val StallTimeoutMultiple = 3 + + /** An explicit window below the floor is refused; a defaulted one is widened to + * [[widenedStall]]. Overriding a number the operator typed is worse than stopping. */ + private[blobexec] def resolveStallTimeout( + blobTimeoutSeconds: Int, + stallTimeoutSeconds: Int, + stallExplicit: Boolean + ): Either[String, Int] = { + val floor = Walker.stallFloorFor(blobTimeoutSeconds) + if (stallTimeoutSeconds >= floor) Right(stallTimeoutSeconds) + else if (stallExplicit) + Left( + s"--stall-timeout=$stallTimeoutSeconds must be at least ${floor}s to go with " + + s"--blob-timeout=$blobTimeoutSeconds. A blob completing or being killed is the only " + + "progress a pure-blob commit makes, and killing one takes the budget plus the kill " + + "grace, so a smaller stall window kills runs whose blobs are merely slow. Try " + + s"--stall-timeout=${widenedStall(blobTimeoutSeconds)} with --blob-timeout=$blobTimeoutSeconds." + ) + else Right(widenedStall(blobTimeoutSeconds)) } - /** Value parser for the `--blob-timeout=` / `--stall-timeout=` seconds: a - * positive whole number, else None (which the caller reports and exits 1 on). - * Zero and negatives are rejected rather than read as "no limit" — an - * unbounded blob is the defect this whole change exists to remove. */ - private[blobexec] def parsePositiveSeconds(spec: String): Option[Int] = - spec.toIntOption.filter(s => s > 0 && s <= MaxTimeoutSeconds) + private[blobexec] def widenedStall(blobTimeoutSeconds: Int): Int = + math.max( + Saturating.product(math.max(1, blobTimeoutSeconds), StallTimeoutMultiple), + Walker.stallFloorFor(blobTimeoutSeconds) + ) - // `raw` (not `s`): the mask example below contains a regex backslash, which a - // processed-escape interpolator rejects. `$$` therefore renders a literal `$`. - private val Usage = - raw"""Usage: blobExec [--abort-on-error] [--pipeline | --pipeline-trees | --shard=K/N] [--warm=] [--blob-timeout=] [--stall-timeout=] + // `raw`: the mask example holds a regex backslash; `$$` renders a literal `$`. + private[blobexec] val Usage = + raw"""Usage: blobExec [options] | - | --abort-on-error exit immediately (status 2) on the first non-zero - | exit from , instead of skipping that blob - | --blob-timeout= - | budget for one invocation (default ${BlobExec.DefaultTimeoutSeconds}). - | A child that exceeds it is killed with everything - | beneath it; that blob is left untokenized, the walk - | records nothing for its commit, and the run exits - | ${TimedOutExitStatus} so an incomplete project cannot be published. - | Re-running retries exactly those blobs. - | --stall-timeout= - | watchdog window (default ${Walker.DefaultStallTimeoutSeconds}, derived from - | --blob-timeout). If nothing completes anywhere in this - | window the run exits ${Walker.StalledExitStatus}. Neither flag normally - | needs setting: the window follows the budget, and a - | window that one blob's lifetime could trip is refused. + | --abort-on-error exit 2 on the first error from instead of skipping the blob + | --pipeline look-ahead pipelined walker; same output as the serial walker + | --pipeline-trees as --pipeline, and also assemble trees on the worker pool + | --shard=K/N tree-only run of shard K (0-based) of N commit ranges, no commits or + | refs (merge with shard_merge.py); not with --pipeline(-trees) + | --warm= read-only fallback mapping DB, consulted on a blob_map/tree_map miss + | --mask-widened resume under a wider , keeping blob_map's tokens; + | refused (status 3) unless every tokenized path still matches the + | new mask and holds the kept ids + | --tokenizer-identity==[,...] + | tokenizer digest per extension (tokenize/tokenizerIdentity.pl); a + | change since the cache was built refuses the run (status 3) + | --retokenize=[,...] drop these extensions' cached tokens (blob_map and memo); needs + | --tokenizer-identity and --memo-dir; a no-op is refused (status ${RetokenizeIneffectiveExitStatus}) + | --memo-dir= the tokenizer memo ($$BFG_MEMO_DIR) that --retokenize purges + | --tokenizer-worker= tokenize through a pool of persistent workers; needs --pipeline(-trees) + | --blob-timeout= budget for one run (default ${BlobExec.DefaultTimeoutSeconds}); then its process + | tree is killed and the blob excluded + | --stall-timeout= exit ${Walker.StalledExitStatus} after this long with no progress (default ${Walker.DefaultStallTimeoutSeconds}); + | must cover one blob's lifetime: below it is refused, a default raised + | bare source repository (read-only) + | bare destination repository (created if missing, reused on resume) + | SQLite mapping file (created if missing, reused on resume) + | absolute path of the per-blob command + | regex matched against each blob's filename (e.g. '\.[ch]$$') | - | Blobs on the shipped denylist (${BlobDenylist.EntriesSource}) are never - | handed to : they are dropped from the rewritten trees, counted as - | blobsDenylisted, named with their reason and citation on an EXCLUDED line, - | and they do NOT change the exit status. srcML 1.1.0 does not terminate on - | the listed blobs, the defect is diagnosed and cited upstream, and a - | diagnosed exclusion must not block publication the way an unexplained - | timeout does. + | Denylisted (${BlobDenylist.EntriesSource}) and oversized blobs, and blobs whose tokenizer + | times out or crashes, are left out of the trees and named on an EXCLUDED line; they do not + | change the exit status. | - | Exit status: 0 = clean, 1 = usage, 2 = aborted on a command error, - | 3 = memo meta mismatch, ${TimedOutExitStatus} = completed - | but some blob timed out (output incomplete, re-run to - | retry), ${Walker.StalledExitStatus} = killed by the stall watchdog, - | ${ParserCrashedExitStatus} = tokenizer crashes past ${(ToleratedCrashRate * 100).toInt}% of the blobs - | tokenized. Below that the blob is excluded and the run - | still publishes: a crash is deterministic, so one - | pathological input must not cost a whole project. - | --pipeline use the look-ahead pipelined walker (producer runs - | ahead so the blob-command pool stays saturated); - | output is identical to the default serial walker - | --pipeline-trees as --pipeline, but also assembles rewritten trees on - | the worker pool (Design B: only commit construction - | and the ordered DB write stay on the consumer); - | output is identical to the default serial walker - | --shard=K/N TREE-ONLY history shard: partition the commits (in - | canonical TOPO+REVERSE order) into N contiguous - | ranges and process only shard K (0-based). Tokenizes - | blobs and builds+persists trees (blob_map + tree_map - | incl. subtree ids) for the slice, but does NOT fold - | commits (no commit_map, no refs). Run N shards to - | their own dst.git + db.sqlite, then merge + serial - | re-fold (shard_merge.py) for byte-identical output. - | Uses the flat-memory serial walker; mutually - | exclusive with --pipeline / --pipeline-trees. - | --warm= optional read-only fallback DB (a frozen prior memo, - | e.g. the paused whole-kernel run). On a blob_map / - | tree_map miss the lookup falls through to this DB, so - | a shard skips re-tokenizing content already done. - | Never written; commit_map is never consulted. - | bare source repo (read-only) - | bare destination repo (created on first run, reused on incremental) - | SQLite mapping file (created on first run, reused on incremental) - | absolute path to the per-blob script to run - | regex matched against each blob's filename (e.g. '\.[ch]$$') + | Exit status: 0 clean, 1 usage, 2 aborted on a command error, 3 cache mismatch, + | ${Walker.StalledExitStatus} stalled, ${RetokenizeIneffectiveExitStatus} --retokenize would invalidate nothing. |""".stripMargin - def main(args: Array[String]): Unit = { - val (flags, positional) = args.partition(_.startsWith("-")) + private[blobexec] final case class Options( + abortOnError: Boolean = false, + pipeline: Boolean = false, + pipelineTrees: Boolean = false, + shard: Option[(Int, Int)] = None, + warm: Option[Path] = None, + maskWidened: Boolean = false, + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenize: Set[String] = Set.empty, + memoDir: Option[Path] = None, + tokenizerWorker: Option[Path] = None, + blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, + stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, + stallExplicit: Boolean = false, + positional: Vector[String] = Vector.empty + ) { + def src: Path = Paths.get(positional(0)) + def dst: Path = Paths.get(positional(1)) + def db: Path = Paths.get(positional(2)) + def command: String = positional(3) + def mask: String = positional(4) + + def stallWindow: Either[String, Int] = + resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) + + def anyPipeline: Boolean = pipeline || pipelineTrees + } - var abortOnError = false - var pipeline = false - var pipelineTrees = false - var shard: Option[(Int, Int)] = None - var warmPath: Option[java.nio.file.Path] = None - var blobTimeoutSeconds = BlobExec.DefaultTimeoutSeconds - var stallTimeoutSeconds = Walker.DefaultStallTimeoutSeconds - var stallExplicit = false - flags.foreach { - case "--abort-on-error" => abortOnError = true - case "--pipeline" => pipeline = true - case "--pipeline-trees" => pipelineTrees = true - case s if s.startsWith("--shard=") => - val spec = s.stripPrefix("--shard=") - spec.split("/", -1) match { - case Array(kStr, nStr) => - val k = kStr.toIntOption.getOrElse(-1) - val n = nStr.toIntOption.getOrElse(-1) - if (n < 1 || k < 0 || k >= n) { - System.err.println(s"Error: --shard must be K/N with 0 <= K < N and N >= 1 [$spec]") - sys.exit(1) - } - shard = Some((k, n)) - case _ => - System.err.println(s"Error: --shard must be of the form K/N [$spec]") - sys.exit(1) - } - case w if w.startsWith("--warm=") => - val p = Paths.get(w.stripPrefix("--warm=")) - if (!Files.isRegularFile(p)) { - System.err.println(s"Error: --warm db [$p] is not a file") - sys.exit(1) - } - warmPath = Some(p) - case t if t.startsWith("--blob-timeout=") => - val spec = t.stripPrefix("--blob-timeout=") - parsePositiveSeconds(spec) match { - case Some(secs) => blobTimeoutSeconds = secs - case None => - System.err.println(s"Error: --blob-timeout must be a positive whole number of seconds [$spec]") - sys.exit(1) - } - case t if t.startsWith("--stall-timeout=") => - val spec = t.stripPrefix("--stall-timeout=") - parsePositiveSeconds(spec) match { - case Some(secs) => stallTimeoutSeconds = secs; stallExplicit = true - case None => - System.err.println(s"Error: --stall-timeout must be a positive whole number of seconds [$spec]") - sys.exit(1) - } - case other => - System.err.println(s"Error: unknown flag [$other]") - System.err.println(Usage) - sys.exit(1) - } + private object Messages { + def withUsage(error: String): String = s"$error\n$Usage" + + def unknownFlag(flag: String): String = withUsage(s"Error: unknown flag [$flag]") + def badFlagValue(flag: String, why: String): String = s"Error: $flag: $why" + + def secondsWanted(spec: String): String = s"must be a positive whole number of seconds [$spec]" + def shardRange(spec: String): String = s"must be K/N with 0 <= K < N and N >= 1 [$spec]" + def shardForm(spec: String): String = s"must be of the form K/N [$spec]" + + val NotAFile = "is not a file" + val NotTheMemo = + "is not a directory. It must be the SAME memo this project's tokenizations were written " + + "into, or the invalidation would leave the real memo's stale entries in place." + + val PipelinesExclusive = withUsage("Error: --pipeline and --pipeline-trees are mutually exclusive") + val ShardWithPipeline = withUsage( + "Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") + val MaskWidenedUnderShard = withUsage( + "Error: --mask-widened has nothing to do under --shard: each shard builds a fresh dst.git and " + + "blobmap.db, so no mask is recorded to widen. Reuse a prior run's tokenizations with --warm= instead.") + + val RetokenizeNeedsIdentity = withUsage( + "Error: --retokenize needs --tokenizer-identity. Invalidating the entries without recording which " + + "tokenizer replaces them leaves nothing for the next run to detect a change against, so the next " + + "tokenizer defect would be just as silent as this one.") + val RetokenizeNeedsMemoDir = withUsage( + "Error: --retokenize needs --memo-dir. There are two caches, not one: dropping a blob_map row makes " + + "the walker re-run , and is tokenizeByBlobId/tokenBySha.pl, which answers from " + + "$BFG_MEMO_DIR keyed on sha1(contents) with no tokenizer in the key. Invalidating one layer " + + "without the other invalidates nothing at all.") + def retokenizeUncovered(unknown: Set[String], known: Set[String]): String = + s"Error: --retokenize names extension(s) --tokenizer-identity does not cover: " + + s"${unknown.toVector.sorted.mkString(", ")}. Known: ${known.toVector.sorted.mkString(", ")}." + val RetokenizeWithMaskWidened = + "Error: --mask-widened and --retokenize cannot be combined. Each verifies its own precondition " + + "against the rows, and together each would verify against a state the other is about to " + + "change. Widen first, then resume with --retokenize." + val RetokenizeUnderShard = + "Error: --retokenize has nothing to do under --shard: each shard builds a fresh dst.git and " + + "blobmap.db, so there are no cached tokenizations to invalidate. Retokenize the merged result, " + + "or drop the shards' --warm=." + val MemoDirWithoutRetokenize = + "Error: --memo-dir has no effect without --retokenize. Nothing else in blobExec reads the memo " + + "— tokenizeByBlobId/tokenBySha.pl takes it from $BFG_MEMO_DIR in the environment." + + def workerNotExecutable(path: Path): String = s"Error: --tokenizer-worker [$path] is not an executable file" + val WorkerNeedsPipeline = "Error: --tokenizer-worker needs --pipeline or --pipeline-trees" + + def srcNotADirectory(path: Path): String = s"Error: src repo [$path] is not a directory" + def commandMissing(command: String): String = s"Error: command [$command] does not exist" + val EmptyMask = "Error: fileMaskRegex must be non-empty" + + def stallRaised(from: Int, to: Int, blobTimeoutSeconds: Int): String = + s"blobExec: raising the stall window from ${from}s to ${to}s, because " + + s"--blob-timeout=${blobTimeoutSeconds}s needs a window that clears one blob's whole lifetime. " + + "Pass --stall-timeout explicitly to choose your own." + + def error(e: Throwable): String = s"Error: ${e.getMessage}" + val MaskWidenedHint = + "Hint: if the new mask is a strict superset of the recorded one, --mask-widened reuses the " + + "tokenizations already in blob_map instead of redoing them. It verifies that against the rows " + + "themselves and refuses if it is not true. It requires the work directory to be intact (resume " + + "at step 2), because blob_map's ids live in dst." + + def denylisted(count: Long, entries: Int): String = + s"blobExec: $count blob(s) were excluded by the blob denylist " + + s"(${BlobDenylist.EntriesSource}, $entries entr${if (entries == 1) "y" else "ies"}). Each one is " + + "named with its sha, path, reason and upstream citation on an 'EXCLUDED denylisted blob' line " + + "above; those lines and that list are the record of what this project's dataset does not " + + "contain. The files are absent from the tokenized repository, not present as raw source, so " + + "they produce no blame and no dataset row. This is not a failure and does not affect the exit status." + + def oversized(count: Long): String = + s"blobExec: $count blob(s) were excluded as oversized (>= ${Walker.MaxBlobBytes} bytes, JGit's " + + "stream-file threshold). Each one is named with its sha, path and size on an 'EXCLUDED " + + "oversized blob' line above; those lines are the record of what this project's dataset does " + + "not contain. The files are absent from the tokenized repository, not present as raw source, " + + "so they produce no blame and no dataset row. This is not a failure and does not affect the exit status." + + def failed(timedOut: Long, crashed: Long, blobTimeoutSeconds: Int): String = + s"blobExec: ${timedOut + crashed} blob(s) were excluded because their tokenizer failed " + + s"($timedOut timed out after ${blobTimeoutSeconds}s, $crashed reported a parser crash). Each " + + "one is named on an 'EXCLUDED failed blob' line above; those lines are the record of what " + + "this project's dataset does not contain. This does not affect the exit status." + } + + /** `--name=` as a pattern: `case Flag(value) =>`. */ + private final class ValuedFlag(val name: String) { + private val prefix = name + "=" + def unapply(arg: String): Option[String] = Option.when(arg.startsWith(prefix))(arg.substring(prefix.length)) + } - Walker.resolveStallTimeout(blobTimeoutSeconds, stallTimeoutSeconds, stallExplicit) match { - case Right(secs) => - if (secs != stallTimeoutSeconds) { - System.err.println( - s"blobExec: raising the stall window from ${stallTimeoutSeconds}s to ${secs}s, because " + - s"--blob-timeout=${blobTimeoutSeconds}s needs a watchdog window larger than itself. " + - "Pass --stall-timeout explicitly to choose your own." - ) - stallTimeoutSeconds = secs + private val TokenizerIdentityFlag = new ValuedFlag("--tokenizer-identity") + private val RetokenizeFlag = new ValuedFlag("--retokenize") + private val MemoDirFlag = new ValuedFlag("--memo-dir") + private val ShardFlag = new ValuedFlag("--shard") + private val WarmFlag = new ValuedFlag("--warm") + private val TokenizerWorkerFlag = new ValuedFlag("--tokenizer-worker") + private val BlobTimeoutFlag = new ValuedFlag("--blob-timeout") + private val StallTimeoutFlag = new ValuedFlag("--stall-timeout") + + private def flagValue[A](flag: ValuedFlag, spec: String)(parse: String => Either[String, A]): Either[String, A] = + parse(spec).left.map(Messages.badFlagValue(flag.name, _)) + + private def seconds(spec: String): Either[String, Int] = + parsePositiveSeconds(spec).toRight(Messages.secondsWanted(spec)) + + private def shardSpec(spec: String): Either[String, (Int, Int)] = + spec.split("/", -1) match { + case Array(k, n) => + (k.toIntOption, n.toIntOption) match { + case (Some(k), Some(n)) if n >= 1 && k >= 0 && k < n => Right((k, n)) + case _ => Left(Messages.shardRange(spec)) } - case Left(why) => - System.err.println(s"Error: $why") - sys.exit(1) + case _ => Left(Messages.shardForm(spec)) } - if (pipeline && pipelineTrees) { - System.err.println("Error: --pipeline and --pipeline-trees are mutually exclusive") - System.err.println(Usage) - sys.exit(1) - } + private def existingPath(isValid: Path => Boolean, otherwise: String)(spec: String): Either[String, Path] = { + val path = Paths.get(spec) + if (isValid(path)) Right(path) else Left(s"[$path] $otherwise") + } - if (shard.isDefined && (pipeline || pipelineTrees)) { - System.err.println("Error: --shard uses the serial tree-only walker and cannot be combined with --pipeline / --pipeline-trees") - System.err.println(Usage) - sys.exit(1) - } + private def parseFlag(o: Options, flag: String): Either[String, Options] = flag match { + case "--abort-on-error" => Right(o.copy(abortOnError = true)) + case "--pipeline" => Right(o.copy(pipeline = true)) + case "--pipeline-trees" => Right(o.copy(pipelineTrees = true)) + case "--mask-widened" => Right(o.copy(maskWidened = true)) + case TokenizerIdentityFlag(spec) => + flagValue(TokenizerIdentityFlag, spec)(TokenizerIdentity.parse).map(id => o.copy(tokenizerIdentity = id)) + case RetokenizeFlag(spec) => + flagValue(RetokenizeFlag, spec)(TokenizerIdentity.parseExtensions).map(exts => o.copy(retokenize = exts)) + case MemoDirFlag(spec) => + flagValue(MemoDirFlag, spec)(existingPath(Files.isDirectory(_), Messages.NotTheMemo)) + .map(dir => o.copy(memoDir = Some(dir))) + case ShardFlag(spec) => + flagValue(ShardFlag, spec)(shardSpec).map(kn => o.copy(shard = Some(kn))) + case WarmFlag(spec) => + flagValue(WarmFlag, spec)(existingPath(Files.isRegularFile(_), Messages.NotAFile)) + .map(db => o.copy(warm = Some(db))) + case TokenizerWorkerFlag(spec) => + Right(o.copy(tokenizerWorker = Some(Paths.get(spec)))) + case BlobTimeoutFlag(spec) => + flagValue(BlobTimeoutFlag, spec)(seconds).map(secs => o.copy(blobTimeoutSeconds = secs)) + case StallTimeoutFlag(spec) => + flagValue(StallTimeoutFlag, spec)(seconds).map(secs => o.copy(stallTimeoutSeconds = secs, stallExplicit = true)) + case other => Left(Messages.unknownFlag(other)) + } - if (positional.length != 5) { - System.err.println(Usage) - sys.exit(1) - } - val srcPath = Paths.get(positional(0)) - val dstPath = Paths.get(positional(1)) - val dbPath = Paths.get(positional(2)) - val command = positional(3) - val mask = positional(4) - - if (!Files.isDirectory(srcPath)) { - System.err.println(s"Error: src repo [$srcPath] is not a directory") - sys.exit(1) - } - if (!Files.exists(Paths.get(command))) { - System.err.println(s"Error: command [$command] does not exist") - sys.exit(1) - } - if (mask.isEmpty) { - System.err.println("Error: fileMaskRegex must be non-empty") - sys.exit(1) - } - // Files.createDirectories throws FileAlreadyExistsException on macOS - // when the target is a symlink (e.g. /tmp -> /private/tmp). Guard - // against that with an explicit isDirectory check. - val dbParent = dbPath.getParent - if (dbParent != null && !Files.isDirectory(dbParent)) Files.createDirectories(dbParent) - - val denylist = BlobDenylist.shipped - - val incremental = Files.isDirectory(dstPath) - val shardStr = shard.map { case (k, n) => s"$k/$n" }.getOrElse("none") - val warmStr = warmPath.map(_.toString).getOrElse("none") - println( - s"blobExec: src=$srcPath dst=$dstPath db=$dbPath command=$command mask=$mask " + - s"abortOnError=$abortOnError pipeline=$pipeline pipelineTrees=$pipelineTrees " + - s"shard=$shardStr warm=$warmStr blobTimeout=${blobTimeoutSeconds}s " + - s"stallTimeout=${stallTimeoutSeconds}s incremental=$incremental " + - s"denylistEntries=${denylist.size}" + /** The first rule a combination of flags breaks, in the order they are checked. */ + private def combinationError(o: Options): Option[String] = { + val retokenizing = o.retokenize.nonEmpty + val uncovered = o.retokenize -- o.tokenizerIdentity.extensions + val rules: Seq[(Boolean, () => String)] = Seq( + (o.pipeline && o.pipelineTrees) -> (() => Messages.PipelinesExclusive), + (o.shard.isDefined && o.maskWidened) -> (() => Messages.MaskWidenedUnderShard), + (retokenizing && o.tokenizerIdentity.isEmpty) -> (() => Messages.RetokenizeNeedsIdentity), + (retokenizing && o.memoDir.isEmpty) -> (() => Messages.RetokenizeNeedsMemoDir), + (retokenizing && uncovered.nonEmpty) -> + (() => Messages.retokenizeUncovered(uncovered, o.tokenizerIdentity.extensions)), + (retokenizing && o.maskWidened) -> (() => Messages.RetokenizeWithMaskWidened), + (retokenizing && o.shard.isDefined) -> (() => Messages.RetokenizeUnderShard), + (!retokenizing && o.memoDir.isDefined) -> (() => Messages.MemoDirWithoutRetokenize), + (o.shard.isDefined && o.anyPipeline) -> (() => Messages.ShardWithPipeline), + o.tokenizerWorker.exists(p => !Files.isRegularFile(p) || !Files.isExecutable(p)) -> + (() => Messages.workerNotExecutable(o.tokenizerWorker.get)), + (o.tokenizerWorker.isDefined && !o.anyPipeline) -> (() => Messages.WorkerNeedsPipeline) ) + rules.collectFirst { case (true, message) => message() } + } - val src: FileRepository = openSrc(srcPath) - val dst: FileRepository = openOrInitDst(dstPath) - val mapping = try Mapping.open(dbPath, command, mask, warmPath) catch { - case m: Mapping.MetaMismatchException => - System.err.println(s"Error: ${m.getMessage}") - src.close(); dst.close() - sys.exit(3) - } + private def positionalError(o: Options): Option[String] = + if (!Files.isDirectory(o.src)) Some(Messages.srcNotADirectory(o.src)) + else if (!Files.exists(Paths.get(o.command))) Some(Messages.commandMissing(o.command)) + else if (o.mask.isEmpty) Some(Messages.EmptyMask) + else None - // The cumulative figure is forensic only: gating on it would make one past - // timeout permanent. - val (stats, timedOutEver) = try { - val parallelism = math.max(1, Runtime.getRuntime.availableProcessors) - val walker = new Walker( - src, dst, mapping, mask.r, command, abortOnError, parallelism, - pipeline, pipelineTrees, shard, - destinationMayContainObjects = incremental, - blobTimeoutSeconds = blobTimeoutSeconds, - stallTimeoutSeconds = stallTimeoutSeconds, - denylist = denylist - ) - val s = walker.run() - val prior = mapping.getMeta(BlobsTimedOutMetaKey).flatMap(_.toLongOption).getOrElse(0L) - val total = prior + s.blobsTimedOut - if (s.blobsTimedOut > 0) mapping.setMeta(BlobsTimedOutMetaKey, total.toString) - (s, total) - } finally { - mapping.close() - dst.close() - src.close() + /** Left is the whole text to print before exiting with the usage status. */ + private[blobexec] def parse(args: Seq[String]): Either[String, Options] = { + val (flags, positional) = args.partition(_.startsWith("-")) + for { + parsed <- flags.foldLeft[Either[String, Options]](Right(Options()))((acc, f) => acc.flatMap(parseFlag(_, f))) + _ <- parsed.stallWindow.left.map(why => s"Error: $why") + _ <- combinationError(parsed).toLeft(()) + _ <- Either.cond(positional.length == 5, (), Usage) + full = parsed.copy(positional = positional.toVector) + _ <- positionalError(full).toLeft(()) + } yield full + } + + def main(args: Array[String]): Unit = { + val options = parse(args.toVector) match { + case Right(o) => o + case Left(why) => exit(why, UsageExitStatus) } + val stats = run(options, stallSeconds(options)) + // Exit explicitly: a timed-out blob's daemon reader threads can keep the JVM alive. + sys.exit(exitStatus(stats)) + } - println( - s"blobExec done: commitsProcessed=${stats.commitsProcessed} " + - s"blobsRunThroughCommand=${stats.blobsRunThroughCommand} " + - s"blobCommandExecutions=${stats.blobCommandExecutions} " + - s"blobsCacheHit=${stats.blobsCacheHit} " + - s"originalBlobCopyRequests=${stats.originalBlobCopyRequests} " + - s"originalBlobCopies=${stats.originalBlobCopies} " + - s"originalBlobAlreadyPresent=${stats.originalBlobAlreadyPresent} " + - s"originalBlobCacheHits=${stats.originalBlobCacheHits} " + - s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + - s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + - s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + - s"refsProjected=${stats.refsProjected} " + - s"blobsTimedOut=${stats.blobsTimedOut} " + - s"blobsTimedOutEver=$timedOutEver " + - s"blobsOversized=${stats.blobsOversized} " + - s"blobsDenylisted=${stats.blobsDenylisted} " + - s"blobsParserCrashed=${stats.blobsParserCrashed} " + - s"aborted=${stats.aborted}" - ) + private def exit(message: String, status: Int): Nothing = { + System.err.println(message) + sys.exit(status) + } - if (stats.blobsDenylisted > 0) { - // Reported, never fatal. - System.err.println( - s"blobExec: ${stats.blobsDenylisted} blob(s) were excluded by the blob denylist " + - s"(${BlobDenylist.EntriesSource}, ${denylist.size} entr" + - s"${if (denylist.size == 1) "y" else "ies"}). Each one is named with its sha, path, " + - "reason and upstream citation on an 'EXCLUDED denylisted blob' line above; those " + - "lines and that file are the record of what this project's dataset does not contain. " + - "The files are absent from the tokenized repository, not present as raw source, so " + - "they produce no blame and no dataset row. This is not a failure and does not affect " + - "the exit status." - ) - } + private def stallSeconds(o: Options): Int = { + val secs = o.stallWindow.getOrElse(o.stallTimeoutSeconds) + if (secs != o.stallTimeoutSeconds) + System.err.println(Messages.stallRaised(o.stallTimeoutSeconds, secs, o.blobTimeoutSeconds)) + secs + } - if (stats.blobsOversized > 0) { - System.err.println( - s"blobExec: ${stats.blobsOversized} blob(s) were excluded as oversized. Each is named " + - "with its sha, path and size on an 'EXCLUDED oversized blob' line above. Reported, " + - "never fatal." - ) - } + private def run(o: Options, stallSeconds: Int): WalkStats = { + createParentDirectory(o.db) + val incremental = Files.isDirectory(o.dst) + println(startLine(o, stallSeconds, incremental)) - if (stats.blobsTimedOut > 0) { - System.err.println( - s"blobExec: INCOMPLETE, DO NOT PUBLISH: ${stats.blobsTimedOut} blob(s) timed out this " + - s"run ($timedOutEver ever for this memo). Resume at step 2 to retry exactly those " + - s"blobs, or raise --blob-timeout (currently ${blobTimeoutSeconds}s)." - ) + val src = openSrc(o.src) + val dst = openOrInitDst(o.dst) + val mapping = openMapping(o, src, dst) match { + case Right(m) => m + case Left(refusal) => + src.close(); dst.close() + exit(refusal.message, refusal.status) } - if (stats.blobsParserCrashed > 0) { - val tokenized = stats.blobsRunThroughCommand.toLong + stats.blobsCacheHit.toLong - val gated = stats.blobsParserCrashed > toleratedCrashes(tokenized) - System.err.println( - s"blobExec: ${stats.blobsParserCrashed} of $tokenized blob(s) crashed their tokenizer " + - s"and were excluded; each is named above. " + - (if (gated) s"That is past the ${ToleratedCrashRate * 100}% this run tolerates: " + - "DO NOT PUBLISH. Deterministic, so re-running will not clear it." - else "Within tolerance, so the run still publishes.") - ) - } + val stats = + try walk(o, stallSeconds, src, dst, mapping, incremental) + finally { mapping.close(); dst.close(); src.close() } + println(doneLine(stats)) + reportExclusions(stats, o.blobTimeoutSeconds) + stats + } - val status = exitStatus(stats) - if (status != 0) System.err.println(s"blobExec: exiting $status") - // Exit explicitly: abandoned daemon readers must not decide JVM exit. - sys.exit(status) + // Not Files.createDirectories alone: it throws on macOS when the target is a symlink (/tmp). + private def createParentDirectory(path: Path): Unit = + Option(path.getParent).filterNot(Files.isDirectory(_)).foreach(Files.createDirectories(_)) + + private def startLine(o: Options, stallSeconds: Int, incremental: Boolean): String = { + def orNone(value: Option[Any]): String = value.map(_.toString).getOrElse("none") + s"blobExec: src=${o.src} dst=${o.dst} db=${o.db} command=${o.command} mask=${o.mask} " + + s"abortOnError=${o.abortOnError} pipeline=${o.pipeline} pipelineTrees=${o.pipelineTrees} " + + s"shard=${orNone(o.shard.map { case (k, n) => s"$k/$n" })} warm=${orNone(o.warm)} " + + s"blobTimeout=${o.blobTimeoutSeconds}s stallTimeout=${stallSeconds}s incremental=$incremental " + + s"maskWidened=${o.maskWidened} denylistEntries=${BlobDenylist.shipped.size} " + + s"tokenizerIdentity=${if (o.tokenizerIdentity.isEmpty) "none" else o.tokenizerIdentity.render} " + + s"retokenize=${if (o.retokenize.isEmpty) "none" else o.retokenize.toVector.sorted.mkString(",")} " + + s"memoDir=${orNone(o.memoDir)} tokenizerWorker=${orNone(o.tokenizerWorker)}" + } + + private final case class Refusal(status: Int, message: String) + + private def openMapping(o: Options, src: FileRepository, dst: FileRepository): Either[Refusal, Mapping] = + Try(Mapping.open(o.db, o.command, o.mask, o.warm, widening(o, dst), o.tokenizerIdentity, retokenize(o, src, dst))) + .toEither + .left.map(e => refusalFor(e, o).getOrElse(throw e)) + + private def refusalFor(e: Throwable, o: Options): Option[Refusal] = e match { + case _: Mapping.MetaMismatchException if !o.maskWidened && e.getMessage.contains("mask") => + Some(Refusal(RefusedExitStatus, s"${Messages.error(e)}\n${Messages.MaskWidenedHint}")) + case _: Mapping.MetaMismatchException | _: Mapping.MaskNarrowedException | + _: Mapping.DanglingNewBlobException | _: Mapping.TokenizerChangedException => + Some(Refusal(RefusedExitStatus, Messages.error(e))) + // Not 0: a corpus script would publish entries that were never invalidated. + case _: Mapping.NothingInvalidatedException => Some(Refusal(RetokenizeIneffectiveExitStatus, Messages.error(e))) + // Mapping.open re-checks the flag rules above, for library callers. + case _: IllegalArgumentException => Some(Refusal(UsageExitStatus, Messages.error(e))) + case _ => None + } + + private val report: String => Unit = msg => println(s"blobExec: $msg") + + private def widening(o: Options, dst: FileRepository): Option[Mapping.MaskWidening] = + Option.when(o.maskWidened)(Mapping.MaskWidening(newBlobResolves = hasObject(dst), report = report)) + + // The memo is keyed on content sha1 alone (tokenizeByBlobId/tokenBySha.pl:76), + // so the purge re-reads the original blobs from src. + private def retokenize(o: Options, src: FileRepository, dst: FileRepository): Option[Mapping.Retokenize] = + Option.when(o.retokenize.nonEmpty)(Mapping.Retokenize( + extensions = o.retokenize, + newBlobResolves = hasObject(dst), + purgeMemo = blobs => TokenizerMemo.purge(o.memoDir.get, blobs, blobBytes(src)), + report = report, + memoDirIsTokenizerMemo = o.memoDir.exists(d => TokenizerMemo.isTokenizerMemo(d, sys.env.get("BFG_MEMO_DIR"))) + )) + + private def hasObject(repo: FileRepository)(id: String): Boolean = { + val reader = repo.newObjectReader() + try reader.has(ObjectId.fromString(id)) + catch { case _: IllegalArgumentException => false } + finally reader.close() + } + + private def blobBytes(repo: FileRepository)(sha: String): Option[Array[Byte]] = { + val reader = repo.newObjectReader() + try Some(reader.open(ObjectId.fromString(sha), Constants.OBJ_BLOB).getBytes) + catch { case _: Exception => None } + finally reader.close() + } + + private def walk( + o: Options, + stallSeconds: Int, + src: FileRepository, + dst: FileRepository, + mapping: Mapping, + incremental: Boolean + ): WalkStats = { + val parallelism = math.max(1, Runtime.getRuntime.availableProcessors) + val workerPool = o.tokenizerWorker.map(path => new TokenizerWorkerPool(path.toString, parallelism)) + try + new Walker( + src, dst, mapping, o.mask.r, o.command, o.abortOnError, parallelism, + o.pipeline, o.pipelineTrees, o.shard, + destinationMayContainObjects = incremental, + blobTimeoutSeconds = o.blobTimeoutSeconds, + stallTimeoutSeconds = stallSeconds, + workerPool = workerPool, + denylist = BlobDenylist.shipped + ).run() + finally workerPool.foreach(_.close()) + } + + private def doneLine(stats: WalkStats): String = + s"blobExec done: commitsProcessed=${stats.commitsProcessed} " + + s"blobsRunThroughCommand=${stats.blobsRunThroughCommand} " + + s"blobCommandExecutions=${stats.blobCommandExecutions} " + + s"blobsCacheHit=${stats.blobsCacheHit} " + + s"originalBlobCopyRequests=${stats.originalBlobCopyRequests} " + + s"originalBlobCopies=${stats.originalBlobCopies} " + + s"originalBlobAlreadyPresent=${stats.originalBlobAlreadyPresent} " + + s"originalBlobCacheHits=${stats.originalBlobCacheHits} " + + s"originalBlobDestinationLookups=${stats.originalBlobDestinationLookups} " + + s"originalBlobBytesCopied=${stats.originalBlobBytesCopied} " + + s"originalBlobBytesAvoided=${stats.originalBlobBytesAvoided} " + + s"refsProjected=${stats.refsProjected} " + + s"blobsTimedOut=${stats.blobsTimedOut} " + + s"blobsOversized=${stats.blobsOversized} " + + s"blobsDenylisted=${stats.blobsDenylisted} " + + s"blobsParserCrashed=${stats.blobsParserCrashed} " + + s"aborted=${stats.aborted}" + + private def reportExclusions(stats: WalkStats, blobTimeoutSeconds: Int): Unit = { + if (stats.blobsDenylisted > 0) + System.err.println(Messages.denylisted(stats.blobsDenylisted, BlobDenylist.shipped.size)) + if (stats.blobsOversized > 0) + System.err.println(Messages.oversized(stats.blobsOversized)) + if (stats.blobsTimedOut + stats.blobsParserCrashed > 0) + System.err.println(Messages.failed(stats.blobsTimedOut, stats.blobsParserCrashed, blobTimeoutSeconds)) } - private def openSrc(path: java.nio.file.Path): FileRepository = { + private def openSrc(path: Path): FileRepository = { val gitDir = if (Files.isDirectory(path.resolve(".git"))) path.resolve(".git").toFile else path.toFile FileRepositoryBuilder.create(gitDir).asInstanceOf[FileRepository] } - private def openOrInitDst(path: java.nio.file.Path): FileRepository = { + private def openOrInitDst(path: Path): FileRepository = { val exists = Files.isDirectory(path) val repo = FileRepositoryBuilder.create(path.toFile).asInstanceOf[FileRepository] - if (!exists) repo.create(true) // bare init + if (!exists) repo.create(true) repo } } diff --git a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala index 11814b18d..d6739ae77 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Mapping.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Mapping.scala @@ -2,6 +2,7 @@ package cregit.blobexec import java.nio.file.Path import java.sql.{Connection, DriverManager, PreparedStatement} +import scala.util.matching.Regex /** SQLite-backed persistent mapping between original and rewritten object ids. * One JDBC connection confined to the walker thread (only the external command @@ -104,6 +105,57 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends def setMeta(key: String, value: String): Unit = Mapping.execute(insMeta, key, value) + def storedTokenizerId(extension: String): Option[String] = + getMeta(Mapping.TokenizerIdKeyPrefix + extension) + + def setTokenizerId(extension: String, value: String): Unit = + setMeta(Mapping.TokenizerIdKeyPrefix + extension, value) + + /** Extensions whose recorded identity differs from `identity`. Nothing recorded + * is not a change: older maps carry no identity at all. */ + def tokenizerIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = + identity.byExtension.toVector.sorted.flatMap { case (ext, requested) => + storedTokenizerId(ext) match { + case Some(stored) if stored != requested => + Some(Mapping.IdentityChange(ext, stored, requested)) + case _ => None + } + } + + /** The changed extensions that also have cached tokenizations. Refusing one + * with no rows would deadlock: `--retokenize` of it fails too, as a no-op. */ + def cachePoisoningIdentityChanges(identity: TokenizerIdentity): Vector[Mapping.IdentityChange] = + tokenizerIdentityChanges(identity) + .filter(c => countTokenizedRowsForExtensions(Set(c.extension)) > 0L) + + /** Zero makes a `--retokenize` request a no-op, and a no-op has to fail. */ + def countTokenizedRowsForExtensions(extensions: Set[String]): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery( + s"SELECT COUNT(*) FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + + /** Distinct, because the memo is keyed on content: one blob under two paths is + * one memo entry. */ + def tokenizedOrigBlobsForExtensions(extensions: Set[String]): Vector[String] = { + val st = conn.createStatement() + try { + val rs = st.executeQuery( + s"SELECT DISTINCT orig_blob FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + try Iterator + .continually(if (rs.next()) Some(rs.getString(1)) else None) + .takeWhile(_.isDefined) + .flatten + .toVector + finally rs.close() + } finally st.close() + } + /** All original-commit SHAs already recorded. Used to mark walk frontier. */ def allCommitOrigShas: Vector[String] = { val st = conn.createStatement() @@ -118,6 +170,132 @@ final class Mapping private (conn: Connection, warm: Option[Connection]) extends } finally st.close() } + private def countWhere(predicate: String): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(s"SELECT COUNT(*) FROM blob_map WHERE $predicate") + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + + def rowCounts: Mapping.RowCounts = { + def one(sql: String): Long = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(sql) + try { rs.next(); rs.getLong(1) } finally rs.close() + } finally st.close() + } + Mapping.RowCounts( + blobTokenized = one(s"SELECT COUNT(*) FROM blob_map WHERE ${Mapping.TokenizedPredicate}"), + blobIdentity = one(s"SELECT COUNT(*) FROM blob_map WHERE NOT (${Mapping.TokenizedPredicate})"), + tree = one("SELECT COUNT(*) FROM tree_map"), + commit = one("SELECT COUNT(*) FROM commit_map"), + ref = one("SELECT COUNT(*) FROM ref_map") + ) + } + + /** The first tokenized-row path `newMask` no longer selects. Identity rows are + * skipped: their paths match no mask. Matched on the basename, as Walker does. */ + def firstPathNarrowedBy(newMask: Regex): Option[String] = { + val st = conn.createStatement() + try { + val rs = st.executeQuery(s"SELECT path FROM blob_map WHERE ${Mapping.TokenizedPredicate}") + try { + var offender: Option[String] = None + while (offender.isEmpty && rs.next()) { + val path = rs.getString(1) + if (newMask.findFirstIn(Mapping.basename(path)).isEmpty) offender = Some(path) + } + offender + } finally rs.close() + } finally st.close() + } + + /** Up to `size` tokenized (path, new_blob) pairs, spread across the table and + * deterministic, so a refusal is reproducible. */ + def sampleTokenizedNewBlobs(size: Int, excludeExtensions: Set[String] = Set.empty): Vector[(String, String)] = { + require(size > 0, "sample size must be positive") + // Skip rows --retokenize is about to delete: their new_blob may already be + // gone from dst. + val keep = + if (excludeExtensions.isEmpty) Mapping.TokenizedPredicate + else s"${Mapping.TokenizedPredicate} AND NOT ${Mapping.extensionPredicate(excludeExtensions)}" + val total = countWhere(keep) + if (total == 0L) return Vector.empty + val stride = math.max(1L, total / size) + val st = conn.prepareStatement( + s"""SELECT path, new_blob FROM ( + | SELECT path, new_blob, ROW_NUMBER() OVER (ORDER BY orig_blob, path) AS rn + | FROM blob_map WHERE $keep + |) WHERE (rn - 1) % ? = 0 LIMIT ?""".stripMargin) + try { + st.setLong(1, stride) + st.setInt(2, size) + val rs = st.executeQuery() + try Iterator + .continually(if (rs.next()) Some((rs.getString(1), rs.getString(2))) else None) + .takeWhile(_.isDefined) + .flatten + .toVector + finally rs.close() + } finally st.close() + } + + /** Drop commit, ref and tree maps and the identity blob rows, which a wider mask + * would otherwise serve as cache hits of raw source, and record the new mask. + * ref_map is deleted explicitly, not left to the per-connection FK pragma. */ + def applyMaskWidening(oldMask: String, newMask: String, atEpochSeconds: Long): Mapping.WideningResult = { + val before = rowCounts + inTx { + val st = conn.createStatement() + try { + st.executeUpdate("DELETE FROM commit_map") + st.executeUpdate("DELETE FROM ref_map") + st.executeUpdate("DELETE FROM tree_map") + st.executeUpdate(s"DELETE FROM blob_map WHERE NOT (${Mapping.TokenizedPredicate})") + } finally st.close() + setMeta("mask", newMask) + setMeta(Mapping.MaskWidenedFromKey, oldMask) + setMeta(Mapping.MaskWidenedAtKey, atEpochSeconds.toString) + } + Mapping.WideningResult(before, rowCounts) + } + + /** Drop the tokenized blob rows for `extensions` plus tree, commit and ref maps, + * and record the new identity, in one transaction: a partial invalidation is + * undetectable. Identity rows stay, because the mask has not changed. */ + def applyRetokenize( + extensions: Set[String], + identity: TokenizerIdentity, + memo: TokenizerMemo.PurgeReport, + atEpochSeconds: Long + ): Mapping.RetokenizeResult = { + val before = rowCounts + val invalidated = countTokenizedRowsForExtensions(extensions) + val previousIds = extensions.toVector.sorted.flatMap(e => storedTokenizerId(e).map(v => s"$e=$v")) + inTx { + val st = conn.createStatement() + try { + st.executeUpdate("DELETE FROM commit_map") + st.executeUpdate("DELETE FROM ref_map") + st.executeUpdate("DELETE FROM tree_map") + st.executeUpdate( + s"DELETE FROM blob_map WHERE ${Mapping.TokenizedPredicate} " + + s"AND ${Mapping.extensionPredicate(extensions)}") + } finally st.close() + // Record every extension, not only the invalidated ones: an unrecorded one + // is a hole the next run cannot detect a change through. + identity.byExtension.foreach { case (ext, value) => + if (extensions.contains(ext) || storedTokenizerId(ext).isEmpty) setTokenizerId(ext, value) + } + setMeta(Mapping.RetokenizedAtKey, atEpochSeconds.toString) + setMeta(Mapping.RetokenizedExtensionsKey, extensions.toVector.sorted.mkString(",")) + setMeta(Mapping.RetokenizedFromKey, previousIds.mkString(",")) + } + Mapping.RetokenizeResult(before, rowCounts, extensions, invalidated, memo) + } + /** Run `body` inside a transaction; commit on success, rollback on throw. */ def inTx[A](body: => A): A = { conn.setAutoCommit(false) @@ -148,6 +326,74 @@ object Mapping { /** Mismatch between the recorded command/mask and the values passed in. */ final class MetaMismatchException(message: String) extends RuntimeException(message) + /** `--mask-widened` was passed but the new mask is not a widening. */ + final class MaskNarrowedException(message: String) extends RuntimeException(message) + + /** Reusing the map would dangle: its `new_blob` ids do not resolve in dst. */ + final class DanglingNewBlobException(message: String) extends RuntimeException(message) + + /** A recorded tokenizer identity changed and `--retokenize` did not name it. */ + final class TokenizerChangedException(message: String) extends RuntimeException(message) + + /** `--retokenize` would invalidate nothing; exiting 0 would look like success. */ + final class NothingInvalidatedException(message: String) extends RuntimeException(message) + + final case class IdentityChange(extension: String, stored: String, requested: String) + + private[blobexec] val TokenizedPredicate = "orig_blob <> new_blob" + + private[blobexec] val MaskWidenedFromKey = "mask_widened_from" + private[blobexec] val MaskWidenedAtKey = "mask_widened_at" + + /** One meta row per extension, so `--retokenize` can invalidate one language. */ + private[blobexec] val TokenizerIdKeyPrefix = "tokenizer_id." + + private[blobexec] val RetokenizedAtKey = "retokenized_at" + private[blobexec] val RetokenizedExtensionsKey = "retokenized_extensions" + private[blobexec] val RetokenizedFromKey = "retokenized_from" + + /** SQL for "path has one of these extensions", case-insensitive like the mask. + * The dot stops `.c` from matching `.cc`; TokenizerIdentity's alphabet leaves + * nothing to escape. */ + private[blobexec] def extensionPredicate(extensions: Set[String]): String = { + require(extensions.nonEmpty, "extensionPredicate needs at least one extension") + extensions.foreach(e => require( + e.matches(TokenizerIdentity.ExtensionPattern), + s"not a usable extension: [$e] (must match ${TokenizerIdentity.ExtensionPattern})")) + extensions.toVector.sorted.map(e => s"lower(path) LIKE '%.$e'").mkString("(", " OR ", ")") + } + + /** A sample, not all 2.7 M ids: it guards against a wiped dst, not one lost blob. */ + private[blobexec] val ReachabilitySampleSize = 256 + + final case class RowCounts(blobTokenized: Long, blobIdentity: Long, tree: Long, + commit: Long, ref: Long) + + final case class WideningResult(before: RowCounts, after: RowCounts) + + final case class RetokenizeResult( + before: RowCounts, + after: RowCounts, + extensions: Set[String], + blobsInvalidated: Long, + memo: TokenizerMemo.PurgeReport) + + /** Opt-in for `--retokenize`. `purgeMemo` is mandatory: the memo is keyed on + * contents alone (tokenBySha.pl), so dropping only blob_map rows would serve + * the stale tokens from one layer down. */ + final case class Retokenize( + extensions: Set[String], + newBlobResolves: String => Boolean, + purgeMemo: Vector[String] => TokenizerMemo.PurgeReport, + report: String => Unit, + memoDirIsTokenizerMemo: Boolean = false, + nowEpochSeconds: () => Long = () => System.currentTimeMillis() / 1000L, + sampleSize: Int = ReachabilitySampleSize) + + /** The part of `path` Walker matches the mask against. */ + private[blobexec] def basename(path: String): String = + path.substring(path.lastIndexOf('/') + 1) + // processed_at stored INTEGER (epoch seconds) for numeric filters. FK // ref_map.orig_commit -> commit_map ON DELETE CASCADE cleans up refs with their // commit; blob_map/tree_map have no FK (a blob/tree spans many commits). @@ -210,10 +456,39 @@ object Mapping { } finally st.close() } - /** Open/create the SQLite DB at `path` (schema created if absent) and validate - * `command`/`mask` against `meta`: mismatch throws MetaMismatchException, - * absence records them. */ - def open(path: Path, command: String, mask: String, warm: Option[Path] = None): Mapping = { + /** Opt-in for `--mask-widened`. `newBlobResolves` has no default: `new_blob` ids + * live only in dst, and a wiped dst would dangle every reused row. */ + final case class MaskWidening(newBlobResolves: String => Boolean, + report: String => Unit, + nowEpochSeconds: () => Long = () => System.currentTimeMillis() / 1000L, + sampleSize: Int = ReachabilitySampleSize) + + /** Open/create the DB at `path` and check `command`/`mask` against `meta`. + * `maskWidening` relaxes only the mask check, never the command check. */ + def open(path: Path, command: String, mask: String, warm: Option[Path] = None, + maskWidening: Option[MaskWidening] = None, + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenize: Option[Retokenize] = None): Mapping = { + // Refused, not composed: each invalidation is verified against a state the + // other would change. Run one, then the other as a resume. + require(!(maskWidening.isDefined && retokenize.isDefined), + "--mask-widened and --retokenize cannot be used in the same run: each verifies its own " + + "precondition against the rows, and together each would be verifying against a state the " + + "other is about to change. Run the widening first, then resume with --retokenize.") + retokenize.foreach { r => + require(tokenizerIdentity.nonEmpty, + "--retokenize needs a tokenizer identity to record. Without one the entries would be " + + "invalidated and nothing would be written to detect the next change with, so the very " + + "next run would reuse the new tokenizations without ever knowing which tokenizer made " + + "them.") + val unknown = r.extensions -- tokenizerIdentity.extensions + require(unknown.isEmpty, + s"--retokenize names extension(s) the tokenizer identity does not cover: " + + s"${unknown.toVector.sorted.mkString(", ")}. There is no identity to record for them, so " + + "the invalidation could not be accounted for. Known: " + + s"${tokenizerIdentity.extensions.toVector.sorted.mkString(", ")}.") + } + val url = s"jdbc:sqlite:${path.toAbsolutePath}" val conn = DriverManager.getConnection(url) enableForeignKeys(conn) @@ -223,11 +498,169 @@ object Mapping { val warmConn = warm.map(p => openWarm(p, command, mask)) val m = new Mapping(conn, warmConn) + // command first, and unconditionally: it gates the widening too. checkOrSetMeta(m, "command", command) - checkOrSetMeta(m, "mask", mask) + maskWidening match { + case Some(w) => widenOrCheckMask(m, mask, w) + case None => checkOrSetMeta(m, "mask", mask) + } + // Last: the only check that deletes rows must not run on a refused map. + retokenize match { + case Some(r) => retokenizeOrRefuse(m, tokenizerIdentity, r) + case None => if (tokenizerIdentity.nonEmpty) checkOrRecordTokenizerIdentity(m, tokenizerIdentity) + } m } + /** Record identities not yet recorded; refuse on a recorded mismatch. Deletes + * nothing, so a re-run with a changed tokenizer stops instead of reusing. */ + private def checkOrRecordTokenizerIdentity(m: Mapping, identity: TokenizerIdentity): Unit = { + val changes = m.cachePoisoningIdentityChanges(identity) + if (changes.nonEmpty) throw new TokenizerChangedException(tokenizerChangedMessage(changes)) + // Also overwrite a superseded value with no rows behind it, or it would + // refuse the next run for no reason. + identity.byExtension.foreach { case (ext, value) => + if (!m.storedTokenizerId(ext).contains(value)) m.setTokenizerId(ext, value) + } + } + + /** Refuse an unnamed change, a no-op, a dangling dst sample, or a purge that + * found nothing. The purge runs before the transaction: a needless purge costs + * a re-tokenize, a skipped one serves stale tokens. */ + private def retokenizeOrRefuse(m: Mapping, identity: TokenizerIdentity, r: Retokenize): Unit = { + val uncovered = m.cachePoisoningIdentityChanges(identity).filterNot(c => r.extensions.contains(c.extension)) + if (uncovered.nonEmpty) + throw new TokenizerChangedException( + "--retokenize refused: " + tokenizerChangedMessage(uncovered) + + s"\n --retokenize named only [${r.extensions.toVector.sorted.mkString(",")}], so those " + + "entries would have been invalidated while the ones above stayed stale, and the recorded " + + "identity would have become a mixture of two tokenizers. Nothing has been changed in the " + + "blob map. Name every changed extension in one run.") + + val affected = m.countTokenizedRowsForExtensions(r.extensions) + if (affected == 0L) + throw new NothingInvalidatedException( + s"--retokenize=${r.extensions.toVector.sorted.mkString(",")} refused: this blob map holds no " + + "tokenized row on a path with any of those extensions, so the flag would have invalidated " + + "NOTHING and the run would have exited 0 having reused every cached tokenization. Check " + + "the extension spelling (lowercase, no dot, as in tokenize/CregitLanguages.pm) and that " + + "this is the right project's blob map. Nothing has been changed.") + + val sample = m.sampleTokenizedNewBlobs(r.sampleSize, excludeExtensions = r.extensions) + sample.find { case (_, newBlob) => !r.newBlobResolves(newBlob) }.foreach { + case (path, newBlob) => + throw new DanglingNewBlobException( + s"--retokenize refused: blob_map's new_blob $newBlob (for '$path') does not exist in the " + + "destination repository. That row is one this invalidation KEEPS, and those ids live " + + "only in dst, so the map is only reusable alongside it. This is a --from-step 2 resume " + + "that keeps the work directory, never a fresh run (which deletes it first). Nothing has " + + s"been changed in the blob map. Sampled ${sample.size} retained tokenized rows.") + } + + val blobs = m.tokenizedOrigBlobsForExtensions(r.extensions) + val memo = r.purgeMemo(blobs) + if (memo.examined > 0L && memo.deleted == 0L) { + if (!r.memoDirIsTokenizerMemo) + throw new NothingInvalidatedException( + s"--retokenize refused: the memo held none of the ${memo.examined} affected blob(s) " + + s"(${memo.render}), and --memo-dir is not the directory in $$BFG_MEMO_DIR, the memo " + + "tokenBySha.pl reads. The --memo-dir is almost certainly not this project's. That " + + "matters because the memo is keyed on sha1(contents) with no tokenizer in the key: " + + "dropping the blob_map rows while the real memo keeps its entries just serves the same " + + "stale tokens through the other door. Nothing has been changed in the blob map.") + r.report( + s"--retokenize: the memo held none of the ${memo.examined} affected blob(s) " + + s"(${memo.render}). Accepted, because --memo-dir is $$BFG_MEMO_DIR, the memo " + + "tokenBySha.pl reads: no stale entry is left to serve, and every affected blob will run " + + "its tokenizer again.") + } + + val res = m.applyRetokenize(r.extensions, identity, memo, r.nowEpochSeconds()) + r.report( + s"--retokenize: invalidating the entries of [${res.extensions.toVector.sorted.mkString(",")}] " + + "because their tokenizer changed.\n" + + s" tokenizer identity now: ${identity.render}\n" + + s" DROPPED ${res.blobsInvalidated} tokenized blob_map rows on those extensions, and purged " + + s"their memo entries (${memo.render}). Both layers, because the memo is keyed on " + + "sha1(contents) alone and would otherwise answer the re-tokenization with the old tokens.\n" + + s" KEPT ${res.after.blobTokenized} tokenized blob_map rows on every other extension, and " + + s"${res.after.blobIdentity} identity rows. Re-tokenizing those is 88% of the pipeline and the " + + "defect has nothing to do with them.\n" + + s" DROPPED ${res.before.tree} tree_map rows (a tree names its blobs, and a retained tree row " + + s"short-circuits the re-walk), ${res.before.commit} commit_map rows (commits name trees) and " + + s"${res.before.ref} ref_map rows (refs name commits).\n" + + s" CHECKED ${math.min(r.sampleSize.toLong, res.after.blobTokenized)} of the retained new_blob " + + "ids resolve in dst.") + } + + private def tokenizerChangedMessage(changes: Vector[IdentityChange]): String = { + val exts = changes.map(_.extension).sorted + val detail = changes.map(c => + s"\n .${c.extension}: recorded ${c.stored}, this run reports ${c.requested}").mkString + s"the tokenizer for ${exts.map("." + _).mkString(", ")} is not the one that produced the " + + s"cached tokens in this blob map.$detail" + + "\n Reusing those rows would reproduce the OLD tokenizer's output for every cached file, and " + + "nothing downstream can tell one token stream from another: that is how a 16-day-old " + + "rustTokenizer binary put a `line:col` prefix into 741,869 .rs entries across 45 " + + "projects with no error anywhere.\n" + + " Nothing has been changed. Either restore the tokenizer this map was built with, or " + + s"invalidate exactly those entries:\n --retokenize=${exts.mkString(",")}\n" + + " That drops the affected blob_map rows and their memo entries, keeps every other " + + "extension's tokenizations, and requires --memo-dir and a step-2 resume." + } + + /** No stored mask or an equal one: no-op. Otherwise refuse a narrowing or a + * dangling dst sample before writing anything. */ + private def widenOrCheckMask(m: Mapping, mask: String, w: MaskWidening): Unit = + m.getMeta("mask") match { + case None => + m.setMeta("mask", mask) + w.report("--mask-widened: no mask recorded yet (fresh blob map), so there is nothing " + + "to reuse and nothing to check. Proceeding as a normal first run.") + case Some(stored) if stored == mask => + w.report(s"--mask-widened: the recorded mask already equals this run's [$mask], so this " + + "is an ordinary resume and the flag changes nothing.") + case Some(stored) => + val counts = m.rowCounts + m.firstPathNarrowedBy(mask.r).foreach { offending => + throw new MaskNarrowedException( + s"--mask-widened refused: [$mask] does not select '$offending', which was tokenized " + + s"under the recorded mask [$stored]. That makes the change a NARROWING, not a " + + "widening, so the rows this flag would reuse are not all valid. Nothing has been " + + "changed in the blob map. Verified against the data, one regex test per tokenized " + + s"row (${counts.blobTokenized} of them), not by comparing the two regexes." + ) + } + val sample = m.sampleTokenizedNewBlobs(w.sampleSize) + sample.find { case (_, newBlob) => !w.newBlobResolves(newBlob) }.foreach { + case (path, newBlob) => + throw new DanglingNewBlobException( + s"--mask-widened refused: blob_map's new_blob $newBlob (for '$path') does not exist " + + "in the destination repository. Those ids live only in dst, so the map is only " + + "reusable alongside it. This is a --from-step 2 resume that KEEPS the work " + + "directory, never a fresh run (which deletes it first). Nothing has been changed " + + s"in the blob map. Sampled ${sample.size} of ${counts.blobTokenized} tokenized rows." + ) + } + val r = m.applyMaskWidening(stored, mask, w.nowEpochSeconds()) + w.report( + s"--mask-widened: reusing the blob map across a mask widening.\n" + + s" old mask: $stored\n" + + s" new mask: $mask\n" + + s" KEPT ${r.after.blobTokenized} tokenized blob_map rows (same blob, same path, so " + + "same content and same extension, so the same tokens)\n" + + s" DROPPED ${r.before.blobIdentity} identity blob_map rows (orig == new). Under the old " + + "mask these recorded 'not selected, bytes pass through'. Some of those paths ARE " + + "selected now, and keeping them would serve raw source as a cache hit.\n" + + s" DROPPED ${r.before.tree} tree_map rows (trees gain entries) and " + + s"${r.before.commit} commit_map rows (commits name trees), and " + + s"${r.before.ref} ref_map rows (refs name commits).\n" + + s" CHECKED every one of the ${r.after.blobTokenized} retained paths still matches the new " + + s"mask, and ${math.min(w.sampleSize.toLong, r.after.blobTokenized)} of their new_blob ids " + + "resolve in dst." + ) + } + /** Open a frozen prior-run DB read-only (query_only + busy_timeout) as a lookup * fallback; never written by us, so concurrent shard readers are safe. Rejects a * warm DB whose command/mask differ from this run -- its ids would be foreign. */ diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala new file mode 100644 index 000000000..f457d65d1 --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerIdentity.scala @@ -0,0 +1,68 @@ +package cregit.blobexec + +/** Opaque tokenizer identity per file extension, so `blob_map` reuse notices a + * changed tokenizer. Per extension because `blob_map` stores paths, not + * languages (see `tokenize/tokenizerIdentity.pl`). */ +final case class TokenizerIdentity(byExtension: Map[String, String]) { + + def isEmpty: Boolean = byExtension.isEmpty + def nonEmpty: Boolean = byExtension.nonEmpty + def extensions: Set[String] = byExtension.keySet + def get(extension: String): Option[String] = byExtension.get(extension) + + /** Sorted, so the same map always renders the same string. */ + def render: String = + byExtension.toVector.sorted.map { case (e, id) => s"$e=$id" }.mkString(",") +} + +object TokenizerIdentity { + + val empty: TokenizerIdentity = TokenizerIdentity(Map.empty) + + /** As `CregitLanguages.pm` spells them (`c++` is real). Interpolated into a SQL + * LIKE in [[Mapping]]: no quote, `%` or `_` allowed, so nothing to escape. */ + private[blobexec] val ExtensionPattern = "[a-z0-9+]+" + + private[blobexec] val ValuePattern = "[0-9a-f]{8,128}" + + private val EntryRe = s"($ExtensionPattern)=($ValuePattern)".r + + /** Parse `ext=value[,ext=value]...`; Left names the malformed entry. */ + def parse(spec: String): Either[String, TokenizerIdentity] = { + val trimmed = spec.trim + if (trimmed.isEmpty) + return Left("a tokenizer identity must name at least one extension, as ext=value") + val entries = trimmed.split(",", -1).toVector + val parsed = entries.map { + case EntryRe(ext, value) => Right((ext, value)) + case other => + Left( + s"cannot read [$other] as a tokenizer identity entry. Wanted " + + s"=, extension matching $ExtensionPattern (lowercase, no " + + s"leading dot, as in CregitLanguages.pm) and value matching $ValuePattern." + ) + } + parsed.collectFirst { case Left(why) => Left(why) }.getOrElse { + val pairs = parsed.collect { case Right(p) => p } + val duplicated = pairs.groupBy(_._1).collect { case (e, vs) if vs.size > 1 => e } + if (duplicated.nonEmpty) + Left(s"extension(s) named more than once in the tokenizer identity: " + + duplicated.toVector.sorted.mkString(", ")) + else Right(TokenizerIdentity(pairs.toMap)) + } + } + + /** Parse a `--retokenize=` extension list. */ + def parseExtensions(spec: String): Either[String, Set[String]] = { + val trimmed = spec.trim + if (trimmed.isEmpty) + return Left("--retokenize must name at least one extension, as in --retokenize=rs") + val parts = trimmed.split(",", -1).toVector.map(_.trim) + val bad = parts.filterNot(_.matches(ExtensionPattern)) + if (bad.nonEmpty) + Left(s"not an extension: ${bad.map(b => s"[$b]").mkString(", ")}. Wanted lowercase " + + s"names without a leading dot, matching $ExtensionPattern (as in CregitLanguages.pm): " + + "--retokenize=rs, --retokenize=c,h") + else Right(parts.toSet) + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala new file mode 100644 index 000000000..d2f773ee5 --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerMemo.scala @@ -0,0 +1,62 @@ +package cregit.blobexec + +import java.nio.file.{Files, Path} +import java.security.MessageDigest + +/** The tokenBySha.pl memo, keyed by `sha1_hex(contents)` alone (tokenBySha.pl:76,83-84). + * A dropped `blob_map` row would be refilled from it with stale tokens, so + * `--retokenize` purges both, the memo first: a lost entry only costs a rerun. */ +object TokenizerMemo { + + /** `missing` is normal (excluded blobs never had an entry); [[Mapping]] refuses + * on `deleted == 0` with `examined > 0`. */ + final case class PurgeReport(examined: Long, deleted: Long, missing: Long, unreadable: Long) { + def render: String = + s"examined=$examined deleted=$deleted missing=$missing unreadable=$unreadable" + } + + object PurgeReport { + val empty: PurgeReport = PurgeReport(0L, 0L, 0L, 0L) + } + + /** `sha1_hex` of the bytes, matching Perl's `Digest::SHA::sha1_hex`. */ + def sha1Hex(bytes: Array[Byte]): String = { + val md = MessageDigest.getInstance("SHA-1") + val digest = md.digest(bytes) + val sb = new StringBuilder(digest.length * 2) + digest.foreach(b => sb.append("%02x".format(b))) + sb.toString + } + + /** Must match tokenBySha.pl:83-84 byte for byte; pinned by `TokenizerMemoSpec`. */ + def relativePathFor(sha1Hex: String): String = { + require(sha1Hex.length >= 4, s"not a sha1: [$sha1Hex]") + s"${sha1Hex.substring(0, 2)}/${sha1Hex.substring(2, 4)}/$sha1Hex" + } + + def entryFor(root: Path, sha1Hex: String): Path = + root.resolve(relativePathFor(sha1Hex)) + + def isTokenizerMemo(memoDir: Path, bfgMemoDir: Option[String]): Boolean = + bfgMemoDir.filter(_.nonEmpty).exists { env => + try Files.isSameFile(memoDir, java.nio.file.Paths.get(env)) + catch { case _: java.io.IOException | _: java.nio.file.InvalidPathException => false } + } + + /** Delete the memo entries of `origBlobs`. `readBytes` is None for a blob that + * cannot be materialised. One blob at a time: an invalidation can be ~10^6 rows. */ + def purge(root: Path, origBlobs: Vector[String], readBytes: String => Option[Array[Byte]]): PurgeReport = { + var deleted = 0L + var missing = 0L + var unreadable = 0L + origBlobs.foreach { sha => + readBytes(sha) match { + case None => unreadable += 1L + case Some(bytes) => + val entry = entryFor(root, sha1Hex(bytes)) + if (Files.deleteIfExists(entry)) deleted += 1L else missing += 1L + } + } + PurgeReport(origBlobs.size.toLong, deleted, missing, unreadable) + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala new file mode 100644 index 000000000..b0bc6161c --- /dev/null +++ b/blobExec/src/main/scala/cregit/blobexec/TokenizerWorkerPool.scala @@ -0,0 +1,128 @@ +package cregit.blobexec + +import java.io.{BufferedOutputStream, DataInputStream, EOFException, IOException} +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} +import java.util.concurrent.{ArrayBlockingQueue, ConcurrentHashMap, CountDownLatch, TimeUnit} +import scala.jdk.CollectionConverters._ + +/** Persistent tokenizer processes that speak tokenizeByBlobId/WORKER_PROTOCOL.md, + * one request in flight per worker. */ +final class TokenizerWorkerPool(workerPath: String, size: Int) extends AutoCloseable { + import TokenizerWorkerPool._ + + private final class Worker(val index: Int, val process: Process) { + val stdin = new BufferedOutputStream(process.getOutputStream) + val stdout = new DataInputStream(process.getInputStream) + } + + private val idle = new ArrayBlockingQueue[Worker](size) + private val workers = ConcurrentHashMap.newKeySet[Worker]() + private val closed = new AtomicBoolean(false) + + try (0 until size).foreach(addWorker) + catch { case failure: Throwable => close(); throw failure } + + def invoke(bytes: Array[Byte], filename: String, timeoutSeconds: Int): ChildRunner.Outcome = { + val worker = idle.take() + val backstop = ChildRunner.maxLifetimeSeconds(timeoutSeconds) + within(backstop.toLong)(request(worker, bytes, filename, timeoutSeconds)) match { + case None => + replace(worker) + ChildRunner.Outcome.Killed(s"tokenizer worker ${worker.index} gave no response within ${backstop}s; replaced it") + case Some(Left(failure)) => + replace(worker) + ChildRunner.Outcome.Killed(s"tokenizer worker ${worker.index} died ($failure); replaced it") + case Some(Right(response)) => + if (worker.process.isAlive) idle.put(worker) else replace(worker) + response + } + } + + override def close(): Unit = + if (closed.compareAndSet(false, true)) { + val all = workers.asScala.toVector + all.foreach(w => try w.stdin.close() catch { case _: IOException => () }) + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(CloseSeconds) + all.foreach { w => + if (!w.process.waitFor(math.max(0L, deadline - System.nanoTime()), TimeUnit.NANOSECONDS)) kill(w.process) + } + } + + private def addWorker(index: Int): Unit = { + val process = new ProcessBuilder(workerPath).redirectError(ProcessBuilder.Redirect.INHERIT).start() + val worker = new Worker(index, process) + within(ReadySeconds)(readLine(worker.stdout)) match { + case Some(Right("READY")) => + workers.add(worker) + idle.put(worker) + case notReady => + kill(process) + throw new IllegalStateException(s"tokenizer worker [$workerPath] did not start: $notReady") + } + } + + private def replace(worker: Worker): Unit = { + workers.remove(worker) + kill(worker.process) + if (!closed.get()) addWorker(worker.index) + } + + private def request(worker: Worker, bytes: Array[Byte], filename: String, timeoutSeconds: Int) = { + val name = filename.getBytes(UTF_8) + worker.stdin.write(s"REQ ${bytes.length} ${name.length} $timeoutSeconds\n".getBytes(UTF_8)) + worker.stdin.write(name) + worker.stdin.write(bytes) + worker.stdin.flush() + readLine(worker.stdout).split(' ') match { + case Array("RES", exit, outLength, errLength) => + val out = readBytes(worker.stdout, outLength.toInt) + val err = new String(readBytes(worker.stdout, errLength.toInt), UTF_8) + if (exit.toInt == WorkerTimeoutExitCode) ChildRunner.Outcome.Killed(s"no exit within ${timeoutSeconds}s") + else ChildRunner.Outcome.Exited(exit.toInt, out, err) + case header => throw new IOException(s"malformed response header [${header.mkString(" ")}]") + } + } +} + +object TokenizerWorkerPool { + /** What a worker answers when it killed the tokenizer for running out of time. */ + val WorkerTimeoutExitCode = 124 + private val ReadySeconds = 30L + private val CloseSeconds = 2L + private val MaxHeaderBytes = 8192 + + private def kill(process: Process): Unit = { ChildRunner.killTree(process); () } + + /** Runs `read` on a daemon thread, so that a worker which never answers cannot block the caller. */ + private def within[A](seconds: Long)(read: => A): Option[Either[Throwable, A]] = { + val result = new AtomicReference[Either[Throwable, A]]() + val done = new CountDownLatch(1) + val reader = new Thread(() => { + try result.set(Right(read)) + catch { case failure: Throwable => result.set(Left(failure)) } + finally done.countDown() + }, "tokenizer-worker-read") + reader.setDaemon(true) + reader.start() + if (done.await(seconds, TimeUnit.SECONDS)) Some(result.get()) else None + } + + private def readLine(in: DataInputStream): String = { + val line = new java.io.ByteArrayOutputStream + var next = in.read() + while (next != '\n') { + if (next == -1) throw new EOFException("tokenizer worker closed its stdout") + if (line.size() >= MaxHeaderBytes) throw new IOException(s"header longer than $MaxHeaderBytes bytes") + line.write(next) + next = in.read() + } + line.toString(UTF_8) + } + + private def readBytes(in: DataInputStream, length: Int): Array[Byte] = { + val bytes = new Array[Byte](length) + in.readFully(bytes) + bytes + } +} diff --git a/blobExec/src/main/scala/cregit/blobexec/Walker.scala b/blobExec/src/main/scala/cregit/blobexec/Walker.scala index a9c9f4741..343abd825 100644 --- a/blobExec/src/main/scala/cregit/blobexec/Walker.scala +++ b/blobExec/src/main/scala/cregit/blobexec/Walker.scala @@ -20,15 +20,11 @@ final case class WalkStats( blobsCacheHit: Int, refsProjected: Int, aborted: Boolean, - /** Blobs whose command was killed for exceeding its budget. Each one is a - * file left holding raw source instead of tokens, so a non-zero count must - * reach the caller rather than living only in stderr. */ blobsTimedOut: Long, - /** Mask-matched blobs JGit would not materialise. Reported, not gating. */ + /** Too large for JGit to materialise: see [[Walker.MaxBlobBytes]]. */ blobsOversized: Long, - /** Distinct mask-matched blobs excluded because they are on the shipped blob - * denylist. Reported, not gating. */ blobsDenylisted: Long, + /** Tokenizer exited [[BlobExec.ParserCrashExitCode]]. */ blobsParserCrashed: Long = 0L, blobCommandExecutions: Long, originalBlobCopyRequests: Long, @@ -57,6 +53,8 @@ final class Walker( destinationMayContainObjects: Boolean = true, blobTimeoutSeconds: Int = BlobExec.DefaultTimeoutSeconds, stallTimeoutSeconds: Int = Walker.DefaultStallTimeoutSeconds, + workerPool: Option[TokenizerWorkerPool] = None, + // A parameter only so tests can supply a fixture. denylist: BlobDenylist = BlobDenylist.shipped ) { import Walker._ @@ -80,10 +78,9 @@ final class Walker( private val originalBlobBytesAvoided = new LongAdder // -- stall watchdog ------------------------------------------------------ - // - // `Await.result` is unbounded on purpose: a duration budget cannot tell a wedged - // run from an honestly large one. The watchdog asserts completed work instead, - // which is safe because ChildRunner bounds every child. + // `Await.result` is unbounded: a duration cannot tell a wedged run from a large + // one, so the watchdog checks that some work completes. Safe because `timeout` + // hard-bounds every child at `blobTimeoutSeconds`. private val lastProgressNanos = new AtomicLong(System.nanoTime()) private val lastProgressWhat = new AtomicReference[String]("startup") @@ -133,22 +130,18 @@ final class Walker( killed } - /** Run `body` under the stall watchdog. The watchdog is a daemon so it can - * never keep the JVM alive, and it halts rather than exits: a shutdown hook - * could block on the same wedged thread. */ + /** Halts rather than exits: a shutdown hook could block on the wedged thread. */ private def withStallWatchdog[A](body: => A): A = { val done = new AtomicBoolean(false) val tick = math.max(1L, math.min(30L, math.max(1, stallTimeoutSeconds).toLong / 4L)) val watchdog = new Thread( () => { - var interrupted = false - while (!done.get() && !interrupted) { - // Exit on interrupt: a set flag makes every later sleep throw at once. + while (!done.get()) { try Thread.sleep(tick * 1000L) - catch { case _: InterruptedException => interrupted = true } - if (!done.get() && !interrupted) stalled().foreach { stalledNanos => + catch { case _: InterruptedException => Thread.currentThread().interrupt() } + if (!done.get()) stalled().foreach { stalledNanos => System.err.println(watchdogReport(stalledNanos)) - // Halting reaps nothing, so kill the descendant tree first. + // halt() reaps nothing: kill the whole descendant tree first. val killed = killDescendants() System.err.println(s"blobExec: killed $killed leftover child process(es) before exiting") System.err.flush() @@ -295,14 +288,13 @@ final class Walker( val procWalk = new RevWalk(src) var aborted = false - var timedOut = false var commits = 0 var blobsRun = 0 var blobsHit = 0 try { val it = sliceIds.iterator - while (it.hasNext && !aborted && !timedOut) { + while (it.hasNext && !aborted) { val rc = procWalk.parseCommit(it.next()) val origTreeId = rc.getTree.getId @@ -314,9 +306,6 @@ final class Walker( if (misses.abort) { aborted = true - } else if (misses.timedOutKeys.nonEmpty) { - timedOut = true - persistRetryableBlobs(misses, rc.getId.name) } else { val subtreeMap = scala.collection.mutable.Map.empty[String, String] val newTreeId = assemble(plan, resolved, treeInserter, subtreeMap) @@ -352,33 +341,6 @@ final class Walker( (commits, blobsRun, blobsHit, aborted) } - /** Persist only what a commit containing a timed-out blob may leave behind. - * - * Three writes would each hide the retry: the blob's `blob_map` row, any - * `tree_map` row above it (a tree hit short-circuits the whole subtree), and - * the commit's `commit_map` row. All three are suppressed. Successful - * tokenizations are kept, because they are content-addressed and already in - * dst; identity rows are not, because their bytes are copied during the tree - * assembly this path skips. - */ - private def persistRetryableBlobs(misses: MissResolution, origCommitSha: String): Unit = { - val keep = misses.persistable - dbLock.synchronized { - mapping.inTx { - keep.foreach { case ((origSha, path), newId) => mapping.putBlob(origSha, path, newId.name) } - } - } - System.err.println( - s"blobExec: commit $origCommitSha contains ${misses.timedOutKeys.size} timed-out blob(s): " + - s"kept ${keep.size} good blob row(s), recorded no tree and no commit for it. " + - "Another run over this memo retries just those blobs (from the pipeline: resume at " + - "step 2, never step 1); nothing needs clearing by hand." - ) - misses.timedOutKeys.toVector.sorted.foreach { case (sha, path) => - System.err.println(s"blobExec: will retry on the next run: $sha ($path)") - } - } - // -- ref preparation ----------------------------------------------------- /** For every commit sha already in `commit_map` that still exists in src, @@ -386,7 +348,7 @@ final class Walker( private def markUninteresting(revWalk: RevWalk): Unit = { mapping.allCommitOrigShas.foreach { sha => val id = ObjectId.fromString(sha) - // The frontier scan is slow but healthy; stamp it. + // A long frontier scan must not look like a stall. progress(s"frontier $sha") try { val rc = revWalk.parseCommit(id) @@ -429,14 +391,13 @@ final class Walker( val commitInserter = dst.newObjectInserter() var aborted = false // local mutation across the walk loop is unavoidable - var timedOut = false // a blob in this commit was killed on its budget var commits = 0 var blobsRun = 0 var blobsHit = 0 try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted && !timedOut) { + while (iter.hasNext && !aborted) { val rc = iter.next() val origCommitSha = rc.getId.name @@ -449,11 +410,6 @@ final class Walker( if (misses.abort) { aborted = true - } else if (misses.timedOutKeys.nonEmpty) { - // Keep the successful tokenizations; persist no tree and no commit, and - // stop, so no descendant is folded onto a parent we did not record. - timedOut = true - persistRetryableBlobs(misses, origCommitSha) } else { // Assemble the new tree id (bottom-up). `subtreeMap` collects every // freshly built (sub)tree's orig->new id so the transaction below @@ -529,8 +485,6 @@ final class Walker( val inFlight = new ConcurrentHashMap[(String, String), Future[BlobResult]]() val aborted = new AtomicBoolean(false) - // Stops the producer and drains the consumer, without aborting the run. - val timedOutStop = new AtomicBoolean(false) val producerError = new AtomicReference[Throwable](null) // -- producer: iterate the walk in order, plan trees, submit blob work -- @@ -538,7 +492,7 @@ final class Walker( def run(): Unit = { try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted.get() && !timedOutStop.get()) { + while (iter.hasNext && !aborted.get()) { val rc = iter.next() val data = snapshotCommit(rc) // decouple consumer from the shared RevWalk val (plan, artifacts) = buildTreePlan(rc.getTree.getId, pathPrefix = "") @@ -578,31 +532,16 @@ final class Walker( queue.take() match { case EndOfWalk => stopped = true case CommitItem(data, plan, artifacts, missFutures) => - if (aborted.get() || timedOutStop.get()) { - // Draining after abort / producer-error / a timed-out commit: - // discard until EndOfWalk. Folding a later commit here would - // reference a parent this run deliberately did not record. + if (aborted.get()) { + // Draining after abort / producer-error: discard until EndOfWalk. } else { // Await only this commit's futures (typically already complete). // Unbounded on purpose: the stall watchdog, not a guessed // duration, is what stops a wedged run (see withStallWatchdog). val results = missFutures.map { case (k, f) => k -> Await.result(f, Duration.Inf) } - val timedOutIds: IMap[(String, String), ObjectId] = - results.iterator.collect { case (k, BlobResult.Unusable(id)) => k -> id }.toMap results.values.collectFirst { case a: BlobResult.Aborted => a } match { case Some(_) => aborted.set(true) // stop the producer; drain the remainder - case None if timedOutIds.nonEmpty => - // Retryable: keep the good blob rows, write no tree and no - // commit, and stop the walk so nothing is folded onto a parent - // this run did not record. See persistRetryableBlobs. - val good: IMap[(String, String), ObjectId] = - results.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap - persistRetryableBlobs( - MissResolution(good ++ timedOutIds, abort = false, timedOutKeys = timedOutIds.keySet), - data.origCommitSha - ) - timedOutStop.set(true) case None => val resolved: IMap[(String, String), ObjectId] = results.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap @@ -679,7 +618,6 @@ final class Walker( val inFlight = new ConcurrentHashMap[(String, String), Future[BlobResult]]() val aborted = new AtomicBoolean(false) - val timedOutStop = new AtomicBoolean(false) // see walkCommitsPipelined val producerError = new AtomicReference[Throwable](null) // -- producer: plan trees, submit blob work, chain tree assembly -------- @@ -687,7 +625,7 @@ final class Walker( def run(): Unit = { try { val iter = revWalk.iterator() - while (iter.hasNext && !aborted.get() && !timedOutStop.get()) { + while (iter.hasNext && !aborted.get()) { val rc = iter.next() val data = snapshotCommit(rc) val (plan, artifacts) = buildTreePlan(rc.getTree.getId, pathPrefix = "") @@ -733,25 +671,15 @@ final class Walker( queue.take() match { case EndOfTreeWalk => stopped = true case CommitTreeItem(data, artifacts, blobKeys, treeFuture) => - if (aborted.get() || timedOutStop.get()) { - // Draining after abort / producer-error / a timed-out commit: - // discard until EndOfTreeWalk. + if (aborted.get()) { + // Draining after abort / producer-error: discard until EndOfTreeWalk. } else { // The assembly (and any abort it observed) is already done off-thread. // Unbounded: the stall watchdog is the backstop, not a budget. Await.result(treeFuture, Duration.Inf) match { case TreeResult.Aborted(_, _) => aborted.set(true) // stop the producer; drain the remainder - case TreeResult.Built(_, resolved, _, tok) if tok.nonEmpty => - // Retryable: the tree was assembled off-thread and is usable, - // but recording it (or the commit) would hide the timeout from - // the next run. Keep the good blob rows only, and stop here. - persistRetryableBlobs( - MissResolution(resolved, abort = false, timedOutKeys = tok), - data.origCommitSha - ) - timedOutStop.set(true) - case TreeResult.Built(newTreeId, resolved, subtrees, _) => + case TreeResult.Built(newTreeId, resolved, subtrees) => val parents = data.parentOrigShas.map { pn => dbLock.synchronized(mapping.getCommit(pn)).getOrElse( throw new IllegalStateException( @@ -809,18 +737,14 @@ final class Walker( blobResults.iterator.map(_._2).collectFirst { case a: BlobResult.Aborted => a } match { case Some(a) => TreeResult.Aborted(a.stderr, a.exitCode) case None => - // `resolved` is what the consumer persists, so a timed-out blob is kept - // out of it; `forTree` is what the tree references, so it is kept in. val resolved: IMap[(String, String), ObjectId] = blobResults.iterator.collect { case (k, BlobResult.Resolved(id)) => k -> id }.toMap - val timedOut: IMap[(String, String), ObjectId] = - blobResults.iterator.collect { case (k, BlobResult.Unusable(id)) => k -> id }.toMap val inserter = dst.newObjectInserter() try { val subtreeMap = scala.collection.mutable.Map.empty[String, String] - val newTreeId = assemble(plan, resolved ++ timedOut, inserter, subtreeMap) + val newTreeId = assemble(plan, resolved, inserter, subtreeMap) inserter.flush() - TreeResult.Built(newTreeId, resolved, subtreeMap.toMap, timedOut.keySet) + TreeResult.Built(newTreeId, resolved, subtreeMap.toMap) } finally inserter.close() } } @@ -831,7 +755,8 @@ final class Walker( * into dst). Never touches `mapping`, keeping the hot path lock-free. */ private def executeBlobTask(task: BlobMissTask): BlobResult = readBlob(task) match { - case None => BlobResult.Oversized(task.origId) + // None: excluded, and already counted and logged (see readBlob). + case None => BlobResult.Excluded(task.origId) case Some(bytes) => executeBlobTask(task, bytes) } @@ -841,7 +766,7 @@ final class Walker( inFlightBlobs.put(label, System.nanoTime()) try { blobCommandExecutions.increment() - val unusable = new AtomicBoolean(false) + val failure = new AtomicReference[String]() val outcome = BlobExec.run( bytes = bytes, origSha = task.origId.name, @@ -851,14 +776,14 @@ final class Walker( abortOnError = abortOnError, inserter = workerInserter, timeoutSeconds = blobTimeoutSeconds, - onTimeout = () => { blobsTimedOut.increment(); unusable.set(true) }, - onParserCrash = () => { blobsParserCrashed.increment(); unusable.set(true) } + onTimeout = () => { blobsTimedOut.increment(); failure.set("timeout") }, + onParserCrash = () => { blobsParserCrashed.increment(); failure.set("parser-crash") }, + workerPool = workerPool ) val res = outcome match { - case BlobExec.Outcome.Skip if unusable.get() => - // The tree must still reference something, so keep the original bytes - ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) - BlobResult.Unusable(task.origId) + case BlobExec.Outcome.Skip if failure.get() != null => + noteFailed(task, failure.get()) + BlobResult.Excluded(task.origId) case BlobExec.Outcome.Skip => ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) BlobResult.Resolved(task.origId) @@ -971,15 +896,12 @@ final class Walker( // -- parallel blob resolution ------------------------------------------- - /** Run each miss through the external command on the pool. Returns the ids the - * tree should reference, an aborted flag, and the keys whose command was - * killed on its budget — those are in the id map (the tree needs them) but - * must be kept out of `blob_map`, so the caller can retry them. */ + /** Excluded blobs get no id in the result. */ private def resolveMisses( misses: Vector[BlobMissTask], pool: java.util.concurrent.ExecutorService )(implicit ec: ExecutionContext): MissResolution = { - if (misses.isEmpty) return MissResolution(IMap.empty, abort = false, timedOutKeys = Set.empty) + if (misses.isEmpty) return MissResolution(IMap.empty, abort = false) // De-dup so we don't run the command twice for the same key within a // single commit (e.g. the same (blob, path) reached via two subtrees). @@ -987,13 +909,13 @@ final class Walker( val futures = unique.map { task => Future { - readBlob(task).map { bytes => + readBlob(task).flatMap { bytes => val workerInserter = dst.newObjectInserter() val label = s"${task.origId.name} (${task.fullPath})" inFlightBlobs.put(label, System.nanoTime()) try { blobCommandExecutions.increment() - val unusable = new AtomicBoolean(false) + val failure = new AtomicReference[String]() val outcome = BlobExec.run( bytes = bytes, origSha = task.origId.name, @@ -1003,17 +925,20 @@ final class Walker( abortOnError = abortOnError, inserter = workerInserter, timeoutSeconds = blobTimeoutSeconds, - onTimeout = () => { blobsTimedOut.increment(); unusable.set(true) }, - onParserCrash = () => { blobsParserCrashed.increment(); unusable.set(true) } + onTimeout = () => { blobsTimedOut.increment(); failure.set("timeout") }, + onParserCrash = () => { blobsParserCrashed.increment(); failure.set("parser-crash") } ) - // A Skip keeps the original id, so those bytes must exist in dst. + val failed = Option(failure.get()) + failed.foreach(noteFailed(task, _)) + // Skip keeps the original id, so its bytes must exist in dst. outcome match { - case BlobExec.Outcome.Skip => ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) - case _ => () + case BlobExec.Outcome.Skip if failed.isEmpty => + ensureOriginalBlobAvailable(task.origId, insertHeldBytes(bytes), workerInserter) + case _ => () } workerInserter.flush() progress(s"blob $label") - (task, outcome, unusable.get()) + if (failed.isDefined) None else Some((task, outcome)) } finally { inFlightBlobs.remove(label) workerInserter.close() @@ -1022,58 +947,74 @@ final class Walker( } } - // Unbounded: the stall watchdog is the backstop, not a budget. + // Unbounded: the stall watchdog is the backstop. `flatten` drops excluded blobs. val results = Await.result(Future.sequence(futures), Duration.Inf).flatten - val abort = results.exists { case (_, o, _) => o.isInstanceOf[BlobExec.Outcome.Abort] } - if (abort) MissResolution(IMap.empty, abort = true, timedOutKeys = Set.empty) + val abort = results.exists { case (_, o) => o.isInstanceOf[BlobExec.Outcome.Abort] } + if (abort) MissResolution(IMap.empty, abort = true) else { val ids = results.iterator.collect { - case (task, BlobExec.Outcome.Replace(newId), _) => + case (task, BlobExec.Outcome.Replace(newId)) => (task.origId.name, task.fullPath) -> newId - case (task, BlobExec.Outcome.Skip, _) => - // identical, non-zero-exit (with abortOnError=false), or timed out + case (task, BlobExec.Outcome.Skip) => + // identical, or non-zero-exit (with abortOnError=false) (task.origId.name, task.fullPath) -> task.origId }.toMap - val timedOutKeys = results.iterator.collect { - case (task, _, true) => (task.origId.name, task.fullPath) - }.toSet - MissResolution(ids, abort = false, timedOutKeys = timedOutKeys) + MissResolution(ids, abort = false) } } + /** `None` when the blob is excluded: denylisted, failed earlier, or oversized. + * The size check is only a fast path; the `LargeObjectException` catch also + * covers a JGit configured below its default threshold. */ private def readBlob(task: BlobMissTask): Option[Array[Byte]] = { val denied = denylist.entryFor(task.origId.name) if (denied.isDefined) { noteDenylisted(task, denied.get); return None } + if (failedKeys.contains((task.origId.name, task.fullPath))) return None val r = src.newObjectReader() try { val loader = r.open(task.origId, OBJ_BLOB) - try Some(loader.getBytes) - catch { - case _: org.eclipse.jgit.errors.LargeObjectException => - noteOversized(task, loader.getSize) - None - } + val size = loader.getSize + if (Walker.isOversized(size)) { noteOversized(task, size); None } + else + try Some(loader.getBytes) + catch { + case _: org.eclipse.jgit.errors.LargeObjectException => + noteOversized(task, size) + None + } } finally r.close() } - /** `(origSha, fullPath)` of blobs excluded as oversized; [[resolveEntry]] drops - * their tree entries. */ + /** Lets [[resolveEntry]] tell a deliberate omission from a missing-key bug. */ private val oversizedKeys = ConcurrentHashMap.newKeySet[(String, String)]() - /** `(origSha, fullPath)` of blobs excluded by the denylist; kept apart from - * [[oversizedKeys]] so the two counts can never be confused. */ private val denylistedKeys = ConcurrentHashMap.newKeySet[(String, String)]() + /** Checked in [[readBlob]] so a failing blob costs one tokenizer run, not one per commit. */ + private val failedKeys = ConcurrentHashMap.newKeySet[(String, String)]() + + private def noteFailed(task: BlobMissTask, reason: String): Unit = + if (failedKeys.add((task.origId.name, task.fullPath))) + System.err.println( + s"blobExec: EXCLUDED failed blob: sha=${task.origId.name} path=${task.fullPath} " + + s"reason=$reason. The blob is left out of the rewritten tree, so it produces no " + + "blame and no dataset row. The walk carries on.") + private def noteDenylisted(task: BlobMissTask, entry: BlobDenylist.Entry): Unit = { val key = (task.origId.name, task.fullPath) + // Once per (sha, path), like noteOversized: one blob at two paths counts twice. if (denylistedKeys.add(key)) { blobsDenylisted.increment() System.err.println( s"blobExec: EXCLUDED denylisted blob: sha=${task.origId.name} " + s"path=${task.fullPath} reason=${entry.reason} citation=${entry.citation}. " + - s"See ${BlobDenylist.EntriesSource}. Reported, never fatal." + "The blob is left out of the rewritten tree, so it produces no blame and no dataset " + + "row, and the file is absent from the tokenized repository rather than present as raw " + + "source. The exclusion is deterministic and cited; the walk carries on and the run " + + "still exits 0. See " + + s"${BlobDenylist.EntriesSource} in blobExec for the list itself." ) } } @@ -1084,22 +1025,19 @@ final class Walker( blobsOversized.increment() System.err.println( s"blobExec: EXCLUDED oversized blob: sha=${task.origId.name} " + - s"path=${task.fullPath} size=${sizeBytes}B. JGit will not materialise an object " + - "this large, so the path is absent from the rewritten tree. Reported, not fatal." + s"path=${task.fullPath} size=${sizeBytes}B limit=${Walker.MaxBlobBytes}B. " + + "JGit will not materialise an object this large, and a source file this size is " + + "machine-generated rather than authored. The blob is left out of the rewritten " + + "tree, so it produces no blame and no dataset row, and the file is absent from " + + "the tokenized repository rather than present as raw source. The walk carries on " + + "and the run still exits 0." ) } } /** Copy one original blob from src into dst by streaming it. Returns (id, size). - * - * Used instead of readBlob on the pass-through path, where a blob keeps the - * size the project committed: `ObjectLoader.getBytes` throws - * LargeObjectException above JGit's stream threshold, and raising the - * threshold would still hold the whole blob in the heap. - * - * The reader stays open for the whole copy. Closing it before the stream is - * consumed would invalidate the stream. - */ + * Streams because a pass-through blob can exceed what `getBytes` will load. + * The reader must stay open until the stream is consumed. */ private def streamBlobInto(id: ObjectId, inserter: ObjectInserter): (ObjectId, Long) = { val r = src.newObjectReader() try { @@ -1126,6 +1064,7 @@ final class Walker( case TreeExisting(id) => id case TreeBuild(origId, entries) => val tf = new org.eclipse.jgit.lib.TreeFormatter + // flatMap: an excluded blob resolves to no entry, so the path is dropped. val resolvedEntries = entries.flatMap(resolveEntry(_, resolved, inserter, subtreeAcc)) // jgit requires sorted entries (git tree order). The TreeWalk visited // them in tree order already, so we keep that order. @@ -1160,11 +1099,12 @@ final class Walker( val key = (origId.name, fullPath) resolved.get(key) match { case Some(newId) => Some(ResolvedEntry(name, mode, newId, copyBytes = false)) - case None if oversizedKeys.contains(key) || denylistedKeys.contains(key) => None + case None if oversizedKeys.contains(key) || denylistedKeys.contains(key) || + failedKeys.contains(key) => None case None => throw new IllegalStateException( s"blob ${origId.name} ($fullPath) is a mask-matched miss with no resolution " + - "and was excluded neither as oversized nor by the blob denylist") + "and was excluded neither as oversized, by the blob denylist, nor as failed") } } @@ -1447,6 +1387,12 @@ final class Walker( object Walker { + /** JGit's default `core.streamFileThreshold`, at or above which `getBytes` + * throws. Only a fast path: [[Walker#readBlob]] also catches the exception. */ + private[blobexec] val MaxBlobBytes: Long = + new org.eclipse.jgit.storage.file.WindowCacheConfig().getStreamFileThreshold.toLong + + private[blobexec] def isOversized(sizeBytes: Long): Boolean = sizeBytes >= MaxBlobBytes private val OriginalBlobCacheSize = 1 << 16 private val OriginalBlobCopyLockCount = 256 @@ -1500,43 +1446,20 @@ object Walker { * memory stays modest. */ private val PipelineWindow = 32 + /** Generous: every child is separately hard-bounded at `--blob-timeout`. */ + private[blobexec] val DefaultStallTimeoutSeconds = 1800 + /** Exit status when the watchdog kills a stalled run. Distinct from 4 (a blob * timed out but the walk finished) so the runner can tell them apart. */ private[blobexec] val StalledExitStatus = 5 - private val WatchdogTickMarginSeconds = 60 - - private val StallTimeoutMultiple = 3 + /** Headroom for the watchdog's tick (30 s at most) and the reap of a killed child. */ + private val StallFloorMarginSeconds = 60 + /** Smallest stall window that cannot fire while one blob is still inside its own + * lifetime: a commit whose last blob is slow completes nothing until it ends. */ private[blobexec] def stallFloorFor(blobTimeoutSeconds: Int): Int = - Saturating.sum(ChildRunner.maxLifetimeSeconds(blobTimeoutSeconds), WatchdogTickMarginSeconds) - - private[blobexec] def stallTimeoutFor(blobTimeoutSeconds: Int): Int = - math.max( - Saturating.product(math.max(1, blobTimeoutSeconds), StallTimeoutMultiple), - stallFloorFor(blobTimeoutSeconds) - ) - - private[blobexec] val DefaultStallTimeoutSeconds: Int = - stallTimeoutFor(BlobExec.DefaultTimeoutSeconds) - - private[blobexec] def resolveStallTimeout( - blobTimeoutSeconds: Int, - stallTimeoutSeconds: Int, - stallExplicit: Boolean - ): Either[String, Int] = { - val floor = stallFloorFor(blobTimeoutSeconds) - if (stallTimeoutSeconds >= floor) Right(stallTimeoutSeconds) - else if (stallExplicit) - Left( - s"--stall-timeout=$stallTimeoutSeconds must be at least ${floor}s to go with " + - s"--blob-timeout=$blobTimeoutSeconds. A blob completing or being killed is the only " + - "progress a pure-blob commit makes, and killing one takes the budget plus the kill " + - "grace, so a smaller stall window kills runs whose blobs are merely slow. Try " + - s"--stall-timeout=${stallTimeoutFor(blobTimeoutSeconds)} with --blob-timeout=$blobTimeoutSeconds." - ) - else Right(stallTimeoutFor(blobTimeoutSeconds)) - } + Saturating.sum(ChildRunner.maxLifetimeSeconds(blobTimeoutSeconds), StallFloorMarginSeconds) /** Pure form of the watchdog's decision, so it can be tested without halting * a JVM: has more than `stallTimeoutSeconds` passed with no progress? */ @@ -1559,17 +1482,7 @@ object Walker { fullMessage: String ) - /** What [[Walker.resolveMisses]] hands back: `ids` is what the tree - * references, `timedOutKeys` is the subset that must not be persisted. */ - final case class MissResolution( - ids: IMap[(String, String), ObjectId], - abort: Boolean, - timedOutKeys: Set[(String, String)] - ) { - /** The rows that are genuinely correct and worth keeping: content-addressed - * tokenizations of blobs that did not time out. */ - def persistable: IMap[(String, String), ObjectId] = ids -- timedOutKeys - } + final case class MissResolution(ids: IMap[(String, String), ObjectId], abort: Boolean) /** Per-blob worker result handed from a pool thread back to the consumer. */ sealed trait BlobResult @@ -1578,13 +1491,8 @@ object Walker { final case class Resolved(newId: ObjectId) extends BlobResult /** abort-on-error tripped by a non-zero command exit. */ final case class Aborted(stderr: String, exitCode: Int) extends BlobResult - /** The command was killed on its budget. The tree still references the - * original blob (dst stays self-consistent), but this is deliberately not - * a `Resolved`: nothing about this blob, the trees containing it, or its - * commit may be persisted, or a re-run would treat raw source as done. */ - final case class Unusable(origId: ObjectId) extends BlobResult - /** Too large for JGit to materialise; contributes no id. */ - final case class Oversized(origId: ObjectId) extends BlobResult + /** Oversized, denylisted or failed: no id, so the path is left out of the tree. */ + final case class Excluded(origId: ObjectId) extends BlobResult } /** Items flowing producer -> consumer across the bounded queue. */ @@ -1608,11 +1516,7 @@ object Walker { final case class Built( newTreeId: ObjectId, resolved: IMap[(String, String), ObjectId], - subtrees: IMap[String, String], - /** Blobs in this commit whose command was killed on its budget. The tree - * is usable for this run's output, but if this is non-empty neither it - * nor the commit may be recorded, or the next run skips the retry. */ - timedOutKeys: Set[(String, String)] + subtrees: IMap[String, String] ) extends TreeResult final case class Aborted(stderr: String, exitCode: Int) extends TreeResult } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala index 88488fb5e..dc033b8bf 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala @@ -7,45 +7,77 @@ import org.scalatest.matchers.should.Matchers * entries are asserted rather than trusted. */ class BlobDenylistSpec extends AnyFunSuite with Matchers { - // Four historical versions of one file at two paths: the list is keyed on content. - private val KnownShas = Set( + // TestNewCastArray.java (4) and CheckErrorsForSource7.java (4): srcML/srcML#2361. + private val JavaAnnotationShas = Set( "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65", "303c1a109f1fdc4b9d84e15111c56931202844ac", "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca", - "c91924d1e87cb82d372a73b24579ccc34609efaa" + "c91924d1e87cb82d372a73b24579ccc34609efaa", + "56c8adf588e697a3e42b600b2e5ec54710592c61", + "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5", + "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7", + "afbd81e5789ba060147b0de3be4f01759963299d" ) - test("the shipped list holds exactly the known blobs, so an addition is deliberate") { + // StatTimes.h: the C parser on C++ in a `.h`. No upstream issue. + private val CParserShas = Set( + "5460d43f1884e3be230e02215151b9b963844a48", + "720914b27e170b73b38ef662fb04239273dd0a34", + "99c191e9a208547f5c9b1224c290f106a640de59", + "dbb885e0bf046755c4fcfbddfc2454f90c550c62" + ) + + private val shipped = BlobDenylistEntries.NonTerminating + + test("the shipped list holds exactly the twelve known hangs, so an addition is deliberate") { // Also catches a sha written twice: a Map literal silently keeps the last. - BlobDenylistEntries.NonTerminating.keySet shouldEqual KnownShas + shipped.keySet shouldEqual JavaAnnotationShas ++ CParserShas + BlobDenylist.shipped.size shouldEqual 12 } test("every key is a 40-character lowercase hex git blob id") { - // Not the memo's content hash, and not a path. - BlobDenylistEntries.NonTerminating.keys.foreach { sha => + shipped.keys.foreach { sha => withClue(s"$sha: ") { sha should fullyMatch regex "[0-9a-f]{40}" } } } - test("every entry cites a defect and says what it is") { - BlobDenylistEntries.NonTerminating.foreach { case (sha, e) => + test("every entry carries a one-line reason and the citation of its own defect") { + shipped.foreach { case (sha, e) => withClue(s"$sha: ") { - e.citation should not be empty + if (JavaAnnotationShas.contains(sha)) e.citation shouldEqual "srcML/srcML#2361" + else e.citation should include("srcml-nontermination-analysis.md") e.reason should not include "\n" + e.reason should include("srcML 1.1.0") + } + } + } + + test("the C parser entries do not claim to be srcML#2361, and carry their reproducer") { + // With no upstream issue, the reproducer in the reason is the evidence. + CParserShas.foreach { sha => + val e = BlobDenylist.shipped.entryFor(sha).getOrElse(fail(s"$sha is not denylisted")) + withClue(s"$sha: ") { + e.citation should not include "2361" + e.reason should include("namespace n{struct T&f(const struct S);}") } } } test("lookup is by git blob id, in either case, and misses everything else") { - val sha = KnownShas.head + val sha = JavaAnnotationShas.head BlobDenylist.shipped.entryFor(sha) should not be empty BlobDenylist.shipped.entryFor(sha.toUpperCase) should not be empty BlobDenylist.shipped.entryFor("0" * 40) shouldEqual None BlobDenylist.shipped.entryFor("") shouldEqual None } + test("a fixture list built from uppercase keys still matches lowercase ids") { + val entry = BlobDenylist.Entry("c", "r") + BlobDenylist(Map("AB" * 20 -> entry)).entryFor("ab" * 20) shouldEqual Some(entry) + } + test("an empty list matches nothing, so a test fixture cannot exclude by accident") { BlobDenylist.empty.isEmpty shouldBe true - BlobDenylist.empty.entryFor(KnownShas.head) shouldEqual None + BlobDenylist.empty.entryFor(JavaAnnotationShas.head) shouldEqual None } } diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala index dbef5ecb6..d973c6139 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobExecSpec.scala @@ -131,6 +131,16 @@ class BlobExecSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { crashes.get shouldEqual 1 } + test("a tokenizer killed by a signal is excluded like a parser crash, never kept as source") { + val crashes = new java.util.concurrent.atomic.AtomicInteger(0) + val cmd = shellScript("cat > /dev/null; kill -9 $$") + BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, + abortOnError = false, inserter, + onParserCrash = () => { crashes.incrementAndGet(); () } + ) shouldBe BlobExec.Outcome.Skip + crashes.get shouldEqual 1 + } + test("a parser crash skips one blob even with abortOnError, like a timeout") { val cmd = shellScript(s"exit ${BlobExec.ParserCrashExitCode}") BlobExec.run("int main(){}".getBytes(UTF_8), sampleSha, "x.c", "src/x.c", cmd, diff --git a/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala new file mode 100644 index 000000000..8674eab5f --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/FailedBlobSpec.scala @@ -0,0 +1,156 @@ +package cregit.blobexec + +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.internal.storage.file.FileRepository +import org.eclipse.jgit.lib.{ObjectId, Repository} +import org.eclipse.jgit.revwalk.RevWalk +import org.eclipse.jgit.storage.file.FileRepositoryBuilder +import org.eclipse.jgit.treewalk.TreeWalk +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +class FailedBlobSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val workRoot: Path = Files.createTempDirectory("failed-blob-") + + override def afterAll(): Unit = deleteRecursive(workRoot) + + private def tokenizer(dir: Path, onB: String): String = { + val f = dir.resolve("tok.sh") + Files.writeString( + f, + s"""|#!/bin/sh + |if [ "$$BFG_FILENAME" = "b.c" ]; then + | $onB + |else + | tr a-z A-Z + |fi + |""".stripMargin + ) + f.toFile.setExecutable(true) + f.toAbsolutePath.toString + } + + private val hang = "sleep 300 & sleep 300" + private val crash = s"cat >/dev/null; exit ${BlobExec.ParserCrashExitCode}" + + private def run(src: Repository, dstPath: Path, dbPath: Path, command: String, mode: String): WalkStats = { + val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] + dst.create(true) + val mapping = Mapping.open(dbPath, command, """\.c$""") + try + new Walker( + src, dst, mapping, """\.c$""".r, command, + abortOnError = false, + parallelism = 2, + pipeline = mode == "pipeline", + pipelineTrees = mode == "pipeline-trees", + blobTimeoutSeconds = 1 + ).run() + finally { + mapping.close() + dst.close() + } + } + + private final case class Fixture(src: Repository, commits: Vector[ObjectId], dir: Path) + + private def fixture(extraCommitInDeep: Boolean): Fixture = { + val dir = Files.createTempDirectory(workRoot, "fixture-") + val srcDir = dir.resolve("src") + Files.createDirectories(srcDir.resolve("deep")) + val git = Git.init().setDirectory(srcDir.toFile).call() + def commit(msg: String) = + git.commit().setMessage(msg).setAuthor("Tester", "t@example.org") + .setCommitter("Tester", "t@example.org").call().getId + Files.writeString(srcDir.resolve("a.c"), "alpha\n") + Files.writeString(srcDir.resolve("deep/b.c"), "beta\n") + git.add().addFilepattern(".").call() + val first = commit("two blobs, one of which fails") + val rest = + if (!extraCommitInDeep) Vector.empty + else { + Files.writeString(srcDir.resolve("deep/c.c"), "gamma\n") + git.add().addFilepattern(".").call() + Vector(commit("a sibling of the failing blob")) + } + Fixture(git.getRepository, first +: rest, dir) + } + + for (mode <- Seq("serial", "pipeline", "pipeline-trees")) { + for ((kind, onB) <- Seq("timed-out" -> hang, "crashed" -> crash)) + test(s"[$mode] a $kind blob is excluded and the commit is still folded") { + val f = fixture(extraCommitInDeep = false) + val dstPath = f.dir.resolve("dst.git") + val dbPath = f.dir.resolve("blobmap.db") + val command = tokenizer(f.dir, onB) + + val stats = run(f.src, dstPath, dbPath, command, mode) + + stats.aborted shouldBe false + stats.commitsProcessed shouldEqual 1 + (stats.blobsTimedOut + stats.blobsParserCrashed) shouldEqual 1 + Main.exitStatus(stats) shouldEqual 0 + val m = Mapping.open(dbPath, command, """\.c$""") + try { + val newCommit = m.getCommit(f.commits.head.name) + newCommit shouldBe defined + filesIn(dstPath, newCommit.get) shouldEqual Map("a.c" -> "ALPHA\n") + m.getBlob(blobIdAt(f.src, f.commits.head, "deep/b.c").name, "deep/b.c") shouldEqual None + } finally m.close() + } + + test(s"[$mode] a failed blob is tried once per run, not once per commit") { + val f = fixture(extraCommitInDeep = true) + val dstPath = f.dir.resolve("dst.git") + val dbPath = f.dir.resolve("blobmap.db") + val command = tokenizer(f.dir, hang) + + val stats = run(f.src, dstPath, dbPath, command, mode) + + stats.commitsProcessed shouldEqual 2 + stats.blobsTimedOut shouldEqual 1 + val m = Mapping.open(dbPath, command, """\.c$""") + try { + val second = m.getCommit(f.commits(1).name) + second shouldBe defined + filesIn(dstPath, second.get) shouldEqual Map("a.c" -> "ALPHA\n", "deep/c.c" -> "GAMMA\n") + } finally m.close() + } + } + + private def blobIdAt(repo: Repository, commitId: ObjectId, path: String): ObjectId = { + val rw = new RevWalk(repo) + try { + val tw = TreeWalk.forPath(repo, path, rw.parseCommit(commitId).getTree) + try tw.getObjectId(0) finally tw.close() + } finally rw.close() + } + + private def filesIn(dstPath: Path, commitSha: String): Map[String, String] = { + val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] + val rw = new RevWalk(dst) + val tw = new TreeWalk(dst) + try { + tw.addTree(rw.parseCommit(ObjectId.fromString(commitSha)).getTree) + tw.setRecursive(true) + val out = Map.newBuilder[String, String] + while (tw.next()) + out += tw.getPathString -> new String(dst.open(tw.getObjectId(0)).getBytes, UTF_8) + out.result() + } finally { tw.close(); rw.close(); dst.close() } + } + + private def deleteRecursive(p: Path): Unit = { + if (Files.isDirectory(p)) { + val s = Files.list(p) + try s.forEach(deleteRecursive) finally s.close() + } + Files.deleteIfExists(p) + () + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala index 2ae5f1aad..d5637fb2b 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MainOptionsSpec.scala @@ -3,12 +3,10 @@ package cregit.blobexec import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -/** `--blob-timeout` and `--stall-timeout` are binding parts of the spec: 600 and - * 1800 by default, and values that are not a positive whole number of seconds - * must be rejected rather than silently read as "no limit". The shared value - * parser is tested here; `main` itself cannot be, because it answers bad input - * with `sys.exit`. The stall watchdog's decision is tested here too, because - * the watchdog itself halts the JVM and so cannot be exercised in-process. */ +import java.nio.file.Files + +/** The option parser, the 600/1800 budget defaults and the stall watchdog's decision. + * `main` itself answers with `sys.exit`, so its parts are tested instead. */ class MainOptionsSpec extends AnyFunSuite with Matchers { test("the default per-blob budget is 600 seconds") { @@ -39,41 +37,39 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.parsePositiveSeconds("10.5") shouldEqual None } - test("timeout and stall have distinct exit statuses, and neither collides") { - Main.TimedOutExitStatus shouldEqual 4 + test("the stall status collides with no other exit status") { Walker.StalledExitStatus shouldEqual 5 - Main.TimedOutExitStatus should not equal Walker.StalledExitStatus - Set(0, 1, 2, 3) should not contain Main.TimedOutExitStatus Set(0, 1, 2, 3) should not contain Walker.StalledExitStatus } // -- the two timeouts are coupled ------------------------------------------- test("a window at or above the floor is accepted unchanged") { - Walker.resolveStallTimeout(600, 1800, stallExplicit = false) shouldEqual Right(1800) - Walker.resolveStallTimeout(600, 1800, stallExplicit = true) shouldEqual Right(1800) - Walker.resolveStallTimeout(600, Walker.stallFloorFor(600), stallExplicit = true) shouldEqual + Main.resolveStallTimeout(600, 1800, stallExplicit = false) shouldEqual Right(1800) + Main.resolveStallTimeout(600, 1800, stallExplicit = true) shouldEqual Right(1800) + Main.resolveStallTimeout(600, Walker.stallFloorFor(600), stallExplicit = true) shouldEqual Right(Walker.stallFloorFor(600)) } test("the floor is the child's whole lifetime, not just the budget") { Walker.stallFloorFor(600) should be > ChildRunner.maxLifetimeSeconds(600) - Walker.resolveStallTimeout(600, 601, stallExplicit = true).isLeft shouldBe true - Walker.resolveStallTimeout(600, 630, stallExplicit = true).isLeft shouldBe true + Main.resolveStallTimeout(600, 601, stallExplicit = true).isLeft shouldBe true + Main.resolveStallTimeout(600, 630, stallExplicit = true).isLeft shouldBe true } test("a refusal names the floor it wants, and the suggestion clears it") { - val why = Walker.resolveStallTimeout(600, 601, stallExplicit = true).swap.getOrElse("") + val why = Main.resolveStallTimeout(600, 601, stallExplicit = true).swap.getOrElse("") why should include(Walker.stallFloorFor(600).toString) - val suggested = Walker.stallTimeoutFor(600) - Walker.resolveStallTimeout(600, suggested, stallExplicit = true) shouldEqual Right(suggested) + val suggested = Main.widenedStall(600) + Main.resolveStallTimeout(600, suggested, stallExplicit = true) shouldEqual Right(suggested) } - test("a small budget is widened past the fixed kill overhead, not just tripled") { - Walker.resolveStallTimeout(5, 3, stallExplicit = false) - .getOrElse(0) should be >= Walker.stallFloorFor(5) - Walker.resolveStallTimeout(1, 1, stallExplicit = false) - .getOrElse(0) should be >= Walker.stallFloorFor(1) + test("a small --blob-timeout no longer lets the watchdog stop a healthy run") { + val oneBlobLifetimeNanos = ChildRunner.maxLifetimeSeconds(5).toLong * 1000000000L + Main.resolveStallTimeout(5, 15, stallExplicit = true).isLeft shouldBe true + val window = Main.resolveStallTimeout(5, 3, stallExplicit = false).getOrElse(0) + Walker.isStalled(oneBlobLifetimeNanos, 0L, window) shouldBe false + Main.resolveStallTimeout(1, 1, stallExplicit = false).getOrElse(0) should be >= Walker.stallFloorFor(1) } test("the floor cannot overflow into a guard that always passes") { @@ -82,7 +78,7 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("the defaults satisfy the relationship") { - Walker.resolveStallTimeout( + Main.resolveStallTimeout( BlobExec.DefaultTimeoutSeconds, Walker.DefaultStallTimeoutSeconds, stallExplicit = false @@ -90,75 +86,33 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { } test("a defaulted window is widened to fit a raised --blob-timeout") { - Walker.resolveStallTimeout(1800, 1800, stallExplicit = false) shouldEqual Right(5400) - Walker.resolveStallTimeout(3600, 1800, stallExplicit = false) shouldEqual Right(10800) + Main.resolveStallTimeout(1800, 1800, stallExplicit = false) shouldEqual Right(5400) + Main.resolveStallTimeout(3600, 1800, stallExplicit = false) shouldEqual Right(10800) } test("an explicit window that is too small is refused, naming both values") { - val bad = Walker.resolveStallTimeout(1800, 1800, stallExplicit = true) + val bad = Main.resolveStallTimeout(1800, 1800, stallExplicit = true) bad.isLeft shouldBe true val why = bad.swap.getOrElse("") why should include("--stall-timeout=1800") why should include("--blob-timeout=1800") - Walker.resolveStallTimeout(600, 60, stallExplicit = true).isLeft shouldBe true + Main.resolveStallTimeout(600, 60, stallExplicit = true).isLeft shouldBe true } test("widening cannot overflow Int") { - Walker.resolveStallTimeout(Int.MaxValue, 1800, stallExplicit = false) shouldEqual Right(Int.MaxValue) + Main.resolveStallTimeout(Int.MaxValue, 1800, stallExplicit = false) shouldEqual Right(Int.MaxValue) } - // -- the stall window's floor ----------------------------------------------- - - test("the floor clears one blob's whole lifetime, kill path included") { - Walker.stallFloorFor(600) should be > ChildRunner.maxLifetimeSeconds(600) - Walker.stallFloorFor(1) should be > ChildRunner.maxLifetimeSeconds(1) - } - - test("the default window is derived from the blob budget, not a second literal") { - Walker.DefaultStallTimeoutSeconds shouldEqual Walker.stallTimeoutFor(BlobExec.DefaultTimeoutSeconds) - } - - test("a defaulted window follows a raised budget past the floor") { - Walker.stallTimeoutFor(1800) shouldEqual 5400 - Walker.stallTimeoutFor(5) should be >= Walker.stallFloorFor(5) - Walker.stallTimeoutFor(1) should be >= Walker.stallFloorFor(1) - } - - test("a blob budget past the default window raises the floor above it") { - Walker.stallFloorFor(3600) should be > Walker.DefaultStallTimeoutSeconds - } - - test("the floor grows with the blob budget, and never underflows") { - Walker.stallFloorFor(60) should be < Walker.stallFloorFor(600) - Walker.stallFloorFor(0) should be > 0 - Walker.stallFloorFor(-5) should be > 0 - } - - test("a run at exactly the floor is not stalled by its own slowest blob") { - val blobBudget = 3600 - val floor = Walker.stallFloorFor(blobBudget) - val quiet = ChildRunner.maxLifetimeSeconds(blobBudget).toLong * 1000000000L - Walker.isStalled(nowNanos = quiet, lastProgressNanos = 0L, floor) shouldBe false - } - - test("the accepted range of a timeout keeps every derived window inside Int") { - Main.parsePositiveSeconds((Main.MaxTimeoutSeconds + 1).toString) shouldEqual None - Main.parsePositiveSeconds(Int.MaxValue.toString) shouldEqual None - Walker.stallTimeoutFor(Main.MaxTimeoutSeconds) should be > 0 - ChildRunner.maxLifetimeSeconds(Main.MaxTimeoutSeconds) should be > 0 - } - - // -- which counters gate publication ---------------------------------------- + // -- which counters change the exit status ---------------------------------- private def stats( aborted: Boolean = false, blobsTimedOut: Long = 0L, blobsOversized: Long = 0L, blobsDenylisted: Long = 0L, - blobsParserCrashed: Long = 0L, - blobsTokenized: Int = 1 + blobsParserCrashed: Long = 0L ) = WalkStats( - commitsProcessed = 1, commitsAlreadyMapped = 0, blobsRunThroughCommand = blobsTokenized, + commitsProcessed = 1, commitsAlreadyMapped = 0, blobsRunThroughCommand = 1, blobsCacheHit = 0, refsProjected = 1, aborted = aborted, blobsTimedOut = blobsTimedOut, blobsOversized = blobsOversized, blobsDenylisted = blobsDenylisted, blobsParserCrashed = blobsParserCrashed, @@ -183,10 +137,11 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.exitStatus(stats(blobsOversized = 3, blobsDenylisted = 4)) shouldEqual 0 } - test("a timeout still blocks publication, even alongside a denylisted blob") { - Main.exitStatus(stats(blobsTimedOut = 1)) shouldEqual Main.TimedOutExitStatus - Main.exitStatus(stats(blobsTimedOut = 1, blobsDenylisted = 4)) shouldEqual - Main.TimedOutExitStatus + test("a failed blob does not change the exit status: it is excluded") { + Main.exitStatus(stats(blobsTimedOut = 1)) shouldEqual 0 + Main.exitStatus(stats(blobsParserCrashed = 36)) shouldEqual 0 + Main.exitStatus(stats(blobsTimedOut = 1, blobsParserCrashed = 1, blobsDenylisted = 4, + blobsOversized = 3)) shouldEqual 0 } test("an abort still wins over everything") { @@ -195,44 +150,26 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { Main.exitStatus(stats(aborted = true, blobsParserCrashed = 1)) shouldEqual 2 } - test("a crash rate past tolerance blocks publication, with its own status") { - Main.exitStatus(stats(blobsParserCrashed = 2, blobsTokenized = 100)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("a crash rate within tolerance still publishes: the blob is excluded either way") { - Main.exitStatus(stats(blobsParserCrashed = 1, blobsTokenized = 100)) shouldEqual 0 - Main.exitStatus(stats(blobsParserCrashed = 500, blobsTokenized = 500000)) shouldEqual 0 - } - - test("one crash in a tiny project does not gate, so rounding cannot zero the tolerance") { - Main.exitStatus(stats(blobsParserCrashed = 1, blobsTokenized = 1)) shouldEqual 0 - Main.exitStatus(stats(blobsParserCrashed = 2, blobsTokenized = 1)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("a crash past tolerance still blocks alongside explained exclusions") { - Main.exitStatus(stats(blobsParserCrashed = 9, blobsTokenized = 100, - blobsDenylisted = 4, blobsOversized = 3)) shouldEqual - Main.ParserCrashedExitStatus - } - - test("the parser-crash status collides with no other blobExec exit status") { + // run_pipeline_process.sh maps each status to its own remedy and marker file. + test("the ineffective-retokenize status collides with no other blobExec exit status") { val others = Map( - "clean" -> 0, - "usage" -> 1, - "aborted" -> 2, - "maskChanged" -> 3, - "timedOut" -> Main.TimedOutExitStatus, - "stalled" -> Walker.StalledExitStatus + "clean" -> 0, + "usage" -> 1, + "aborted" -> 2, + "maskChanged" -> 3, + "stalled" -> Walker.StalledExitStatus ) others.foreach { case (name, status) => - withClue(s"parser-crash status must differ from $name ($status): ") { - Main.ParserCrashedExitStatus should not equal status + withClue(s"ineffective-retokenize status must differ from $name ($status): ") { + Main.RetokenizeIneffectiveExitStatus should not equal status } } } + test("an ineffective --retokenize is not reported as a clean walk") { + Main.RetokenizeIneffectiveExitStatus should not equal 0 + } + // -- the watchdog's decision ------------------------------------------------ private val second = 1000000000L @@ -262,4 +199,71 @@ class MainOptionsSpec extends AnyFunSuite with Matchers { test("nanoTime is monotonic but not epoch-based, so negative elapsed is never a stall") { Walker.isStalled(nowNanos = -5 * second, lastProgressNanos = -3 * second, 30) shouldBe false } + // -- the option parser -------------------------------------------------------- + + private lazy val srcDir = Files.createTempDirectory("main-options-src-") + private def positional = Seq(srcDir.toString, "/tmp/dst.git", "/tmp/db.sqlite", "/bin/sh", "\\.c$") + + test("parse reads every flag into one options value") { + val parsed = Main.parse(Seq("--abort-on-error", "--pipeline", "--mask-widened", + "--blob-timeout=60", "--stall-timeout=900") ++ positional) + inside(parsed) { case Right(o) => + (o.abortOnError, o.pipeline, o.maskWidened) shouldEqual ((true, true, true)) + (o.blobTimeoutSeconds, o.stallTimeoutSeconds, o.stallExplicit) shouldEqual ((60, 900, true)) + (o.command, o.mask) shouldEqual (("/bin/sh", "\\.c$")) + } + } + + test("a shard is read as K and N") { + inside(Main.parse("--shard=1/4" +: positional)) { case Right(o) => o.shard shouldEqual Some((1, 4)) } + } + + test("parse leaves the defaults when no flag is given") { + inside(Main.parse(positional)) { case Right(o) => + o shouldEqual Main.Options(positional = positional.toVector) + } + } + + test("an unknown flag is refused with the usage text") { + inside(Main.parse("--frobnicate" +: positional)) { case Left(why) => + why should include("unknown flag [--frobnicate]") + why should include(Main.Usage) + } + } + + test("a bad flag value is refused, naming the flag") { + Seq( + "--blob-timeout=0" -> "--blob-timeout", + "--stall-timeout=x" -> "--stall-timeout", + "--shard=4/4" -> "--shard", + "--shard=1" -> "--shard", + "--warm=/nonexistent/db" -> "--warm", + "--tokenizer-identity=" -> "--tokenizer-identity", + "--memo-dir=/nonexistent/" -> "--memo-dir" + ).foreach { case (flag, name) => + withClue(s"$flag: ") { + inside(Main.parse(flag +: positional)) { case Left(why) => why should startWith(s"Error: $name") } + } + } + } + + test("a wrong number of positional arguments prints the usage text alone") { + Main.parse(positional.dropRight(1)) shouldEqual Left(Main.Usage) + } + + test("every option description in the usage text is at most three lines, and most are one") { + val continuation = "^ {29}\\S".r + val entries = Main.Usage.linesIterator.foldLeft(Vector.empty[Vector[String]]) { + case (acc, line) if line.startsWith(" --") || line.startsWith(" <") => acc :+ Vector(line) + case (acc, line) if acc.nonEmpty && continuation.findFirstIn(line).isDefined => acc.init :+ (acc.last :+ line) + case (acc, _) => acc + } + val lengths = entries.map(e => e.size - (if (e.head.trim.contains(' ')) 0 else 1)) + entries should not be empty + all(lengths) should be <= 3 + lengths.count(_ == 1) should be > entries.size / 2 + } + + private def inside[T](v: T)(pf: PartialFunction[T, Unit]): Unit = + if (pf.isDefinedAt(v)) pf(v) else fail(s"value did not match: $v") } diff --git a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala index e24139c99..46af59266 100644 --- a/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/MappingSpec.scala @@ -126,8 +126,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } finally Files.deleteIfExists(tmp) } - // A mask change invalidates tree_map: rows assembled under the old mask omit - // every newly selected file, and the trees still look complete afterwards. + // Reused tree_map rows would silently omit every newly selected file, so a + // mask change must rebuild. Pinned with the real mask strings. test("a widened mask is refused on resume, not silently reused") { val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" @@ -147,7 +147,6 @@ class MappingSpec extends AnyFunSuite with Matchers { ex.getMessage should include(oldMask) ex.getMessage should include(newMask) - // Same mask still resumes. val m2 = Mapping.open(tmp, "/bin/cat", oldMask) try { m2.getTree("origtree") shouldBe Some("newtree") @@ -157,7 +156,8 @@ class MappingSpec extends AnyFunSuite with Matchers { } test("the warm-DB fallback refuses a widened mask too") { - // shard_build.sh passes a prior run's DB as --warm; openWarm has its own check. + // openWarm (shard_build.sh --warm) has its own mask check; it must not drift + // from checkOrSetMeta's. val oldMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" val newMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" val warm = Files.createTempFile("mapping-warm-", ".db") @@ -177,6 +177,262 @@ class MappingSpec extends AnyFunSuite with Matchers { } } + // --mask-widened. Trap: blob_map holds identity rows for blobs the old mask + // skipped; a wider mask selects some, and a cache hit would ship raw source. + + private val OldCMask = """\.[ch]$""" + private val OldCppMask = """\.(c|cc|cp|cpp|cxx|h|hh|hpp)$""" + private val UniversalMask = """(?i)\.(c|c\+\+|cc|cp|cpp|cxx|h|h\+\+|hh|hpp|hxx|java|rs|tcc)$""" + + /** A widening that says every id resolves, collecting the report. */ + private def widening(resolves: String => Boolean = _ => true) = { + val log = new StringBuilder + (Mapping.MaskWidening( + newBlobResolves = resolves, + report = m => { log.append(m); log.append('\n'); () }, + nowEpochSeconds = () => 1700000000L + ), log) + } + + /** Tokenized rows for selected paths, identity rows for the rest, plus trees, a commit and a ref. */ + private def seedUnderOldMask(tmp: java.nio.file.Path, mask: String): Unit = { + val m = Mapping.open(tmp, "/bin/cat", mask) + try { + m.putBlob("b1", "src/a.c", "tok1") + m.putBlob("b2", "src/b.h", "tok2") + // Identity rows the universal mask selects: reusing them would serve raw source. + m.putBlob("b3", "src/a.cpp", "b3") + m.putBlob("b4", "src/T.java", "b4") + // Never selected by any mask. + m.putBlob("b5", "README.md", "b5") + m.putBlob("b6", "CMakeLists.txt", "b6") + m.putTree("t1", "nt1") + m.putCommit("c1", "nc1") + m.putRef(Mapping.RefRow("refs/heads/main", "head", "c1", "nc1", "c1", "nc1")) + } finally m.close() + } + + test("without the flag a mask change is still refused: the default does not move") { + val tmp = Files.createTempFile("mapping-widen-default-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val ex = intercept[Mapping.MetaMismatchException] { + Mapping.open(tmp, "/bin/cat", UniversalMask) + } + ex.getMessage should include("mask") + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + m.getMeta("mask") shouldBe Some(OldCMask) + m.getTree("t1") shouldBe Some("nt1") + m.getCommit("c1") shouldBe Some("nc1") + m.getBlob("b3", "src/a.cpp") shouldBe Some("b3") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("with the flag: tokenized rows are kept, identity rows and trees and commits go") { + val tmp = Files.createTempFile("mapping-widen-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, log) = widening() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + try { + m.getBlob("b1", "src/a.c") shouldBe Some("tok1") + m.getBlob("b2", "src/b.h") shouldBe Some("tok2") + m.getBlob("b3", "src/a.cpp") shouldBe None + m.getBlob("b4", "src/T.java") shouldBe None + m.getBlob("b5", "README.md") shouldBe None + m.getBlob("b6", "CMakeLists.txt") shouldBe None + m.getTree("t1") shouldBe None + m.getCommit("c1") shouldBe None + m.getRef("refs/heads/main") shouldBe None + m.allCommitOrigShas shouldBe empty + m.getMeta("mask") shouldBe Some(UniversalMask) + m.getMeta(Mapping.MaskWidenedFromKey) shouldBe Some(OldCMask) + m.getMeta(Mapping.MaskWidenedAtKey) shouldBe Some("1700000000") + } finally m.close() + log.toString should include("KEPT 2 tokenized blob_map rows") + log.toString should include("DROPPED 4 identity blob_map rows") + log.toString should include("DROPPED 1 tree_map rows") + } finally Files.deleteIfExists(tmp) + } + + test("after a widening the same run resumes normally, with no flag needed") { + val tmp = Files.createTempFile("mapping-widen-again-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, _) = widening() + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)).close() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask) + try m.getBlob("b1", "src/a.c") shouldBe Some("tok1") finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("a NARROWING mask is refused, names the path, and leaves the databases untouched") { + val tmp = Files.createTempFile("mapping-narrow-", ".db") + try { + val seed = Mapping.open(tmp, "/bin/cat", OldCppMask) + try { + seed.putBlob("b1", "src/a.c", "tok1") + seed.putBlob("b2", "src/a.cpp", "tok2") + seed.putBlob("b3", "README.md", "b3") + seed.putTree("t1", "nt1") + seed.putCommit("c1", "nc1") + seed.putRef(Mapping.RefRow("refs/heads/main", "head", "c1", "nc1", "c1", "nc1")) + } finally seed.close() + + val (w, log) = widening() + val ex = intercept[Mapping.MaskNarrowedException] { + Mapping.open(tmp, "/bin/cat", OldCMask, None, Some(w)) + } + ex.getMessage should include("src/a.cpp") + ex.getMessage should include(OldCMask) + ex.getMessage should include(OldCppMask) + ex.getMessage should include("NARROWING") + log.toString shouldBe "" + + val m = Mapping.open(tmp, "/bin/cat", OldCppMask) + try { + m.getMeta("mask") shouldBe Some(OldCppMask) + m.getMeta(Mapping.MaskWidenedFromKey) shouldBe None + m.getBlob("b1", "src/a.c") shouldBe Some("tok1") + m.getBlob("b2", "src/a.cpp") shouldBe Some("tok2") + m.getBlob("b3", "README.md") shouldBe Some("b3") + m.getTree("t1") shouldBe Some("nt1") + m.getCommit("c1") shouldBe Some("nc1") + m.getRef("refs/heads/main").map(_.newCommit) shouldBe Some("nc1") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + // Every blob map holds such rows (README.md, ...); counting them would refuse + // every project. + test("identity rows on never-selected paths do not count as a narrowing") { + val tmp = Files.createTempFile("mapping-identity-ok-", ".db") + try { + val seed = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + seed.putBlob("b1", "src/a.c", "tok1") + seed.putBlob("b5", "README.md", "b5") + seed.putBlob("b6", "docs/manual.pdf", "b6") + seed.putBlob("b7", "Makefile", "b7") + } finally seed.close() + val (w, _) = widening() + val m = Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + try m.getMeta("mask") shouldBe Some(UniversalMask) finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the narrowing check reads the basename, the way Walker matches the mask") { + // Walker matches tw.getNameString, not the full path. + Mapping.basename("src/kernel/a.c") shouldEqual "a.c" + Mapping.basename("a.c") shouldEqual "a.c" + Mapping.basename("dir.c/plain") shouldEqual "plain" + } + + test("a new_blob missing from dst is refused, and nothing is changed") { + val tmp = Files.createTempFile("mapping-dangling-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, log) = widening(resolves = _ => false) + val ex = intercept[Mapping.DanglingNewBlobException] { + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)) + } + ex.getMessage should include("does not exist in the destination repository") + ex.getMessage should include("step 2") + log.toString shouldBe "" + + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + m.getMeta("mask") shouldBe Some(OldCMask) + m.getTree("t1") shouldBe Some("nt1") + m.getBlob("b3", "src/a.cpp") shouldBe Some("b3") + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the reachability check probes only retained rows, never identity ones") { + // An identity row's new_blob is the source blob id, which need not be in dst + // yet; probing it would refuse a good resume. + val tmp = Files.createTempFile("mapping-probe-set-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val probed = scala.collection.mutable.Set.empty[String] + val (w, _) = widening(resolves = id => { probed += id; true }) + Mapping.open(tmp, "/bin/cat", UniversalMask, None, Some(w)).close() + probed shouldEqual Set("tok1", "tok2") + } finally Files.deleteIfExists(tmp) + } + + test("the command is still checked under the flag: a different tokenizer is refused") { + // Widening is about which blobs, not about a different tokenizer. + val tmp = Files.createTempFile("mapping-widen-cmd-", ".db") + try { + seedUnderOldMask(tmp, OldCMask) + val (w, _) = widening() + val ex = intercept[Mapping.MetaMismatchException] { + Mapping.open(tmp, "/bin/echo", UniversalMask, None, Some(w)) + } + ex.getMessage should include("command") + } finally Files.deleteIfExists(tmp) + } + + test("the flag is a no-op on a fresh blob map and on an unchanged mask") { + val fresh = Files.createTempFile("mapping-widen-fresh-", ".db") + Files.deleteIfExists(fresh) + try { + val (w1, log1) = widening() + val m1 = Mapping.open(fresh, "/bin/cat", UniversalMask, None, Some(w1)) + try m1.getMeta("mask") shouldBe Some(UniversalMask) finally m1.close() + log1.toString should include("nothing to reuse") + log1.toString should include("nothing to check") + + val (w2, log2) = widening() + val m2 = Mapping.open(fresh, "/bin/cat", UniversalMask, None, Some(w2)) + try m2.getMeta(Mapping.MaskWidenedFromKey) shouldBe None finally m2.close() + log2.toString should include("ordinary resume") + } finally Files.deleteIfExists(fresh) + } + + test("rowCounts separates tokenized rows from identity rows") { + withMapping() { m => + m.putBlob("b1", "a.c", "tok1") + m.putBlob("b2", "b.c", "tok2") + m.putBlob("b3", "README.md", "b3") + m.putTree("t1", "nt1") + m.putCommit("c1", "nc1") + val c = m.rowCounts + c.blobTokenized shouldEqual 2L + c.blobIdentity shouldEqual 1L + c.tree shouldEqual 1L + c.commit shouldEqual 1L + c.ref shouldEqual 0L + } + } + + test("the sample is spread across the table, bounded, and deterministic") { + val tmp = Files.createTempFile("mapping-sample-", ".db") + try { + val m = Mapping.open(tmp, "/bin/cat", OldCMask) + try { + (0 until 100).foreach(i => m.putBlob(f"orig$i%03d", f"f$i%03d.c", f"tok$i%03d")) + m.sampleTokenizedNewBlobs(10) should have size 10 + // A spread, not the head: a head sample only sees the first commit. + m.sampleTokenizedNewBlobs(10).map(_._2) shouldEqual + m.sampleTokenizedNewBlobs(10).map(_._2) + m.sampleTokenizedNewBlobs(10).map(_._2).toSet should contain("tok090") + m.sampleTokenizedNewBlobs(1000) should have size 100 + } finally m.close() + } finally Files.deleteIfExists(tmp) + } + + test("the sample of an all-identity blob map is empty rather than an error") { + withMapping() { m => + m.putBlob("b1", "README.md", "b1") + m.sampleTokenizedNewBlobs(16) shouldBe empty + } + } + test("allCommitOrigShas returns all rows") { withMapping() { m => m.putCommit("a", "x") diff --git a/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala new file mode 100644 index 000000000..85b8c5af1 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/RetokenizeSpec.scala @@ -0,0 +1,449 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} + +/** Tokenizer-identity recording, refusal and `--retokenize`, at the blob-map level. */ +class RetokenizeSpec extends AnyFunSuite with Matchers { + + private val cmd = "/bin/cat" + private val mask = """(?i)\.(c|h|rs)$""" + + private val rustV1 = "1111111111111111" + private val rustV2 = "2222222222222222" + private val cV1 = "aaaaaaaaaaaaaaaa" + private val cV2 = "bbbbbbbbbbbbbbbb" + + private def id(pairs: (String, String)*) = TokenizerIdentity(pairs.toMap) + + private def withDb[A](body: Path => A): A = { + val d = Files.createTempDirectory("retok-") + try body(d.resolve("map.sqlite")) + finally { + def rm(p: Path): Unit = { + if (Files.isDirectory(p)) { val s = Files.list(p); try s.forEach(rm) finally s.close() } + Files.deleteIfExists(p) + () + } + rm(d) + } + } + + /** A finished run: two `.rs` rows, two `.c` rows and one identity row. */ + private def seed(db: Path, identity: TokenizerIdentity): Unit = { + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = identity) + try { + m.putBlob("rsblob1", "src/one.rs", "newrs1") + m.putBlob("rsblob2", "src/two.rs", "newrs2") + m.putBlob("cblob1", "src/one.c", "newc1") + m.putBlob("cblob2", "lib/two.h", "newc2") + m.putBlob("mdblob", "README.md", "mdblob") // identity row: not selected + m.putTree("tree1", "newtree1") + m.putCommit("commit1", "newcommit1") + m.putRef(Mapping.RefRow("refs/heads/master", "head", "commit1", "newcommit1", "commit1", "newcommit1")) + } finally m.close() + } + + private def resolvesEverything: String => Boolean = _ => true + + private def retokenize( + exts: Set[String], + purge: Vector[String] => TokenizerMemo.PurgeReport = + blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L), + resolves: String => Boolean = resolvesEverything, + report: String => Unit = _ => (), + memoDirIsTokenizerMemo: Boolean = false + ): Mapping.Retokenize = + Mapping.Retokenize( + extensions = exts, + newBlobResolves = resolves, + purgeMemo = purge, + report = report, + memoDirIsTokenizerMemo = memoDirIsTokenizerMemo, + nowEpochSeconds = () => 1700000000L) + + private val emptyMemo: Vector[String] => TokenizerMemo.PurgeReport = + blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, 0L, blobs.size.toLong, 0L) + + test("an identity absent from meta is recorded, so the first run establishes it") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1, "c" -> cV1)) + try { + m.storedTokenizerId("rs") shouldBe Some(rustV1) + m.storedTokenizerId("c") shouldBe Some(cV1) + } finally m.close() + } + } + + test("an unchanged identity is an ordinary resume and changes nothing") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.rowCounts.blobTokenized shouldEqual 4L + finally m.close() + } + } + + test("a CHANGED identity refuses the run; this is the bug, and the refusal is the fix") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val ex = intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1)) + } + ex.getMessage should include("rs") + ex.getMessage should include(rustV1) + ex.getMessage should include(rustV2) + ex.getMessage should include("--retokenize=rs") + // Naming the unchanged extension would tell the operator to discard valid work. + ex.getMessage should not include "--retokenize=c,rs" + } + } + + test("a refusal writes nothing: the map is exactly as it was") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV2)) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("no identity passed at all leaves behaviour exactly as it was") { + // Already-published projects must not start refusing. + withDb { db => + seed(db, TokenizerIdentity.empty) + val m = Mapping.open(db, cmd, mask) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.storedTokenizerId("rs") shouldBe None + } finally m.close() + } + } + + test("a changed identity for an extension with no cached rows is recorded, not refused") { + // Every project gets an identity for every extension the checkout parses, so a + // refusal here would have no remedy: --retokenize would have nothing to drop. + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "9999999999999999")) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "8888888888888888")) + try { + m.storedTokenizerId("java") shouldBe Some("8888888888888888") + m.rowCounts.blobTokenized shouldEqual 4L + } finally m.close() + } + } + + test("a changed identity IS refused as soon as one cached row carries the extension") { + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "9999999999999999")) + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m0.putBlob("javablob", "src/One.java", "newjava") finally m0.close() + intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "8888888888888888")) + } + } + } + + test("--retokenize drops only the named extension's tokenized rows") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + var reported = "" + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), report = s => reported = s))) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe None + m.getBlob("rsblob2", "src/two.rs") shouldBe None + // The C rows survive: what makes this cheaper than --drop-memo. + m.getBlob("cblob1", "src/one.c") shouldBe Some("newc1") + m.getBlob("cblob2", "lib/two.h") shouldBe Some("newc2") + m.storedTokenizerId("rs") shouldBe Some(rustV2) + m.storedTokenizerId("c") shouldBe Some(cV1) + reported should include("rs") + } finally m.close() + } + } + + test("--retokenize drops tree_map, commit_map and ref_map, because trees name blobs") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try { + // A retained tree_map row would skip the re-walk and keep the old token blob. + m.getTree("tree1") shouldBe None + m.getCommit("commit1") shouldBe None + m.getRef("refs/heads/master") shouldBe None + } finally m.close() + } + } + + test("--retokenize keeps the identity pass-through rows, which are not tokenizations") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try m.getBlob("mdblob", "README.md") shouldBe Some("mdblob") + finally m.close() + } + } + + test("--retokenize purges the memo for exactly the invalidated blobs, before touching the map") { + withDb { db => + seed(db, id("rs" -> rustV1)) + var purged: Vector[String] = Vector.empty + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), purge = { blobs => + purged = blobs + TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L) + }))) + try { + purged.sorted shouldEqual Vector("rsblob1", "rsblob2") + // The memo key has no tokenizer in it (tokenBySha.pl:76), so it would + // serve the same stale tokens. + purged should not contain "cblob1" + } finally m.close() + } + } + + test("a memo purge that fails aborts before the map is changed") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[RuntimeException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), + purge = _ => throw new RuntimeException("memo is read-only")))) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("a memo that held none of the affected blobs refuses: the --memo-dir is wrong") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val ex = intercept[Mapping.NothingInvalidatedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), + purge = blobs => TokenizerMemo.PurgeReport(blobs.size.toLong, 0L, blobs.size.toLong, 0L)))) + } + ex.getMessage should include("--memo-dir") + ex.getMessage should include("$BFG_MEMO_DIR") + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + finally m.close() + } + } + + test("an empty memo is accepted when --memo-dir is the memo the tokenizer reads") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val reports = Vector.newBuilder[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), purge = emptyMemo, report = reports += _, + memoDirIsTokenizerMemo = true))) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe None + m.getBlob("rsblob2", "src/two.rs") shouldBe None + m.getBlob("cblob1", "src/one.c") shouldBe Some("newc1") + m.storedTokenizerId("rs") shouldBe Some(rustV2) + val log = reports.result().mkString("\n") + log should include("the memo held none of the 2 affected blob(s)") + log should include("Accepted, because --memo-dir is $BFG_MEMO_DIR") + log should include("DROPPED 2 tokenized blob_map rows") + } finally m.close() + } + } + + test("a memo that held entries is purged without the empty-memo notice") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val reports = Vector.newBuilder[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs"), report = reports += _, memoDirIsTokenizerMemo = true))) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe None + reports.result().mkString("\n") should not include "Accepted, because" + } finally m.close() + } + } + + test("--retokenize on an extension with no tokenized rows refuses, it does not exit quietly") { + withDb { db => + seed(db, id("rs" -> rustV1, "java" -> "cccccccccccccccc")) + val ex = intercept[Mapping.NothingInvalidatedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV1, "java" -> "dddddddddddddddd"), + retokenize = Some(retokenize(Set("java")))) + } + ex.getMessage should include("java") + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try { + m.rowCounts.blobTokenized shouldEqual 4L + m.getTree("tree1") shouldBe Some("newtree1") + } finally m.close() + } + } + + test("--retokenize that does not cover every changed tokenizer refuses first, changing nothing") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + val ex = intercept[Mapping.TokenizerChangedException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV2), + retokenize = Some(retokenize(Set("rs")))) + } + ex.getMessage should include("c") + // Invalidating .rs and then refusing would leave a half-invalidated map. + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1, "c" -> cV1)) + try { + m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + m.storedTokenizerId("rs") shouldBe Some(rustV1) + } finally m.close() + } + } + + test("naming every changed tokenizer at once is accepted") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1, "h" -> cV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV2, "h" -> cV2), + retokenize = Some(retokenize(Set("rs", "c", "h")))) + try { + m.rowCounts.blobTokenized shouldEqual 0L + m.storedTokenizerId("rs") shouldBe Some(rustV2) + m.storedTokenizerId("c") shouldBe Some(cV2) + } finally m.close() + } + } + + test("--retokenize refuses when a retained new_blob does not resolve in dst") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + // blob_map's new_blob ids live in dst and nowhere else. Reusing the .c rows + // beside a dst that was wiped leaves every retained reference dangling. + intercept[Mapping.DanglingNewBlobException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), resolves = _ => false))) + } + val m = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m.getBlob("rsblob1", "src/one.rs") shouldBe Some("newrs1") + finally m.close() + } + } + + test("the reachability probe asks about the rows that are KEPT, never the ones dropped") { + withDb { db => + seed(db, id("rs" -> rustV1, "c" -> cV1)) + var asked = Vector.empty[String] + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2, "c" -> cV1), + retokenize = Some(retokenize(Set("rs"), resolves = { b => asked :+= b; true }))) + try { + asked should contain allOf ("newc1", "newc2") + // Probing a dropped row would refuse a correct invalidation whenever the + // old token blobs had already been garbage-collected out of dst. + asked should not contain "newrs1" + } finally m.close() + } + } + + test("extensions are matched case-insensitively, as the mask is") { + withDb { db => + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("rs" -> rustV1)) + try m0.putBlob("upper", "src/Shouty.RS", "newupper") finally m0.close() + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try m.getBlob("upper", "src/Shouty.RS") shouldBe None + finally m.close() + } + } + + test("a longer extension is not matched by a shorter one's suffix") { + withDb { db => + val m0 = Mapping.open(db, cmd, mask, tokenizerIdentity = id("c" -> cV1)) + try { + m0.putBlob("ccblob", "src/x.cc", "newcc") + m0.putBlob("cppblob", "src/x.cpp", "newcpp") + m0.putBlob("cblob", "src/x.c", "newc") + } finally m0.close() + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("c" -> cV2), + retokenize = Some(retokenize(Set("c")))) + try { + m.getBlob("cblob", "src/x.c") shouldBe None + // `.cc` and `.cpp` end in neither `.c`; a naive LIKE '%c' would take both. + m.getBlob("ccblob", "src/x.cc") shouldBe Some("newcc") + m.getBlob("cppblob", "src/x.cpp") shouldBe Some("newcpp") + } finally m.close() + } + } + + test("the invalidation is recorded in meta, so a reused map says where it came from") { + withDb { db => + seed(db, id("rs" -> rustV1)) + val m = Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + try { + m.getMeta(Mapping.RetokenizedAtKey) shouldBe Some("1700000000") + m.getMeta(Mapping.RetokenizedExtensionsKey) shouldBe Some("rs") + m.getMeta(Mapping.RetokenizedFromKey) shouldBe Some(s"rs=$rustV1") + } finally m.close() + } + } + + test("--mask-widened and --retokenize together are refused, not composed") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, """(?i)\.(c|h|rs|java)$""", + maskWidening = Some(Mapping.MaskWidening(newBlobResolves = _ => true, report = _ => ())), + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs")))) + } + } + } + + test("--retokenize without an identity to record is refused") { + withDb { db => + seed(db, TokenizerIdentity.empty) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, mask, retokenize = Some(retokenize(Set("rs")))) + } + } + } + + test("--retokenize naming an extension the identity does not cover is refused") { + withDb { db => + seed(db, id("rs" -> rustV1)) + intercept[IllegalArgumentException] { + Mapping.open(db, cmd, mask, + tokenizerIdentity = id("rs" -> rustV2), + retokenize = Some(retokenize(Set("rs", "java")))) + } + } + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala b/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala deleted file mode 100644 index a32e445c5..000000000 --- a/blobExec/src/test/scala/cregit/blobexec/TimeoutRetrySpec.scala +++ /dev/null @@ -1,179 +0,0 @@ -package cregit.blobexec - -import org.eclipse.jgit.api.Git -import org.eclipse.jgit.internal.storage.file.FileRepository -import org.eclipse.jgit.lib.{ObjectId, Repository} -import org.eclipse.jgit.revwalk.RevWalk -import org.eclipse.jgit.storage.file.FileRepositoryBuilder -import org.eclipse.jgit.treewalk.TreeWalk -import org.scalatest.BeforeAndAfterAll -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers - -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Path} - -/** - * A blob whose tokenizer was killed on its budget must be *retryable*: the run - * reports it and refuses to publish, but simply running again retries the blob - * with no surgery on the mapping database. - * - * Three writes would each hide such a blob from the next run: its `blob_map` row, - * any `tree_map` row above it, and its commit's `commit_map` row. All three are - * asserted absent, and the commit's successful tokenizations asserted present. - */ -class TimeoutRetrySpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { - - private val workRoot: Path = Files.createTempDirectory("timeout-retry-") - - override def afterAll(): Unit = deleteRecursive(workRoot) - - /** The tokenizer under test: it hangs on `b.c` only while `hangMarker` exists, - * so both runs can use the identical command path (the memo's `meta` table - * refuses a command change, exactly as in production). */ - private def tokenizer(dir: Path, hangMarker: Path): String = { - val f = dir.resolve("tok.sh") - Files.writeString( - f, - s"""|#!/bin/sh - |if [ "$$BFG_FILENAME" = "b.c" ] && [ -e "${hangMarker.toAbsolutePath}" ]; then - | sleep 300 & sleep 300 - |else - | tr a-z A-Z - |fi - |""".stripMargin - ) - f.toFile.setExecutable(true) - f.toAbsolutePath.toString - } - - private def run( - src: Repository, - dstPath: Path, - dbPath: Path, - command: String, - mode: String - ): WalkStats = { - val dstExisted = Files.isDirectory(dstPath) - val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] - if (!dstExisted) dst.create(true) - val mapping = Mapping.open(dbPath, command, """\.c$""") - try - new Walker( - src, dst, mapping, """\.c$""".r, command, - abortOnError = false, - parallelism = 2, - pipeline = mode == "pipeline", - pipelineTrees = mode == "pipeline-trees", - destinationMayContainObjects = dstExisted, - blobTimeoutSeconds = 1 - ).run() - finally { - mapping.close() - dst.close() - } - } - - private def withMapping[A](dbPath: Path, command: String)(body: Mapping => A): A = { - val m = Mapping.open(dbPath, command, """\.c$""") - try body(m) finally m.close() - } - - // Each walker persists a commit its own way, so each must suppress all three. - for (mode <- Seq("serial", "pipeline", "pipeline-trees")) - test(s"[$mode] a timed-out blob is retried by simply running again, and good work is kept") { - val dir = Files.createTempDirectory(workRoot, "fixture-") - val srcDir = dir.resolve("src") - Files.createDirectories(srcDir) - val git = Git.init().setDirectory(srcDir.toFile).setBare(false).call() - Files.writeString(srcDir.resolve("a.c"), "alpha\n") - Files.createDirectories(srcDir.resolve("deep")) - Files.writeString(srcDir.resolve("deep/b.c"), "beta\n") - git.add().addFilepattern("a.c").call() - git.add().addFilepattern("deep/b.c").call() - val commit = git.commit() - .setMessage("two blobs, one of which will wedge") - .setAuthor("Tester", "t@example.org") - .setCommitter("Tester", "t@example.org") - .call() - - val src = git.getRepository - val commitSha = commit.getId.name - val topTreeSha = commit.getTree.getId.name - val aSha = blobIdAt(src, commit.getId, "a.c").name - val bSha = blobIdAt(src, commit.getId, "deep/b.c").name - val subTreeSha = blobIdAt(src, commit.getId, "deep").name - - val hangMarker = dir.resolve("HANG") - Files.writeString(hangMarker, "hang b.c\n") - val command = tokenizer(dir, hangMarker) - val dstPath = dir.resolve("dst.git") - val dbPath = dir.resolve("blobmap.db") - - // -- run 1: b.c wedges ----------------------------------------------------- - val first = run(src, dstPath, dbPath, command, mode) - - first.blobsTimedOut shouldEqual 1 - first.aborted shouldBe false // a timeout is not an abort - first.commitsProcessed shouldEqual 0 // the containing commit was not folded - - withMapping(dbPath, command) { m => - // The three writes that would each hide the retry, all absent: - m.getBlob(bSha, "deep/b.c") shouldEqual None - m.getTree(subTreeSha) shouldEqual None - m.getTree(topTreeSha) shouldEqual None - m.getCommit(commitSha) shouldEqual None - // ...and the work that was genuinely correct, kept: - m.getBlob(aSha, "a.c") shouldBe defined - m.getBlob(aSha, "a.c").get should not equal aSha // tokenized, not raw - } - - // -- run 2: same command, nothing cleared by hand -------------------------- - Files.delete(hangMarker) - val second = run(src, dstPath, dbPath, command, mode) - - second.blobsTimedOut shouldEqual 0 - second.aborted shouldBe false - second.commitsProcessed shouldEqual 1 - // a.c came from the memo rather than being re-tokenized: the retry costs one - // blob, not the whole commit. - second.blobsCacheHit should be >= 1 - - withMapping(dbPath, command) { m => - m.getCommit(commitSha) shouldBe defined - m.getTree(topTreeSha) shouldBe defined - val bNew = m.getBlob(bSha, "deep/b.c") - bNew shouldBe defined - bNew.get should not equal bSha // the retry tokenized it - readBlob(dstPath, bNew.get) shouldEqual "BETA\n" - } - } - - // -- helpers --------------------------------------------------------------- - - private def blobIdAt(repo: Repository, commitId: ObjectId, path: String): ObjectId = { - val rw = new RevWalk(repo) - try { - val tree = rw.parseCommit(commitId).getTree - val tw = TreeWalk.forPath(repo, path, tree) - try tw.getObjectId(0) finally tw.close() - } finally rw.close() - } - - private def readBlob(dstPath: Path, sha: String): String = { - val dst = FileRepositoryBuilder.create(dstPath.toFile).asInstanceOf[FileRepository] - try { - val r = dst.newObjectReader() - try new String(r.open(ObjectId.fromString(sha)).getBytes, UTF_8) finally r.close() - } finally dst.close() - } - - private def deleteRecursive(p: Path): Unit = { - if (Files.isDirectory(p)) { - val s = Files.list(p) - try s.forEach(deleteRecursive) finally s.close() - } - Files.deleteIfExists(p) - () - } -} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala new file mode 100644 index 000000000..09a03dc77 --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerIdentitySpec.scala @@ -0,0 +1,64 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class TokenizerIdentitySpec extends AnyFunSuite with Matchers { + + test("one extension round-trips") { + TokenizerIdentity.parse("rs=deadbeefcafe") shouldBe + Right(TokenizerIdentity(Map("rs" -> "deadbeefcafe"))) + } + + test("several extensions round-trip, and render is sorted and stable") { + val id = TokenizerIdentity.parse("h=aaaaaaaa,c=bbbbbbbb,rs=cccccccc").toOption.get + id.extensions shouldBe Set("c", "h", "rs") + id.render shouldEqual "c=bbbbbbbb,h=aaaaaaaa,rs=cccccccc" + TokenizerIdentity.parse("c=bbbbbbbb,rs=cccccccc,h=aaaaaaaa").toOption.get.render shouldEqual id.render + } + + test("c++ and h++ are extensions, because CregitLanguages.pm says so") { + val id = TokenizerIdentity.parse("c++=aaaaaaaa,h++=bbbbbbbb").toOption.get + id.extensions shouldBe Set("c++", "h++") + } + + test("an empty spec is refused, not read as 'no extensions'") { + TokenizerIdentity.parse("").isLeft shouldBe true + TokenizerIdentity.parse(" ").isLeft shouldBe true + } + + test("a leading dot is refused: the tables spell extensions without one") { + TokenizerIdentity.parse(".rs=deadbeefcafe").isLeft shouldBe true + } + + test("an uppercase extension is refused rather than silently lowercased") { + TokenizerIdentity.parse("RS=deadbeefcafe").isLeft shouldBe true + } + + test("a value that is not a long-enough hex digest is refused") { + TokenizerIdentity.parse("rs=zzzz").isLeft shouldBe true + TokenizerIdentity.parse("rs=ab").isLeft shouldBe true + TokenizerIdentity.parse("rs=").isLeft shouldBe true + TokenizerIdentity.parse("rs").isLeft shouldBe true + } + + test("a quote or a LIKE wildcard cannot get into an extension") { + // Mapping interpolates extensions into a SQL LIKE. + Seq("r's", "r%s", "r_s", "rs;", "rs)", "r s").foreach { bad => + TokenizerIdentity.parse(s"$bad=deadbeefcafe").isLeft shouldBe true + TokenizerIdentity.parseExtensions(bad).isLeft shouldBe true + } + } + + test("an extension named twice is refused: one of the two values would win silently") { + TokenizerIdentity.parse("rs=aaaaaaaa,rs=bbbbbbbb").isLeft shouldBe true + } + + test("extension lists for --retokenize parse and reject the same way") { + TokenizerIdentity.parseExtensions("rs") shouldBe Right(Set("rs")) + TokenizerIdentity.parseExtensions("c,h,cpp") shouldBe Right(Set("c", "h", "cpp")) + TokenizerIdentity.parseExtensions("") .isLeft shouldBe true + TokenizerIdentity.parseExtensions(".rs").isLeft shouldBe true + TokenizerIdentity.parseExtensions("c,,h").isLeft shouldBe true + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala new file mode 100644 index 000000000..cbc45a54d --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerMemoSpec.scala @@ -0,0 +1,143 @@ +package cregit.blobexec + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +/** The memo layout must match tokenizeByBlobId/tokenBySha.pl byte for byte. */ +class TokenizerMemoSpec extends AnyFunSuite with Matchers { + + private def withTempDir[A](body: Path => A): A = { + val d = Files.createTempDirectory("memo-spec-") + try body(d) + finally { + def rm(p: Path): Unit = { + if (Files.isDirectory(p)) { val s = Files.list(p); try s.forEach(rm) finally s.close() } + Files.deleteIfExists(p) + () + } + rm(d) + } + } + + test("sha1Hex matches a known Digest::SHA value") { + // printf 'int a;\n' | sha1sum + TokenizerMemo.sha1Hex("int a;\n".getBytes(UTF_8)) shouldEqual + "56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + // printf 'fn main() {}\n' | sha1sum + TokenizerMemo.sha1Hex("fn main() {}\n".getBytes(UTF_8)) shouldEqual + "c135f41eadc2f7248d34e03e64a04e8420bf5e4b" + } + + test("the path is /xx/yy/, the fan-out tokenBySha.pl builds") { + val sha = "56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + TokenizerMemo.relativePathFor(sha) shouldEqual + "56/f5/56f54d1636dfec63c3e1586e5e4bdc9a455bb9f6" + // Both halves from the front (substr 0,2 then 2,2), not 0,2 then -2. + TokenizerMemo.relativePathFor(sha) should startWith("56/f5/") + } + + test("sha1Hex agrees with perl's Digest::SHA on bytes perl might mangle") { + // If either side hashed text rather than bytes, CRLF and non-ASCII keys would diverge. + val perl = Option(System.getenv("PATH")).toVector + .flatMap(_.split(java.io.File.pathSeparator)) + .map(p => Path.of(p, "perl")) + .find(Files.isExecutable) + assume(perl.isDefined, "perl not on PATH") + + val samples = Vector( + "int a;\n", + "a\r\nb\r\n", + "no trailing newline", + "caf\u00e9 \u00fcber\n", + "" + ) + withTempDir { d => + samples.foreach { s => + val f = d.resolve("input.bin") + Files.write(f, s.getBytes(UTF_8)) + val pb = new ProcessBuilder( + perl.get.toString, "-MDigest::SHA=sha1_hex", "-e", + "local $/; open(F, '<', $ARGV[0]) or die; binmode F; print sha1_hex( // '')", + f.toString) + pb.redirectErrorStream(true) + val proc = pb.start() + val out = new String(proc.getInputStream.readAllBytes(), UTF_8).trim + proc.waitFor() + withClue(s"sample [${s.replace("\n", "\\n").replace("\r", "\\r")}] perl said [$out]: ") { + out shouldEqual TokenizerMemo.sha1Hex(s.getBytes(UTF_8)) + } + } + } + } + + test("purge deletes the entries of the named blobs and leaves the rest alone") { + withTempDir { root => + val contents = Map( + "blob-rs-1" -> "fn one() {}\n", + "blob-rs-2" -> "fn two() {}\n", + "blob-c-1" -> "int c;\n" + ) + contents.values.foreach { s => + val p = TokenizerMemo.entryFor(root, TokenizerMemo.sha1Hex(s.getBytes(UTF_8))) + Files.createDirectories(p.getParent) + Files.writeString(p, "TOKENS FOR " + s) + } + + val report = TokenizerMemo.purge( + root, + Vector("blob-rs-1", "blob-rs-2"), + sha => contents.get(sha).map(_.getBytes(UTF_8))) + + report.examined shouldEqual 2L + report.deleted shouldEqual 2L + report.missing shouldEqual 0L + report.unreadable shouldEqual 0L + + Files.exists(TokenizerMemo.entryFor(root, + TokenizerMemo.sha1Hex(contents("blob-rs-1").getBytes(UTF_8)))) shouldBe false + Files.exists(TokenizerMemo.entryFor(root, + TokenizerMemo.sha1Hex(contents("blob-c-1").getBytes(UTF_8)))) shouldBe true + } + } + + test("a blob with no memo entry is counted missing, not deleted") { + withTempDir { root => + val report = TokenizerMemo.purge(root, Vector("absent"), _ => Some("x\n".getBytes(UTF_8))) + report.deleted shouldEqual 0L + report.missing shouldEqual 1L + } + } + + test("a blob whose bytes cannot be read is counted, not skipped in silence") { + withTempDir { root => + val report = TokenizerMemo.purge(root, Vector("oversized"), _ => None) + report.examined shouldEqual 1L + report.unreadable shouldEqual 1L + report.deleted shouldEqual 0L + } + } + + test("isTokenizerMemo holds for the same directory, by a symlink or a trailing slash too") { + withTempDir { root => + val memo = Files.createDirectory(root.resolve("memo")) + val link = Files.createSymbolicLink(root.resolve("link"), memo) + TokenizerMemo.isTokenizerMemo(memo, Some(memo.toString)) shouldBe true + TokenizerMemo.isTokenizerMemo(memo, Some(memo.toString + "/")) shouldBe true + TokenizerMemo.isTokenizerMemo(link, Some(memo.toString)) shouldBe true + } + } + + test("isTokenizerMemo is false for another directory, an unset or empty variable, or a missing path") { + withTempDir { root => + val memo = Files.createDirectory(root.resolve("memo")) + val other = Files.createDirectory(root.resolve("other")) + TokenizerMemo.isTokenizerMemo(memo, Some(other.toString)) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, None) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, Some("")) shouldBe false + TokenizerMemo.isTokenizerMemo(memo, Some(root.resolve("gone").toString)) shouldBe false + } + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala new file mode 100644 index 000000000..e9cfd78bf --- /dev/null +++ b/blobExec/src/test/scala/cregit/blobexec/TokenizerWorkerPoolSpec.scala @@ -0,0 +1,140 @@ +package cregit.blobexec + +import org.eclipse.jgit.internal.storage.dfs.{DfsRepositoryDescription, InMemoryRepository} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Paths} +import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class TokenizerWorkerPoolSpec extends AnyFunSuite with Matchers with BeforeAndAfterAll { + + private val tmpDir = Files.createTempDirectory("tokenizer-worker-pool-") + private val worker = tmpDir.resolve("fake-worker.pl") + private val grandchildPidFile = Paths.get(s"$worker.grandchild") + + override def beforeAll(): Unit = { + Files.writeString( + worker, + """|#!/usr/bin/env perl + |use strict; + |use warnings; + |$| = 1; + |sub reply { my ($exit, $out) = @_; print "RES $exit ", length($out), " 0\n", $out } + |print "READY\n"; + |while (my $header = ) { + | my (undef, $bodyLength, $nameLength, $timeout) = split ' ', $header; + | read(STDIN, my $name, $nameLength); + | read(STDIN, my $body, $bodyLength) if $bodyLength; + | if ($name eq 'hang.c') { + | my $child = fork() // exit 3; + | exec 'sleep', '300' if $child == 0; + | open(my $pids, '>', "$0.grandchild") or exit 3; + | print {$pids} $child; + | close($pids); + | sleep 300; + | } + | exit 7 if $name eq 'die.c'; + | sleep 2 if $name eq 'slow.c'; + | if ($name eq 'crash.c') { reply(33, '') } + | elsif ($name eq 'timeout.c') { reply(124, '') } + | elsif ($name eq 'budget.c') { reply(0, $timeout) } + | else { reply(0, uc $body) } + |} + |""".stripMargin + ) + worker.toFile.setExecutable(true) + } + + override def afterAll(): Unit = + Files.walk(tmpDir).sorted(java.util.Comparator.reverseOrder()).forEach(p => Files.delete(p)) + + private def withPool[A](size: Int)(body: TokenizerWorkerPool => A): A = { + val pool = new TokenizerWorkerPool(worker.toString, size) + try body(pool) finally pool.close() + } + + private def stdout(pool: TokenizerWorkerPool, filename: String, body: String, timeoutSeconds: Int = 2) = { + pool.invoke(body.getBytes(UTF_8), filename, timeoutSeconds) match { + case ChildRunner.Outcome.Exited(0, out, _) => new String(out, UTF_8) + case other => fail(s"expected exit 0, got $other") + } + } + + test("returns the worker's output and sends each request's own budget") { + withPool(1) { pool => + stdout(pool, "hello.c", "hello") shouldEqual "HELLO" + stdout(pool, "budget.c", "x", timeoutSeconds = 7) shouldEqual "7" + stdout(pool, "budget.c", "x", timeoutSeconds = 21) shouldEqual "21" + } + } + + test("a worker's timeout and parser crash take BlobExec's timeout and crash paths") { + val inserter = new InMemoryRepository(new DfsRepositoryDescription("pool-spec")).newObjectInserter() + withPool(1) { pool => + def run(filename: String): (BlobExec.Outcome, String) = { + var called = "none" + val outcome = BlobExec.run( + "body".getBytes(UTF_8), "a" * 40, filename, s"src/$filename", "tokenBySha.pl", + abortOnError = true, inserter = inserter, + onTimeout = () => called = "timeout", onParserCrash = () => called = "parser-crash", + workerPool = Some(pool)) + (outcome, called) + } + run("timeout.c") shouldEqual ((BlobExec.Outcome.Skip, "timeout")) + run("crash.c") shouldEqual ((BlobExec.Outcome.Skip, "parser-crash")) + run("die.c") shouldEqual ((BlobExec.Outcome.Skip, "timeout")) + run("ok.c")._1 shouldBe a[BlobExec.Outcome.Replace] + } + } + + test("kills a wedged worker with its children, replaces it, and serves the next request") { + withPool(1) { pool => + val started = System.nanoTime() + pool.invoke("x".getBytes(UTF_8), "hang.c", 1) shouldBe a[ChildRunner.Outcome.Killed] + (System.nanoTime() - started).nanos should be < 15.seconds + eventuallyStopped(Seq(Files.readString(grandchildPidFile).trim.toLong)) shouldBe true + stdout(pool, "next.c", "next") shouldEqual "NEXT" + } + } + + test("a worker that dies in a request fails that blob only") { + withPool(1) { pool => + pool.invoke("x".getBytes(UTF_8), "die.c", 2) should not matchPattern { + case ChildRunner.Outcome.Exited(0, _, _) => + } + stdout(pool, "ok.c", "ok") shouldEqual "OK" + } + } + + test("serves pool-sized requests in parallel") { + implicit val ec: ExecutionContext = ExecutionContext.global + withPool(3) { pool => + val started = System.nanoTime() + val requests = (1 to 3).map(n => Future(stdout(pool, "slow.c", s"body$n", timeoutSeconds = 5))) + Await.result(Future.sequence(requests), 10.seconds).toSet shouldEqual Set("BODY1", "BODY2", "BODY3") + (System.nanoTime() - started).nanos should be < 4.seconds + } + } + + test("close stops every worker") { + val pool = new TokenizerWorkerPool(worker.toString, 2) + val pids = ProcessHandle.current().children().iterator().asScala + .filter(_.info().commandLine().orElse("").contains(worker.toString)) + .map(_.pid()).toVector + pids should have size 2 + pool.close() + eventuallyStopped(pids) shouldBe true + } + + private def eventuallyStopped(pids: Seq[Long]): Boolean = { + def alive(pid: Long) = ProcessHandle.of(pid).map[Boolean](_.isAlive).orElse(false) + val deadline = System.nanoTime() + 5.seconds.toNanos + while (System.nanoTime() < deadline && pids.exists(alive)) Thread.sleep(25) + !pids.exists(alive) + } +} diff --git a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala index 91205d835..7b1ab37fe 100644 --- a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala @@ -85,11 +85,38 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // Empty by default: a fixture blob must never be excluded because it happens // to collide with the shipped list, and the shipped list must never be // exercised by accident. The denylist tests pass their own. - denylist: BlobDenylist = BlobDenylist.empty + denylist: BlobDenylist = BlobDenylist.empty, + maskWidened: Boolean = false, + tokenizerIdentity: TokenizerIdentity = TokenizerIdentity.empty, + retokenizeExtensions: Set[String] = Set.empty, + memoDir: Option[Path] = None ): WalkStats = { val destinationMayContainObjects = Files.isDirectory(dstPath) val dst = openBare(dstPath) - val mapping = Mapping.open(dbPath, command, mask) + val resolvesInDst: String => Boolean = id => { + val r = dst.newObjectReader() + try r.has(ObjectId.fromString(id)) finally r.close() + } + val widening = + if (!maskWidened) None + else Some(Mapping.MaskWidening(newBlobResolves = resolvesInDst, report = _ => ())) + val retokenize = + if (retokenizeExtensions.isEmpty) None + else Some(Mapping.Retokenize( + extensions = retokenizeExtensions, + newBlobResolves = resolvesInDst, + purgeMemo = blobs => memoDir match { + case None => TokenizerMemo.PurgeReport(blobs.size.toLong, blobs.size.toLong, 0L, 0L) + case Some(root) => TokenizerMemo.purge(root, blobs, sha => { + val r = srcRepo.newObjectReader() + try Some(r.open(ObjectId.fromString(sha), OBJ_BLOB).getBytes) + catch { case _: Exception => None } + finally r.close() + }) + }, + report = _ => () + )) + val mapping = Mapping.open(dbPath, command, mask, None, widening, tokenizerIdentity, retokenize) try { val walker = new Walker( srcRepo, dst, mapping, mask.r, command, abortOnError, parallelism, @@ -640,10 +667,10 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte // -- shared dir helpers ---------------------------------------------------- + // -- large pass-through blob (regression) ---------------------------------- - // - // Lowering the threshold lets a small blob cross it, so the regression - // reproduces in milliseconds instead of needing a 50 MB fixture. + // getBytes refuses blobs above JGit's stream threshold, so the verbatim copy + // must stream. The threshold is lowered so a small blob reproduces it. private def withStreamFileThreshold[T](bytes: Int)(body: => T): T = { val lowered = new WindowCacheConfig lowered.setStreamFileThreshold(bytes) @@ -703,7 +730,20 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- oversized MASK-MATCHED blob (regression) ------------------------------- + // Streaming cannot help here: the tokenizer cannot take it, so it is excluded. + + test("the oversized limit is jgit's own threshold, not a constant above it") { + val jgitDefault = new WindowCacheConfig().getStreamFileThreshold.toLong + // Read from jgit, so the two cannot drift. + Walker.MaxBlobBytes shouldEqual jgitDefault + jgitDefault shouldEqual 50L * 1024 * 1024 + Walker.isOversized(200L * 1024 * 1024) shouldBe true + Walker.isOversized(55L * 1024 * 1024) shouldBe true + Walker.isOversized(4096L) shouldBe false + } + /** The 78 KB fixture is below Walker.MaxBlobBytes, so only the LargeObjectException + * catch (under a lowered jgit threshold) can exclude it. */ private def checkOversizedExclusion(label: String, pipeline: Boolean, pipelineTrees: Boolean): Unit = { val dir = freshWorkDir("oversized-masked-" + label) val git = initSrc(dir.resolve("src")) @@ -715,6 +755,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte "add a normal source file and a generated one") } finally git.close() + Walker.isOversized(generated.length.toLong) shouldBe false + val srcRepo = openBare(dir.resolve("src/.git")) val stats = try { withStreamFileThreshold(4 * 1024) { @@ -762,17 +804,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- DENYLISTED mask-matched blob ------------------------------------------- - // - // srcML 1.1.0 does not terminate on the denylisted blobs: 0 bytes of output, one - // core busy, at any budget. Upstream srcML/srcML#2361 is open with no patch and - // v1.1.0 is the latest release, so there is nothing to upgrade to. The timeout - // path contains the hang but answers it with exit 4, "incomplete, do not - // publish", which holds the project back for a diagnosed third-party defect. - // - // So a denylisted blob takes Task 5's EXCLUSION path instead: dropped in - // microseconds, never handed to the tokenizer, counted on its own, and NOT - // gating the exit status. A timeout still gates it, because a timeout is a hang - // nobody has explained yet. + // srcML 1.1.0 never terminates on these (srcML/srcML#2361). A denylisted blob is + // excluded and does not gate the exit status; an unexplained timeout still does. /** The git blob id of `content`, computed the way git computes it, so a fixture * can be denylisted without first asking the repository what its sha is. */ @@ -793,8 +826,8 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } finally git.close() val denied = blobIdOf(hangs) - val denylist = BlobDenylist(Map(denied.name -> - BlobDenylist.Entry("srcML/srcML#2361", "srcML 1.1.0 does not terminate on it"))) + val denylist = BlobDenylist(Map( + denied.name -> BlobDenylist.Entry("srcML/srcML#2361", "srcML 1.1.0 does not terminate on it"))) // Every invocation of the tokenizer records the blob it was given. "Excluded in // milliseconds, never handed to the tokenizer" is only checkable from the @@ -882,6 +915,183 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte finally dst.close() } + // -- --mask-widened, end to end --------------------------------------------- + // Trap: a newly selected file already has an identity row; serving it as a cache + // hit would put raw source in the tokenized repository. + + test("a widened mask reuses the tokenized rows and still tokenizes the newly selected ones") { + val w = freshWorkDir("mask-widened") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val cmd = shellScript(w, "tr a-z A-Z") + val narrow = """\.[ch]$""" + val wide = """\.(c|cpp|h)$""" + + val git = initSrc(srcDir) + writeAndCommit(git, Map( + "a.c" -> "int a;\n", + "b.h" -> "int b;\n", + "c.cpp" -> "int c;\n", + "README.md" -> "leave-me-alone\n" + ), "only commit") + + val first = runWalker(git.getRepository, dstDir, db, cmd, narrow) + first.blobCommandExecutions shouldEqual 2L + locally { + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + fileAtHead(dst, "master", "c.cpp") shouldBe Some("int c;\n") + } finally dst.close() + } + + val second = runWalker(git.getRepository, dstDir, db, cmd, wide, maskWidened = true) + + second.blobCommandExecutions shouldEqual 1L + second.blobsCacheHit shouldEqual 2 + + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + fileAtHead(dst, "master", "b.h") shouldBe Some("INT B;\n") + // Without the identity-row purge this would be the raw "int c;\n". + fileAtHead(dst, "master", "c.cpp") shouldBe Some("INT C;\n") + fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") + } finally dst.close() + } + + test("without the opt-in the same second run refuses, and dst is left alone") { + val w = freshWorkDir("mask-widened-refused") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val cmd = shellScript(w, "tr a-z A-Z") + + val git = initSrc(srcDir) + writeAndCommit(git, Map("a.c" -> "int a;\n", "c.cpp" -> "int c;\n"), "only commit") + runWalker(git.getRepository, dstDir, db, cmd, """\.[ch]$""") + + intercept[Mapping.MetaMismatchException] { + runWalker(git.getRepository, dstDir, db, cmd, """\.(c|cpp|h)$""") + } + + val dst = openBare(dstDir) + try fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") finally dst.close() + } + + // -- tokenizer identity ---------------------------------------------------- + // `command` is a constant path, so a corrected tokenizer must be detected by its + // identity, and --retokenize must redo only the affected extension. + + test("a changed tokenizer identity refuses the incremental run instead of reusing its tokens") { + val w = freshWorkDir("tokid-refuse") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val mask = """\.(c|rs)$""" + val v1 = shellScript(w, "tr a-z A-Z") + + val git = initSrc(srcDir) + writeAndCommit(git, Map("a.c" -> "int a;\n", "b.rs" -> "fn b() {}\n"), "only commit") + runWalker(git.getRepository, dstDir, db, v1, mask, + tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "1111111111"))) + + // Only the Rust tokenizer's identity moved. + val ex = intercept[Mapping.TokenizerChangedException] { + runWalker(git.getRepository, dstDir, db, v1, mask, + tokenizerIdentity = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "2222222222"))) + } + ex.getMessage should include("--retokenize=rs") + + val dst = openBare(dstDir) + try fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + finally dst.close() + } + + test("--retokenize on one extension re-tokenizes that extension's files and reuses the rest") { + val w = freshWorkDir("tokid-retokenize") + val srcDir = w.resolve("src") + val dstDir = w.resolve("dst.git") + val db = w.resolve("map.sqlite") + val memo = w.resolve("memo") + Files.createDirectories(memo) + val mask = """\.(c|rs)$""" + + // One command path for both runs, as in the pipeline (tokenBySha.pl): a second + // path would trip the `command` meta check and prove nothing. + val cmd = w.resolve("tokenize.sh") + def writeTokenizer(rustBody: String): String = { + Files.writeString(cmd, + s"""#!/bin/sh + |case "$$BFG_FILENAME" in + | *.rs) $rustBody ;; + | *) tr a-z A-Z ;; + |esac + |""".stripMargin) + cmd.toFile.setExecutable(true) + cmd.toAbsolutePath.toString + } + // v1 emits the buggy `line:col` prefix; v2 does not. + val v1 = writeTokenizer("""sed 's/^/1:1\t/'""") + + val git = initSrc(srcDir) + writeAndCommit(git, Map( + "a.c" -> "int a;\n", + "b.rs" -> "fn b() {}\n", + "c.rs" -> "fn c() {}\n", + "README.md" -> "leave-me-alone\n" + ), "only commit") + + val idV1 = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "1111111111")) + val idV2 = TokenizerIdentity(Map("c" -> "cccccccccccc", "rs" -> "2222222222")) + + val first = runWalker(git.getRepository, dstDir, db, v1, mask, tokenizerIdentity = idV1) + first.blobCommandExecutions shouldEqual 3L + locally { + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "b.rs") shouldBe Some("1:1\tfn b() {}\n") + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + } finally dst.close() + } + + // The entries tokenBySha.pl would have written. + val memoEntries = Map( + "fn b() {}\n" -> "1:1\tfn b() {}\n", + "fn c() {}\n" -> "1:1\tfn c() {}\n", + "int a;\n" -> "INT A;\n" + ).map { case (content, tokens) => + val p = TokenizerMemo.entryFor(memo, TokenizerMemo.sha1Hex(content.getBytes(UTF_8))) + Files.createDirectories(p.getParent) + Files.writeString(p, tokens) + content -> p + } + + val v2 = writeTokenizer("cat") + v2 shouldEqual v1 + + val second = runWalker(git.getRepository, dstDir, db, v2, mask, + tokenizerIdentity = idV2, retokenizeExtensions = Set("rs"), memoDir = Some(memo)) + + // Only the .rs files are re-tokenized; a.c stays a cache hit. + second.blobCommandExecutions shouldEqual 2L + second.blobsCacheHit shouldEqual 1 + + val dst = openBare(dstDir) + try { + fileAtHead(dst, "master", "b.rs") shouldBe Some("fn b() {}\n") + fileAtHead(dst, "master", "c.rs") shouldBe Some("fn c() {}\n") + fileAtHead(dst, "master", "a.c") shouldBe Some("INT A;\n") + fileAtHead(dst, "master", "README.md") shouldBe Some("leave-me-alone\n") + } finally dst.close() + + // An unpurged memo would serve the same stale tokens by sha1(contents). + Files.exists(memoEntries("fn b() {}\n")) shouldBe false + Files.exists(memoEntries("fn c() {}\n")) shouldBe false + Files.exists(memoEntries("int a;\n")) shouldBe true + } + private def deleteRecursive(p: Path): Unit = { if (Files.isDirectory(p)) { val s = Files.list(p) diff --git a/devenv.nix b/devenv.nix index 1c48d0229..f9608171f 100644 --- a/devenv.nix +++ b/devenv.nix @@ -47,7 +47,6 @@ in pkgs.gcc pkgs.sqlite pkgs.git-filter-repo - pkgs.shellcheck srcml pkgs.universal-ctags @@ -66,14 +65,15 @@ in languages.rust.enable = true; env.LEGACY_JAVA_HOME = "${legacyJdk}"; + env.SRCML_PREFIX = "${srcml}"; cachix.pull = [ "ccslcodevcregit" ]; - # `devenv test` gates every tracked shell script. The threshold is `error` - # because the two scripts already carry 31 lower findings; raise it to - # `warning` once those are fixed. + # `devenv test` gates every tracked shell script at `error`. shellcheck is not + # in `packages`: that would change the shell env in srcml2token's RUNPATH, and + # so the tokenizer identity of every C, C++, Java and Rust blob. enterTest = '' - git ls-files -z '*.sh' | xargs -0 --no-run-if-empty shellcheck -S error + git ls-files -z '*.sh' | xargs -0 --no-run-if-empty ${pkgs.shellcheck}/bin/shellcheck -S error echo "shellcheck: no errors" pytest -q prove -r $(git ls-files '*/t/*.t' | xargs -n1 dirname | sort -u) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 25df827c0..6fdaddcb3 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -300,18 +300,15 @@ produces no blame and no dataset row rather than rows of unparsed text: | Mechanism | Recorded as | Effect here | |---|---|---| -| the **blob denylist**, `blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list ships in the jar; nothing at run time can extend or override it | +| the **blob denylist**, `BlobDenylistEntries.scala` | `blobsDenylisted`, plus one `EXCLUDED denylisted blob` line per blob naming its sha, path and cited reason | no rows for those blobs. The list is compiled into the jar | | **oversized** blobs (at or above JGit's stream-file threshold) | `blobsOversized`, plus one `EXCLUDED oversized blob` line each | no rows for those blobs | -| a blob the tokenizer **timed out** on | `blobsTimedOut`, exit 4 | **no Parquet at all**: steps 3-10 never run, so this generator is never reached and the project cannot publish while a timeout is unexplained | +| a blob the tokenizer **timed out** or **crashed** on | `blobsTimedOut` or `blobsParserCrashed`, plus one `EXCLUDED failed blob` line each with `reason=timeout` or `reason=parser-crash` | no rows for those blobs. The run still exits 0 | -One example, so the shape is clear: [`2ee2673a`](https://github.com/tencent/tencentkona-21/blob/2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65/test/langtools/tools/javac/annotations/typeAnnotations/newlocations/TestNewCastArray.java) -— srcML 1.1.0's Java parser does not terminate on type-annotated array types -(upstream [`srcML/srcML#2361`](https://github.com/srcML/srcML/issues/2361), closed by the -revision `nix/srcml.nix` pins). -The list is keyed by content, not path: this file moved in a repository -reorganisation, so its four revisions at two paths are four entries. Failure is -non-termination, not slowness — 0 bytes out, one core at 100%, at every budget -from 5 s to 600 s. +The denylist holds 209 blob ids, keyed by content, in three srcML 1.1.0 defects: +8 Java blobs that do not terminate (upstream +[`srcML/srcML#2361`](https://github.com/srcML/srcML/issues/2361)), 4 C blobs that do +not terminate (no upstream issue), and 197 C and C++ blobs on which srcML crashes +under `--position`. The header of the file carries the reproducers and citations. Also outside the dataset, by mask rather than by exclusion: M4 (`.am`, `.ac`), whose tokenizer's lexer is not fit for real autotools input, and `.ixx`, `.inl`, @@ -355,9 +352,13 @@ persons.db ─────────────┤ For each `.blame` file: 1. Parse each line as `commit_sha;token_content` -2. Walk through the original source file character-by-character to match tokens -3. Classify each token (structural vs content, type, value) -4. Insert into SQLite `token_map` +2. Repair the token line: drop the Rust tokenizer's `line:col` prefix and, + in a strict UTF-8, non-`.rs` file without a byte-order mark, undo srcML + 1.1.0's Latin-1 misreading (`ö` becomes `ö`). The source is read as + `utf-8-sig`, so a BOM does not shift the walk. Counts go to a `Repaired:` line. +3. Walk through the original source file character-by-character to match tokens +4. Classify each token (structural vs content, type, value) +5. Insert into SQLite `token_map` ### Phase 2: DuckDB JOIN → Parquet diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index c4f4a1da6..587370f99 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -25,12 +25,14 @@ """ import argparse +import io import logging import os import re import sqlite3 import sys import tempfile +from collections import Counter from pathlib import Path logger = logging.getLogger(__name__) @@ -289,14 +291,72 @@ def parse_blame_line(line: str) -> tuple[str, str] | None: return commit_sha, token_content +# The Rust tokenizer prefixes token lines with `L:C` (also `N:-`, `-:-`); +# left in, it lands in token_type and shifts the walk. +_POSITION_PREFIX = re.compile(r"^(?:\d+|-):(?:\d+|-)\t") + +# The prebuilt srcML 1.1.0 reads UTF-8 as Latin-1 (`ö` becomes `ö`): a lead +# byte lands in U+00C2..U+00F4, a continuation byte in U+0080..U+00BF. +_MOJIBAKE = re.compile("[Â-ô][\u0080-¿]") + +_UTF8_BOM = b"\xef\xbb\xbf" + + +def undo_mojibake(text: str) -> str: + """Return the UTF-8 text that srcML misread as Latin-1, or `text` as is.""" + try: + return text.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + return text + + +def read_source(source_path: Path) -> tuple[str, bool, bool]: + """Return (text, had a BOM, mojibake repair applies). The BOM is dropped as + the tokenizers drop it. Repair needs strict UTF-8, no BOM (srcML decodes + those right) and not Rust (no srcML).""" + raw = source_path.read_bytes() + has_bom = raw.startswith(_UTF8_BOM) + try: + raw.decode("utf-8") + strict = True + except UnicodeDecodeError: + strict = False + # TextIOWrapper, not raw.decode: it keeps text-mode newline translation. + with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-8-sig", + errors="replace") as f: + text = f.read() + repairable = strict and not has_bom and not str(source_path).endswith(".rs") + return text, has_bom, repairable + + +def repair_token_line(token_content: str, repairable: bool, + stats: Counter) -> str: + """Strip the Rust position prefix and, if `repairable`, undo srcML mojibake, + counting each in `stats`. Run before classify_and_skip: a wrong token + length shifts every later position in the file.""" + prefix = _POSITION_PREFIX.match(token_content) + if prefix: + stats["position_prefix"] += 1 + token_content = token_content[prefix.end():] + if repairable and _MOJIBAKE.search(token_content): + fixed = undo_mojibake(token_content) + if fixed != token_content: + stats["mojibake_tokens"] += 1 + token_content = fixed + return token_content + + def process_blame_file( - blame_path: Path, source_path: Path, rel_path: str, db_cursor + blame_path: Path, source_path: Path, rel_path: str, db_cursor, + stats: Counter | None = None, ) -> int: with open(blame_path, encoding="utf-8", errors="replace") as f: blame_lines = f.readlines() - with open(source_path, encoding="utf-8", errors="replace") as f: - source_text = f.read() + source_text, has_bom, repairable = read_source(source_path) + if stats is None: + stats = Counter() + stats["bom_files"] += has_bom reader = SourceReader(source_text) counted = [0] @@ -310,6 +370,7 @@ def rows(): continue commit_sha, token_content = parsed + token_content = repair_token_line(token_content, repairable, stats) info = classify_and_skip(token_content, reader) counted[0] += 1 yield ( @@ -577,6 +638,7 @@ def main(): cursor = sync_conn.cursor() total_tokens = 0 files_processed = 0 + repairs = Counter() for bf in blame_files: rel = bf.relative_to(blame_root) @@ -591,7 +653,7 @@ def main(): if args.verbose: print(f" {bf.name} -> {rel_str}") - count = process_blame_file(bf, source_path, rel_str, cursor) + count = process_blame_file(bf, source_path, rel_str, cursor, repairs) total_tokens += count files_processed += 1 @@ -608,6 +670,9 @@ def main(): sync_conn.close() print(f"Synced {files_processed} files, {total_tokens} tokens") + print(f"Repaired: {repairs['mojibake_tokens']} mojibake tokens, " + f"{repairs['position_prefix']} position prefixes, " + f"{repairs['bom_files']} files with a byte-order mark") if total_tokens == 0: print("ERROR: no tokens processed, nothing to output", file=sys.stderr) diff --git a/nix/srcml.nix b/nix/srcml.nix index 93f91a247..62250b5dc 100644 --- a/nix/srcml.nix +++ b/nix/srcml.nix @@ -1,6 +1,6 @@ -# srcML at a pinned source revision. The 1.1.0 release binary segfaults under -# `--position` (srcML/srcML#2395, fixed upstream by cebd3ab, still unreleased), -# and the pipeline always passes `--position`. +# srcML 1.1.0 built from its release tag, for the libsrcml headers. Later +# revisions change the tokens of files that 1.1.0 parses (srcML/srcML#2325 marks +# up some macro calls as expressions; UTF-8 input is no longer read as Latin-1). { lib , stdenv , fetchFromGitHub @@ -13,7 +13,6 @@ , libarchive , curl , libiconv -, libxcb }: let @@ -36,33 +35,25 @@ let hash = "sha256-O9l7k1/2fVfEcyfJmWzXmju6TRQHK+NZFoaYmdd5KRg="; }; - clip = fetchFromGitHub { - owner = "dacap"; - repo = "clip"; - rev = "f2bd226fd96f3431f4675d9c13aa7334dd06ddda"; # tag v1.15 - hash = "sha256-qviH7XwNOTA3sMnfwaJL+GqXHxeG6giIZneuTeB+Lm8="; - }; - cli11 = fetchurl { - url = "https://github.com/CLIUtils/CLI11/releases/download/v2.7.0/CLI11.hpp"; - hash = "sha256-WqjWN4eSFj0eDfZC7uMTrO7KaKU0ZFZq5yBNWcL14Wg="; + url = "https://github.com/CLIUtils/CLI11/releases/download/v2.5.0/CLI11.hpp"; + hash = "sha256-S/CpSQqnIJF2zNpwVE+VQT5ZTSIHzKM8nNGN7RiaY6Y="; }; in stdenv.mkDerivation { pname = "srcml"; - version = "1.1.0-unstable-2026-09-20"; + version = "1.1.0"; src = fetchFromGitHub { owner = "srcML"; repo = "srcML"; - rev = "8a3a629d84ac0e0228a5757e6b9876742f88ed25"; - hash = "sha256-stn1VP+D9Awd9B+r3UWIjHJUkoNU52SOJndX9hLEnrY="; + rev = "af1efb78ddd6fc4a8fc75bc6dc3c2e22c5fd812d"; # tag v1.1.0 + hash = "sha256-AdPVoUKI4/EigJz34BuGIbbN0jUGZTIxOQbmXr8IImM="; }; nativeBuildInputs = [ cmake ninja jdk_headless ]; buildInputs = [ libxml2 libxslt libarchive curl ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ libxcb ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ]; # Without this the manpage step downloads a prebuilt page from a bare IP over @@ -80,7 +71,6 @@ stdenv.mkDerivation { tar xf ${antlr} -C "$deps" cp -r --no-preserve=mode,ownership ${tinysha1} "$deps/tinysha1" cp -r --no-preserve=mode,ownership ${ctpl} "$deps/ctpl" - cp -r --no-preserve=mode,ownership ${clip} "$deps/clip" cp --no-preserve=mode,ownership ${cli11} "$deps/cli11/CLI11.hpp" cp --no-preserve=mode,ownership \ ${lib.getDev libarchive}/include/archive.h \ @@ -91,7 +81,6 @@ stdenv.mkDerivation { "-DFETCHCONTENT_SOURCE_DIR_ANTLRSRC=$deps/antlr-2.7.7" "-DFETCHCONTENT_SOURCE_DIR_TINYSHA1=$deps/tinysha1" "-DFETCHCONTENT_SOURCE_DIR_CTPL_STL_SRC=$deps/ctpl" - "-DFETCHCONTENT_SOURCE_DIR_CLIP=$deps/clip" "-DFETCHCONTENT_SOURCE_DIR_CLI11=$deps/cli11" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE1=$deps/libarchive-include" "-DFETCHCONTENT_SOURCE_DIR_LIBARCHIVEINCLUDE=$deps/libarchive-include" diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index b7ed4ed09..7ac200e42 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -12,6 +12,8 @@ artifacts (jars, tokenizers) are built automatically first — run inside Build: --build-only build all pipeline artifacts and exit, running nothing + --ensure-artifacts + build missing artifacts and a stale Rust tokenizer, then exit Target repository: --repo-url URL git URL (or local path) of the repository to process (REQUIRED) @@ -20,9 +22,13 @@ Target repository: --commit-url URL base URL for the commit links in the generated HTML (default: derived from --repo-url as /commit/, which is correct for GitHub/GitLab-style hosts) - --mask REGEX regex selecting the files to tokenize; quote it - (default: `perl tokenize/fileMask.pl`, every extension - cregit can parse). A different mask forces a full rebuild. + --mask REGEX regex selecting the files to tokenize; quote it (default: + `perl tokenize/fileMask.pl`, every extension cregit parses). + A different mask forces a full rebuild, unless --mask-widened. + --mask-widened resume across a wider mask, keeping blob_map's tokenizations. + Needs FROM_STEP>=2, not with --mode sharded. + --retokenize EXTS redo the cached tokens of these extensions (e.g. rs or c,h) + after their tokenizer changed. Needs FROM_STEP=2, not sharded. --work DIR working/output directory (default: ../cregit-files). NOTE: a full run (FROM_STEP=1) starts by deleting this directory; use one directory per target repository. @@ -46,12 +52,8 @@ Target repository: A step-1 wipe that would delete a memo holding 10,000 entries or more (MEMO_KEEP_THRESHOLD, overridable as CREGIT_MEMO_KEEP_THRESHOLD) is refused; see --force-clean. - --blob-timeout N wall-clock budget in seconds for one blob's tokenizer - (blobExec default: 600). A child that exceeds it is killed, - that blob is left untokenized, step 2 stops with exit 4 and - a step-2 resume retries exactly those blobs. Also settable - as CREGIT_BLOB_TIMEOUT in the environment, which is how to - reach it through ctp.py. + --blob-timeout N seconds before one blob's tokenizer is killed and the blob + excluded (blobExec default: 600; env CREGIT_BLOB_TIMEOUT). --stall-timeout N watchdog window in seconds (blobExec default: 1800). If no blob, tree, commit or blob copy completes anywhere in this window the run is killed with exit 5. Must be larger than @@ -72,6 +74,7 @@ Output: should skip them. NOTE: the HTML views are the fallback output when python3+duckdb is missing, so --skip-html without duckdb leaves the run with no final artifact. + --reblame re-blame every file in step 7, overwriting its .blame output --gc MODE how to pack the generated cregit repo after tokenizing (default: plain) none do not pack at all. Fastest, but every later @@ -111,6 +114,9 @@ Tokenizer: for repos too large to tokenize in one process; delegates to blobExec/shard_build.sh --shards N shard count for --mode sharded (default: 4) + --tokenizer-worker + tokenize through a pool of persistent tokenizer processes; same + output. Needs --mode pipeline or pipeline-trees (default: off) --jobs N concurrent blame/HTML processes (default: CREGIT_JOBS, otherwise min(4, available CPUs)). Blame is the pipeline's bottleneck. Each file is independent, so the output does not @@ -189,6 +195,8 @@ DEFAULT_JOBS=4 JOBS=${CREGIT_JOBS:-$DEFAULT_JOBS} FROM_STEP=1 BUILD_ONLY=0 +# Unlike --build-only, runs exactly the guard a normal run runs (and tests it). +ENSURE_ARTIFACTS_ONLY=0 REPO_GIT_URL="" REPO_NAME="" REPO_COMMIT_URL="" @@ -197,6 +205,7 @@ WORK="../cregit-files" # Empty means "/memo"; resolved after parsing because it depends on --work. MEMO_DIR="" SKIP_HTML=0 +REBLAME=0 GC_MODE="plain" MEMORY_LIMIT="" DUCKDB_THREADS="" @@ -206,10 +215,13 @@ DUCKDB_THREADS="" PROJECT_META="" PROJECT_KEY="" FORCE_CLEAN=0 +MASK_WIDENED=0 +RETOKENIZE="" # Empty means "do not pass the flag". The CREGIT_* fallbacks are how ctp.py, # which has no passthrough of its own, reaches these values. BLOB_TIMEOUT="${CREGIT_BLOB_TIMEOUT:-}" STALL_TIMEOUT="${CREGIT_STALL_TIMEOUT:-}" +TOKENIZER_WORKER=0 KEEP_MARKERS="TOKENIZE-TIMEOUTS TOKENIZE-STALLED TOKENIZE-PARSER-CRASHES" @@ -339,6 +351,8 @@ tokenize_gate() { marker="TOKENIZE-PARSER-CRASHES" summary="$stage hit a parser crash: deterministic, so --blob-timeout will not help. Fix srcML, or denylist the diagnosed blob." ;; + 7) die "$stage refused --retokenize, which would have invalidated nothing (exit 7). + Check the extension spelling and that --memo-dir is this project's memo ($MEMO_DIR)." ;; *) die "$stage failed (exit $status)" ;; esac @@ -356,6 +370,7 @@ need_val() { while [ $# -gt 0 ]; do case "$1" in --build-only) BUILD_ONLY=1; shift ;; + --ensure-artifacts) ENSURE_ARTIFACTS_ONLY=1; shift ;; --repo-url) need_val "$@"; REPO_GIT_URL="$2"; shift 2 ;; --repo-name) need_val "$@"; REPO_NAME="$2"; shift 2 ;; --commit-url) need_val "$@"; REPO_COMMIT_URL="$2"; shift 2 ;; @@ -363,8 +378,12 @@ while [ $# -gt 0 ]; do --work) need_val "$@"; WORK="$2"; shift 2 ;; --memo-dir) need_val "$@"; MEMO_DIR="$2"; shift 2 ;; --skip-html) SKIP_HTML=1; shift ;; + --reblame) REBLAME=1; shift ;; --force-clean) FORCE_CLEAN=1; shift ;; + --mask-widened) MASK_WIDENED=1; shift ;; + --retokenize) need_val "$@"; RETOKENIZE="$2"; shift 2 ;; --blob-timeout) need_val "$@"; BLOB_TIMEOUT="$2"; shift 2 ;; + --tokenizer-worker) TOKENIZER_WORKER=1; shift ;; --stall-timeout) need_val "$@"; STALL_TIMEOUT="$2"; shift 2 ;; --gc) need_val "$@"; GC_MODE="$2"; shift 2 ;; --memory-limit) need_val "$@"; MEMORY_LIMIT="$2"; shift 2 ;; @@ -393,6 +412,52 @@ if [ -z "$MASK" ]; then [ -n "$MASK" ] || { echo "${SELF_DIR}/tokenize/fileMask.pl printed nothing" >&2; exit 2; } fi +if [ "$MASK_WIDENED" = 1 ] && [ "$FROM_STEP" = "1" ]; then + echo "--mask-widened needs FROM_STEP>=2: step 1 deletes $WORK and the blob map it reuses. + Resume with: $0 [same flags] --mask-widened 2 (ctp.py: --mask-widened --from-step 2)" >&2 + exit 2 +fi + +if [ "$MASK_WIDENED" = 1 ] && [ "$MODE" = "sharded" ]; then + echo "--mask-widened is not available with --mode sharded: shards keep no mask to widen. + Reuse a prior run's tokenizations with shard_build.sh --warm-db instead." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$FROM_STEP" != "2" ]; then + echo "--retokenize needs FROM_STEP=2 exactly (got $FROM_STEP): step 1 deletes the cache, 3+ skip it. + Resume with: $0 [same flags] --retokenize $RETOKENIZE 2 (ctp.py: --from-step 2)" >&2 + exit 2 +fi + +if [ "$REBLAME" = 1 ] && [ "$FROM_STEP" -gt 7 ]; then + echo "--reblame needs FROM_STEP<=7 (got $FROM_STEP): later steps skip the re-blame. + Resume with: $0 [same flags] --reblame 7 (ctp.py: --reblame --from-step 7)" >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$MODE" = "sharded" ]; then + echo "--retokenize is not available with --mode sharded: shards keep no cached tokens." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ] && [ "$MASK_WIDENED" = 1 ]; then + echo "--mask-widened and --retokenize cannot be used in the same run. + Widen first, then resume again with --retokenize." >&2 + exit 2 +fi + +if [ -n "$RETOKENIZE" ]; then + for _ext in ${RETOKENIZE//,/ }; do + case "$_ext" in + ''|*[!a-z0-9+]*) + echo "invalid --retokenize: '$_ext'. Want lowercase extensions without a dot, + as in tokenize/CregitLanguages.pm: --retokenize rs or --retokenize c,h" >&2 + exit 2 ;; + esac + done +fi + # Validate early: pack_cregit_repo runs only once tokenizing has finished. case "$GC_MODE" in none|plain|aggressive) ;; @@ -486,6 +551,9 @@ case "$MODE" in serial|pipeline|pipeline-trees|sharded) ;; *) echo "invalid --mode: $MODE (serial|pipeline|pipeline-trees|sharded)" >&2; exit 2 ;; esac +if [ "$TOKENIZER_WORKER" = 1 ] && [ "$MODE" != pipeline ] && [ "$MODE" != pipeline-trees ]; then + echo "--tokenizer-worker needs --mode pipeline or pipeline-trees" >&2; exit 2 +fi if [ "$MODE" = "sharded" ]; then [ "$SHARDS" -ge 1 ] 2>/dev/null || { echo "--shards must be a positive integer" >&2; exit 2; } fi @@ -561,30 +629,45 @@ build_all() { log "build complete" } -# needs_build ...: true when the artifact is missing, or -# when any source is newer than it. Staleness has to count, not just absence: a -# jar left behind by another branch's checkout is reused otherwise, and the -# pipeline then runs that branch's code without saying so. +RUST_TOKENIZER_SOURCES=( + "$CREGIT/tokenize/rustTokenizer/Makefile" + "$CREGIT/tokenize/rustTokenizer/Cargo.toml" + "$CREGIT/tokenize/rustTokenizer/Cargo.lock" + "$CREGIT/tokenize/rustTokenizer/src" +) + +NEEDS_BUILD_REASON="" needs_build() { - artifact=$1 - shift - [ -e "$artifact" ] || return 0 - [ -n "$(find "$@" -type f -newer "$artifact" -print -quit 2>/dev/null)" ] + local artifact=$1; shift + NEEDS_BUILD_REASON="" + [ $# -gt 0 ] || die "needs_build: no sources declared for $artifact (a source list is missing)" + local p + # A missing source path would silently stop guarding the artifact. + for p in "$@"; do + [ -e "$p" ] || die "declared source path does not exist: $p (it guards $artifact). + The checkout is incomplete or the source list in this script is wrong." + done + if [ ! -e "$artifact" ]; then + NEEDS_BUILD_REASON="missing" + return 0 + fi + local newer + newer=$(find "$@" -newer "$artifact" -print -quit 2>/dev/null) || true + [ -n "$newer" ] || return 1 + NEEDS_BUILD_REASON="$newer" + return 0 } -# Build only what is missing or out of date, so an up-to-date checkout starts -# instantly. Each list names sources only, never a target/ directory: build -# output is newer than the artifact by definition and would always look stale. ensure_artifacts() { - S="$CREGIT/tokenize/srcMLtoken" - R="$CREGIT/tokenize/rustTokenizer" - needs_build "$SRCML2TOKEN" "$S"/*.cpp "$S"/*.hpp "$S/Makefile" && build_srcml2token - needs_build "$RUST_TOKENIZER" "$R/src" "$R/Cargo.toml" "$R/Cargo.lock" && build_rust_tokenizer - needs_build "$BFG" "$CREGIT/blobExec/src" "$CREGIT/blobExec/build.sbt" "$CREGIT/blobExec/project" && build_blobexec - needs_build "$SLICKGITLOG_JAR" "$CREGIT/slickGitLog/src" "$CREGIT/slickGitLog/build.sbt" && build_legacy_jar slickGitLog - needs_build "$PERSONS_JAR" "$CREGIT/persons/src" "$CREGIT/persons/build.sbt" && build_legacy_jar persons - needs_build "$REMAPCOMMITS_JAR" "$CREGIT/remapCommits/src" "$CREGIT/remapCommits/build.sbt" && build_legacy_jar remapCommits - return 0 # a false needs_build above must not fail the function under `set -e` + [ -x "$SRCML2TOKEN" ] || build_srcml2token + if needs_build "$RUST_TOKENIZER" "${RUST_TOKENIZER_SOURCES[@]}"; then + log "artifact out of date ($NEEDS_BUILD_REASON): $RUST_TOKENIZER" + build_rust_tokenizer + fi + [ -f "$BFG" ] || build_blobexec + [ -f "$SLICKGITLOG_JAR" ] || build_legacy_jar slickGitLog + [ -f "$PERSONS_JAR" ] || build_legacy_jar persons + [ -f "$REMAPCOMMITS_JAR" ] || build_legacy_jar remapCommits } if [ "$BUILD_ONLY" = 1 ]; then @@ -592,10 +675,14 @@ if [ "$BUILD_ONLY" = 1 ]; then exit 0 fi -# Auto-build any missing artifact BEFORE the work directory is touched, so a -# build failure never disturbs the outputs of a previous run. +# Before the work directory is touched, so a failed build keeps earlier outputs. ensure_artifacts +if [ "$ENSURE_ARTIFACTS_ONLY" = 1 ]; then + log "artifacts up to date" + exit 0 +fi + REPO_PATH_ORIGINAL="${WORK}/${REPO_NAME}-original" REPO_PATH_CREGIT="${WORK}/${REPO_NAME}-cregit" REPO_PATH_ORIGINAL_BARE="${REPO_PATH_ORIGINAL}.git" @@ -695,6 +782,40 @@ echo " Log: $LOG_FILE" echo "████████████████████████████████████████████████████████████████████████" echo "" +# A re-fold gives every cregit commit a new sha: step 6 cannot clone over its old +# dir and step 7 would keep blame of the old shas. Keeps the bare repos, blob map +# and memo. Named paths only, never a glob or $WORK itself. +drop_refold_derived_artifacts() { # $1 = which flag is asking, for the log + log "$1: dropping what derives from the tokenized repo (the re-fold changes every sha)." + log " KEEPING: $REPO_PATH_ORIGINAL_BARE, $REPO_PATH_CREGIT_BARE, $DB_PATH_BLOBMAP, $MEMO_DIR" + local _stale + for _stale in \ + "$WORK/blame" \ + "$WORK/html" \ + "$REPO_PATH_ORIGINAL" \ + "$REPO_PATH_CREGIT" \ + "$DB_PATH_ORIGINAL" \ + "$DB_PATH_CREGIT" \ + "$DB_PATH_PERSONS" \ + "$XLS_PATH_PERSONS" \ + "$DATASET_PATH" \ + "${WORK}/${REPO_NAME}.validated" + do + if [ -e "$_stale" ]; then + log " dropping $_stale" + rm -rf -- "$_stale" + fi + done + mkdir -p "$WORK/blame" + [ "$SKIP_HTML" = 1 ] || mkdir -p "$WORK/html" +} + +if [ "$MASK_WIDENED" = 1 ]; then + drop_refold_derived_artifacts "--mask-widened" +fi + +# --retokenize drops them at the end of step 2 instead. + # --------------------------------------------------------------------------- # Step 1 — clone bare original repo # --------------------------------------------------------------------------- @@ -716,14 +837,21 @@ if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then export BFG_MEMO_DIR="$MEMO_DIR" -# Route through the tokenize.pl dispatcher (not tokenizeSrcMl.pl directly) so it can -# fan out by language: srcML for .c/.h, rustTokenizer for .rs, etc. The --srcml* / -# --ctags paths are forwarded only to the srcML parser. Behavior-preserving for C. +# tokenize.pl dispatches by language; --srcml2token/--ctags reach only srcML. export BFG_TOKENIZE_CMD="${CREGIT}/tokenize/tokenize.pl \ --srcml2token=${SRCML2TOKEN} \ - --srcml=$(which srcml) \ --ctags=$(which ctags)" +# Fatal when empty: an empty identity turns blobExec's cache check off. +TOKENIZER_IDENTITY=$(perl "${CREGIT}/tokenize/tokenizerIdentity.pl" \ + --srcml2token="${SRCML2TOKEN}" \ + --ctags="$(which ctags)") \ + || die "cannot compute the tokenizer identity (tokenize/tokenizerIdentity.pl). + Build the artifacts first: $0 --ensure-artifacts" +[ -n "$TOKENIZER_IDENTITY" ] \ + || die "tokenize/tokenizerIdentity.pl printed nothing; refusing to run with no tokenizer identity" +log "tokenizer identity: $TOKENIZER_IDENTITY" + TOKENIZE_RC=0 if [ "$MODE" = "sharded" ]; then # Memory-bounded path: N tree-only shards in parallel, then merge + serial @@ -739,11 +867,24 @@ if [ "$MODE" = "sharded" ]; then ${BLOB_TIMEOUT:+--blob-timeout "$BLOB_TIMEOUT"} \ ${STALL_TIMEOUT:+--stall-timeout "$STALL_TIMEOUT"} || TOKENIZE_RC=$? tokenize_gate "$TOKENIZE_RC" "sharded tokenize (shard_build.sh)" + log "note: --mode sharded records no tokenizer identity in the merged blob map," + log " so a later resume cannot detect a tokenizer change against it." else MODE_FLAG="" [ "$MODE" = "pipeline" ] && MODE_FLAG="--pipeline" [ "$MODE" = "pipeline-trees" ] && MODE_FLAG="--pipeline-trees" - java -jar "$BFG" $MODE_FLAG \ + WIDENED_FLAG="" + [ "$MASK_WIDENED" = 1 ] && WIDENED_FLAG="--mask-widened" + WORKER_FLAG="" + [ "$TOKENIZER_WORKER" = 1 ] && WORKER_FLAG="--tokenizer-worker=${CREGIT}/tokenizeByBlobId/tokenWorker.pl" + # blobExec refuses --retokenize without --memo-dir: the memo caches the same tokens. + RETOKENIZE_FLAGS=() + if [ -n "$RETOKENIZE" ]; then + RETOKENIZE_FLAGS=("--retokenize=$RETOKENIZE" "--memo-dir=$MEMO_DIR") + fi + java -jar "$BFG" $MODE_FLAG $WIDENED_FLAG ${WORKER_FLAG:+"$WORKER_FLAG"} \ + "--tokenizer-identity=$TOKENIZER_IDENTITY" \ + ${RETOKENIZE_FLAGS[@]+"${RETOKENIZE_FLAGS[@]}"} \ ${BLOB_TIMEOUT:+--blob-timeout=$BLOB_TIMEOUT} \ ${STALL_TIMEOUT:+--stall-timeout=$STALL_TIMEOUT} \ "$REPO_PATH_ORIGINAL_BARE" \ @@ -765,6 +906,11 @@ for _m in $KEEP_MARKERS; do fi done +# Only after blobExec accepted it: a refused one (exit 7) must not cost the blame. +if [ -n "$RETOKENIZE" ]; then + drop_refold_derived_artifacts "--retokenize $RETOKENIZE" +fi + pack_cregit_repo fi end_step @@ -821,9 +967,14 @@ end_step step "blame" if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then [ -d "$REPO_PATH_CREGIT" ] || die "step 6 did not produce $REPO_PATH_CREGIT" +# Without --overwrite, a re-blame skips every file that has .blame output. +REBLAME_OPTS=() +[ "$REBLAME" = 1 ] && REBLAME_OPTS+=(--overwrite) +[ "$REBLAME" = 1 ] && log "--reblame: replacing every .blame file, not resuming" perl $CREGIT/blameRepo/blameRepoFiles.pl \ --jobs="$JOBS" \ --formatBlame=$CREGIT/blameRepo/formatBlame.pl \ + ${REBLAME_OPTS[@]+"${REBLAME_OPTS[@]}"} \ $REPO_PATH_CREGIT $WORK/blame "$MASK" fi end_step diff --git a/tests/ensure_artifacts.sh b/tests/ensure_artifacts.sh new file mode 100755 index 000000000..ba7721927 --- /dev/null +++ b/tests/ensure_artifacts.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# Tests for ensure_artifacts(): only the Rust tokenizer (the one compiled +# tokenizer) is rebuilt when older than its sources; other artifacts are +# existence-only. make and sbt are stubbed on PATH and log their argv. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# The runner takes CREGIT from $(pwd), so a fixture holds a copy of it. +# Sources are stamped OLD and artifacts NEW: an up-to-date checkout. +OLD_STAMP=202601010000 +NEW_STAMP=202602010000 + +SOURCES=( + tokenize/srcMLtoken/Makefile + tokenize/srcMLtoken/srcml2token.cpp + tokenize/srcMLtoken/srcml2token.hpp + tokenize/srcMLtoken/srcml2tokenHandlers.cpp + tokenize/srcMLtoken/srcml2tokenHandlers.hpp + tokenize/rustTokenizer/Makefile + tokenize/rustTokenizer/Cargo.toml + tokenize/rustTokenizer/Cargo.lock + tokenize/rustTokenizer/src/main.rs + blobExec/build.sbt + blobExec/project/build.properties + blobExec/project/plugins.sbt + blobExec/src/main/scala/cregit/blobexec/Main.scala + slickGitLog/build.sbt + slickGitLog/project/build.properties + slickGitLog/project/plugins.sbt + slickGitLog/src/main/scala/gitLogToDb.scala + persons/build.sbt + persons/project/build.properties + persons/project/plugins.sbt + persons/src/main/scala/unifyPersons.scala + remapCommits/build.sbt + remapCommits/project/build.properties + remapCommits/project/plugins.sbt + remapCommits/src/main/scala/remapCommits.scala +) + +ARTIFACTS=( + tokenize/srcMLtoken/srcml2token + tokenize/rustTokenizer/target/release/rust_tokenizer + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar +) + +fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/ensureart-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + for p in "${SOURCES[@]}"; do + mkdir -p "$f/$(dirname "$p")" + echo "source" > "$f/$p" + touch -t "$OLD_STAMP" "$f/$p" + done + # The guard also compares directory mtimes; unstamped, they look just edited. + find "$f" -type d -exec touch -t "$OLD_STAMP" {} + + for p in "${ARTIFACTS[@]}"; do + mkdir -p "$f/$(dirname "$p")" + echo "artifact" > "$f/$p" + chmod +x "$f/$p" + touch -t "$NEW_STAMP" "$f/$p" + done + printf '%s' "$f" +} + +# cwd is logged too: the three legacy jars share one sbt command line. +make_stubs() { # $1 = bin dir + local b=$1 tool + for tool in make sbt; do + cat > "$b/$tool" <> "\$STUB_ARGV" +exit 0 +STUB + chmod +x "$b/$tool" + done +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/ensurebin-XXXXXX") +make_stubs "$BIN" +trap 'rm -rf "$BIN"' EXIT + +run_guard() { # $1 = fixture dir + local f=$1 + ARGV="$f/argv.log" + : > "$ARGV" + ( cd "$f" \ + && STUB_ARGV="$ARGV" PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 \ + timeout 120 bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$f/work" \ + --mask '\.c$' \ + --ensure-artifacts 2>&1 ) +} + +built() { # $1 = fixture, $2 = grep pattern for the build command + grep -q -- "$2" "$1/argv.log" +} + +build_count() { # $1 = fixture + if [ -s "$1/argv.log" ]; then wc -l < "$1/argv.log" | tr -d ' '; else echo 0; fi +} + +# --------------------------------------------------------------------------- +echo "case 1: everything current — the fast path builds nothing" +W=$(fixture) +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +N=$(build_count "$W"); [ "$N" -eq 0 ]; check "no build was invoked (built: $N)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: a newer Rust source — the exact defect this guard exists for" +W=$(fixture) +touch -t "$NEW_STAMP" "$W/tokenize/rustTokenizer/src/main.rs" +# An equal timestamp is not stale; make the source strictly newer. +touch "$W/tokenize/rustTokenizer/src/main.rs" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +built "$W" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? +N=$(build_count "$W"); [ "$N" -eq 1 ]; check "and nothing else was (built: $N)" $? +grep -q 'main.rs' <<<"$OUT"; check "the log names the source that triggered it" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: a missing artifact still rebuilds (the old behaviour is kept)" +W=$(fixture) +rm -f "$W/tokenize/rustTokenizer/target/release/rust_tokenizer" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +built "$W" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? +N=$(build_count "$W"); [ "$N" -eq 1 ]; check "and nothing else was (built: $N)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +rust_stale_case() { # $1 = source path, relative to the fixture + echo "case: a newer $1 rebuilds the Rust tokenizer" + local w + w=$(fixture) + touch -t "$NEW_STAMP" "$w/$1" + touch "$w/$1" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + built "$w" 'rustTokenizer'; check "rustTokenizer was rebuilt" $? + local n; n=$(build_count "$w"); [ "$n" -eq 1 ]; check "nothing else was (built: $n)" $? + rm -rf "$w" +} + +rust_stale_case tokenize/rustTokenizer/Cargo.lock +rust_stale_case tokenize/rustTokenizer/Cargo.toml +rust_stale_case tokenize/rustTokenizer/Makefile + +# --------------------------------------------------------------------------- +# Pins the accepted gap: a stale sbt jar or srcml2token is used, not rebuilt. +# --------------------------------------------------------------------------- +gap_case() { # $1 = label, $2 = source to touch, $3 = grep pattern for its build + echo "case: a newer $1 source does NOT rebuild it — known, accepted gap" + local w + w=$(fixture) + touch -t "$NEW_STAMP" "$w/$2" + touch "$w/$2" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + local n; n=$(build_count "$w"); [ "$n" -eq 0 ]; check "nothing was built (built: $n)" $? + rm -rf "$w" +} + +gap_case srcml2token tokenize/srcMLtoken/srcml2token.cpp 'srcMLtoken' +gap_case blobExec blobExec/src/main/scala/cregit/blobexec/Main.scala 'assembly' +gap_case slickGitLog slickGitLog/src/main/scala/gitLogToDb.scala 'one-jar' +gap_case persons persons/src/main/scala/unifyPersons.scala 'one-jar' +gap_case remapCommits remapCommits/src/main/scala/remapCommits.scala 'one-jar' + +missing_case() { # $1 = label, $2 = artifact to delete, $3 = grep pattern + echo "case: a missing $1 is still built" + local w + w=$(fixture) + rm -f "$w/$2" + local out rc + out=$(run_guard "$w"); rc=$? + [ "$rc" -eq 0 ]; check "exits 0" $? + built "$w" "$3"; check "$1 was built" $? + local n; n=$(build_count "$w"); [ "$n" -eq 1 ]; check "and only it (built: $n)" $? + rm -rf "$w" +} + +missing_case srcml2token tokenize/srcMLtoken/srcml2token 'srcMLtoken' +missing_case blobExec blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar 'assembly' +missing_case slickGitLog slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar 'one-jar' + +# --------------------------------------------------------------------------- +echo "case: a Rust source path the guard names but the checkout lacks is fatal" +# Otherwise a typo in the source list would silently disable the guard. +W=$(fixture) +rm -f "$W/tokenize/rustTokenizer/Cargo.lock" +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -ne 0 ]; check "the run fails (exit $RC)" $? +grep -q 'Cargo.lock' <<<"$OUT"; check "and names the missing source path" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case: --ensure-artifacts does not run the pipeline" +W=$(fixture) +OUT=$(run_guard "$W"); RC=$? +[ "$RC" -eq 0 ]; check "exits 0" $? +[ ! -d "$W/work" ]; check "the work directory was never created" $? +grep -q 'Step 1' <<<"$OUT"; [ $? -ne 0 ]; check "no pipeline step ran" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/fixtures/Tiny.java b/tests/fixtures/Tiny.java new file mode 100644 index 000000000..c9b995f9b --- /dev/null +++ b/tests/fixtures/Tiny.java @@ -0,0 +1,4 @@ +package p; +public class Tiny { + int f(int a) { return a; } +} diff --git a/tests/fixtures/Utf8.java b/tests/fixtures/Utf8.java new file mode 100644 index 000000000..f6b8783b3 --- /dev/null +++ b/tests/fixtures/Utf8.java @@ -0,0 +1,3 @@ +class Café { + int grüß(String ä) { return 1; } +} diff --git a/tests/fixtures/empty.c b/tests/fixtures/empty.c new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/latin1.c b/tests/fixtures/latin1.c new file mode 100644 index 000000000..4191f97fb --- /dev/null +++ b/tests/fixtures/latin1.c @@ -0,0 +1 @@ +int h(int bé) { return 2; } diff --git a/tests/fixtures/tiny.c b/tests/fixtures/tiny.c new file mode 100644 index 000000000..78f2de106 --- /dev/null +++ b/tests/fixtures/tiny.c @@ -0,0 +1 @@ +int main(void) { return 0; } diff --git a/tests/fixtures/tiny.cpp b/tests/fixtures/tiny.cpp new file mode 100644 index 000000000..88e4ae717 --- /dev/null +++ b/tests/fixtures/tiny.cpp @@ -0,0 +1,2 @@ +struct S { int m(int a) const { return a; } }; +template T id(T v) { return v; } diff --git a/tests/fixtures/tiny.rs b/tests/fixtures/tiny.rs new file mode 100644 index 000000000..2954eacc2 --- /dev/null +++ b/tests/fixtures/tiny.rs @@ -0,0 +1,3 @@ +fn add(a: i32, b: i32) -> i32 { + a + b // "x|y" +} diff --git a/tests/fixtures/upper.H b/tests/fixtures/upper.H new file mode 100644 index 000000000..e039d96f3 --- /dev/null +++ b/tests/fixtures/upper.H @@ -0,0 +1,6 @@ +namespace n { +class A { +public: + int x; +}; +} diff --git a/tests/fixtures/utf8.c b/tests/fixtures/utf8.c new file mode 100644 index 000000000..a5d166fda --- /dev/null +++ b/tests/fixtures/utf8.c @@ -0,0 +1,2 @@ +int café(int ä) { return 0; } +int g(int 😀x) { return 1; } diff --git a/tests/pipeline_workdir_guard.sh b/tests/pipeline_workdir_guard.sh index d12aba9ad..11229242d 100755 --- a/tests/pipeline_workdir_guard.sh +++ b/tests/pipeline_workdir_guard.sh @@ -175,6 +175,81 @@ OUT=$(CREGIT_MEMO_KEEP_THRESHOLD=2 run_runner "$W" --memo-dir "$M"); RC=$? grep -q "$W/memo" <<<"$OUT"; check "the refusal names the memo at risk" $? rm -rf "$W" "$M" +echo "case 12: --mask-widened with FROM_STEP=1 must refuse before touching anything" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened); RC=$? +[ "$RC" -eq 2 ]; check "refuses with exit 2 (got $RC)" $? +grep -q "needs FROM_STEP>=2" <<<"$OUT"; check "says which step it needs" $? +grep -q -- "--from-step 2" <<<"$OUT"; check "gives the ctp.py resume form" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map it would have reused is untouched" $? +[ ! -f "$W/pipeline.log" ]; check "and it refused before opening the log, so nothing ran" $? +rm -rf "$W" + +echo "case 13: --mask-widened with FROM_STEP=2 reaches blobExec and is accepted there" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened 2); RC=$? +! grep -q "needs FROM_STEP>=2" <<<"$OUT"; check "no refusal from the step guard" $? +grep -q "maskWidened=true" <<<"$OUT"; check "blobExec reports the flag as set" $? +! grep -q "unknown flag" <<<"$OUT"; check "the shipped jar understands it" $? +[ -f "$W/proj-blobmap.db" ]; check "and the work dir was kept, as a resume does" $? +rm -rf "$W" + +echo "case 14: --mask-widened with --mode sharded must refuse: shards have no mask to widen" +W=$(fixture none) +OUT=$(run_runner "$W" --mask-widened --mode sharded 2); RC=$? +[ "$RC" -eq 2 ]; check "refuses with exit 2 (got $RC)" $? +grep -q "not available with --mode sharded" <<<"$OUT"; check "says why" $? +grep -q -- "--warm-db" <<<"$OUT"; check "names the sharded equivalent" $? +rm -rf "$W" + +echo "case 15: --mask-widened drops what the re-fold invalidates and keeps what it reuses" +W=$(fixture none) +mkdir -p "$W/blame/src" "$W/proj-cregit" "$W/proj-original" "$W/proj-cregit.git" +echo stale > "$W/blame/src/a.c.blame" +echo stale > "$W/proj-cregit/keepout" +echo stale > "$W/proj-cregit.db" +echo stale > "$W/proj-original.db" +echo stale > "$W/proj-persons.db" +echo stale > "$W/proj-persons.xls" +echo stale > "$W/proj-dataset.parquet" +echo stale > "$W/proj.validated" +plant_memo "$W/memo" 3 +OUT=$(run_runner "$W" --mask-widened 2); RC=$? +[ ! -f "$W/blame/src/a.c.blame" ]; check "stale blame is gone (step 7 would have skipped it)" $? +[ ! -e "$W/proj-cregit/keepout" ]; check "the non-bare cregit clone is gone (step 6 clones into it)" $? +[ ! -e "$W/proj-original" ]; check "the non-bare original clone is gone (step 6 clones into it too)" $? +[ ! -f "$W/proj-cregit.db" ]; check "the cregit log DB is gone" $? +[ ! -f "$W/proj-original.db" ]; check "the original log DB is gone" $? +[ ! -f "$W/proj-persons.db" ]; check "the persons DB is gone" $? +[ ! -f "$W/proj-persons.xls" ]; check "the persons sheet is gone" $? +[ ! -f "$W/proj-dataset.parquet" ]; check "the old Parquet is gone" $? +[ ! -f "$W/proj.validated" ]; check "the stale validated stamp is gone" $? +[ -d "$W/blame" ]; check "but blame/ itself is recreated for step 7" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map is KEPT" $? +[ -d "$W/proj-cregit.git" ]; check "the cregit bare repo is KEPT (new_blob ids live there)" $? +[ -d "$W/proj-original.git" ]; check "the original bare repo is KEPT" $? +[ "$(count_memo "$W/memo")" -eq 3 ]; check "every memo entry is KEPT" $? +grep -q "KEEPING" <<<"$OUT"; check "and the log says what it kept" $? +rm -rf "$W" + +echo "case 16: without --mask-widened a resume touches none of that" +W=$(fixture none) +mkdir -p "$W/blame/src" +echo keepme > "$W/blame/src/a.c.blame" +echo keepme > "$W/proj-cregit.db" +OUT=$(run_runner "$W" 2); RC=$? +[ -f "$W/blame/src/a.c.blame" ]; check "blame is untouched" $? +[ -f "$W/proj-cregit.db" ]; check "the cregit log DB is untouched" $? +! grep -q "KEEPING" <<<"$OUT"; check "and the widening block did not run" $? +rm -rf "$W" + +echo "case 17: without the flag nothing changes, and the flag is what reaches blobExec" +grep -q 'WIDENED_FLAG=""' "$RUNNER"; check "the flag defaults to absent" $? +grep -q '\[ "\$MASK_WIDENED" = 1 \] && WIDENED_FLAG="--mask-widened"' "$RUNNER" +check "and is only added when asked for" $? +grep -q 'java -jar "\$BFG" \$MODE_FLAG \$WIDENED_FLAG' "$RUNNER" +check "so it reaches blobExec's argv" $? + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ] diff --git a/tests/reblame_passthrough.sh b/tests/reblame_passthrough.sh new file mode 100755 index 000000000..549f608d6 --- /dev/null +++ b/tests/reblame_passthrough.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tests that --reblame reaches step 7 as --overwrite (blameRepoFiles.pl skips +# existing .blame files) and is refused where step 7 would not run. +# blameRepoFiles.pl and java are stubbed and record their argv. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB + chmod +x "$1/java" +} + +# The runner takes CREGIT from $(pwd) and runs its artifact guard before any +# step, so the tree needs build files and placeholder artifacts or it builds. +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/reblamerepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p/src" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + done + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + mkdir -p "$f/blameRepo" + cat > "$f/blameRepo/blameRepoFiles.pl" <<'STUB' +#!/usr/bin/env perl +open(my $fh, '>>', $ENV{BLAME_ARGV}) or exit 0; +print $fh join(' ', @ARGV), "\n"; +close $fh; +exit 0; +STUB + chmod +x "$f/blameRepo/blameRepoFiles.pl" + : > "$f/blameRepo/formatBlame.pl" + printf '%s' "$f" +} + +# A work dir finished through step 6, with blame output a re-blame must replace. +fixture() { + local w + w=$(mktemp -d "${TMPDIR:-/tmp}/reblamework-XXXXXX") + mkdir -p "$w/proj-cregit" "$w/proj-cregit.git" "$w/blame" "$w/html" + echo "days of blame" > "$w/blame/one.c.blame" + echo "db" > "$w/proj-cregit.db" + printf '%s' "$w" +} + +run_from() { # $1 = work dir, $2 = FROM_STEP, rest = extra runner args + local w=$1 from=$2; shift 2 + # Set on the command: `VAR=x somefunc` does not export VAR to the stub. + ( cd "$REPO" \ + && PATH="$BIN:$PATH" BLAME_ARGV="${BLAME_ARGV:-}" timeout 120 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --gc none \ + --mask '\.c$' \ + "$@" "$from" 2>&1 ) +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/reblamebin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: without --reblame, step 7 does NOT get --overwrite" +W=$(fixture); ARGV="$W/blame-argv.log" +OUT=$(BLAME_ARGV="$ARGV" run_from "$W" 7); RC=$? +[ -f "$ARGV" ]; check "step 7 invoked blameRepoFiles.pl" $? +grep -q -- '--jobs=' "$ARGV"; check "and passed --jobs" $? +grep -q -- '--overwrite' "$ARGV"; [ $? -ne 0 ] +check "and NOT --overwrite: a resume stays cheap by default" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: with --reblame, step 7 gets --overwrite" +W=$(fixture); ARGV="$W/blame-argv.log" +OUT=$(BLAME_ARGV="$ARGV" run_from "$W" 7 --reblame); RC=$? +[ -f "$ARGV" ]; check "step 7 invoked blameRepoFiles.pl" $? +grep -q -- '--overwrite' "$ARGV"; check "--overwrite reached blameRepoFiles.pl" $? +grep -q -- '--formatBlame=' "$ARGV"; check "and the formatter is still named" $? +# Not the bare word "reblame": usage and refusal messages contain it too. +grep -q 'replacing every .blame file' <<<"$OUT" +check "and the run says it is replacing every .blame file" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: --reblame after step 7 is refused, because it would be skipped" +for FROM in 8 9 10; do + W=$(fixture) + OUT=$(BLAME_ARGV="$W/blame-argv.log" run_from "$W" "$FROM" --reblame); RC=$? + [ "$RC" -ne 0 ]; check "refused at step $FROM (exit $RC)" $? + grep -q 'FROM_STEP<=7' <<<"$OUT"; check "and names the step it needs" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 4: --reblame is accepted at every step that still reaches step 7" +for FROM in 2 7; do + W=$(fixture); ARGV="$W/blame-argv.log" + OUT=$(BLAME_ARGV="$ARGV" run_from "$W" "$FROM" --reblame); RC=$? + grep -q 'FROM_STEP<=7' <<<"$OUT"; [ $? -ne 0 ] + check "not refused at step $FROM" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/retokenize_passthrough.sh b/tests/retokenize_passthrough.sh new file mode 100755 index 000000000..e7107aa6e --- /dev/null +++ b/tests/retokenize_passthrough.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# Tests the runner's tokenizer identity, the --retokenize passthrough (always +# with --memo-dir: the memo is keyed on content, not tokenizer) and its guards. +# java is stubbed on PATH and records its argv. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNNER="$ROOT/run_pipeline_process.sh" +PASS=0 +FAIL=0 + +check() { # $1 = description, $2 = condition result (0/1) + if [ "$2" -eq 0 ]; then + echo " ok — $1"; PASS=$((PASS + 1)) + else + echo " FAIL — $1"; FAIL=$((FAIL + 1)) + fi +} + +# On success, creates the destination repo the caller checks for. +make_stub_java() { # $1 = bin dir + cat > "$1/java" <<'STUB' +#!/usr/bin/env bash +: "${STUB_RC:=0}" +[ -n "${STUB_ARGV:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGV" +pos=(); skip=0 +for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + case "$a" in + -jar) skip=1 ;; + -*) ;; + *) pos+=("$a") ;; + esac +done +if [ "$STUB_RC" = 0 ] && [ "${#pos[@]}" -ge 2 ]; then mkdir -p "${pos[1]}"; fi +exit "$STUB_RC" +STUB + chmod +x "$1/java" +} + +# --------------------------------------------------------------------------- +# The runner takes CREGIT from $(pwd) and needs tokenize/ for the identity, plus +# build files and placeholder artifacts, or the staleness guard would build. +# --------------------------------------------------------------------------- +repo_fixture() { + local f p + f=$(mktemp -d "${TMPDIR:-/tmp}/retokrepo-XXXXXX") + cp "$RUNNER" "$f/run_pipeline_process.sh" + cp -r "$ROOT/tokenize" "$f/tokenize" + rm -rf "$f/tokenize/rustTokenizer/target" "$f/tokenize/srcMLtoken/srcml2token" + for p in blobExec slickGitLog persons remapCommits; do + mkdir -p "$f/$p" + cp "$ROOT/$p/build.sbt" "$f/$p/build.sbt" + cp -r "$ROOT/$p/project" "$f/$p/project" + rm -rf "$f/$p/project/target" "$f/$p/project/project" + mkdir -p "$f/$p/src" + done + for p in tokenize/srcMLtoken/srcml2token \ + tokenize/rustTokenizer/target/release/rust_tokenizer \ + blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar \ + slickGitLog/target/scala-2.10/slickgitlog_2.10-0.1-SNAPSHOT-one-jar.jar \ + persons/target/scala-2.10/persons_2.10-0.1-SNAPSHOT-one-jar.jar \ + remapCommits/target/scala-2.10/remapcommits_2.10-0.1-SNAPSHOT-one-jar.jar + do + mkdir -p "$f/$(dirname "$p")" + echo "placeholder v1" > "$f/$p" + chmod +x "$f/$p" + done + # The identity also digests the libsrcml that srcml2token names. + printf '#!/bin/sh\necho "%s/libsrcml"\n' "$f" > "$f/tokenize/srcMLtoken/srcml2token" + echo "placeholder v1" > "$f/libsrcml" + printf '%s' "$f" +} + +# A finished run, resumable at step 2, with the artifacts a re-fold invalidates. +fixture() { + local w + w=$(mktemp -d "${TMPDIR:-/tmp}/retokwork-XXXXXX") + mkdir -p "$w/memo/ab/cd" "$w/proj-original.git" "$w/blame" "$w/html" + echo "hours of tokenizing" > "$w/proj-blobmap.db" + echo "tokens" > "$w/memo/ab/cd/abcdef" + echo "days of blame" > "$w/blame/one.c.blame" + echo "html" > "$w/html/index.html" + echo "db" > "$w/proj-original.db" + printf '%s' "$w" +} + +run_step2() { # $1 = work dir, rest = extra runner args + FROM_STEP + local w=$1; shift + ( cd "$REPO" \ + && PATH="$BIN:$PATH" LEGACY_JAVA_HOME=/nonexistent/jdk8 timeout 300 \ + bash ./run_pipeline_process.sh \ + --repo-url /nonexistent/does-not-exist.git \ + --repo-name proj \ + --work "$w" \ + --skip-html --gc none \ + "$@" 2>&1 ) +} + +BIN=$(mktemp -d "${TMPDIR:-/tmp}/retokbin-XXXXXX") +make_stub_java "$BIN" +REPO=$(repo_fixture) +trap 'rm -rf "$BIN" "$REPO"' EXIT + +# --------------------------------------------------------------------------- +echo "case 1: every run passes a tokenizer identity, whether or not it invalidates" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" 2); RC=$? +# The run later dies at step 5 (the stub writes no databases); reaching step 3 +# means step 2 finished and the tokenize gate passed. +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep -q -- '--tokenizer-identity=' "$ARGV"; check "blobExec was given --tokenizer-identity" $? +grep -qE -- '--tokenizer-identity=[a-z0-9+]+=[0-9a-f]{64}' "$ARGV" +check "and the value is ext=sha256 pairs" $? +grep -q -- '--retokenize=' "$ARGV"; [ $? -ne 0 ] +check "and NO --retokenize: nothing is invalidated by default" $? +grep -q -- '--memo-dir=' "$ARGV"; [ $? -ne 0 ] +check "and no --memo-dir either, which blobExec would refuse on its own" $? +[ -f "$W/blame/one.c.blame" ]; check "the blame output was left alone" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 2: --retokenize is passed through, and always with --memo-dir" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +grep -q 'Step 3' <<<"$OUT"; check "step 2 completed and the pipeline moved on" $? +grep -q -- '--retokenize=rs' "$ARGV"; check "--retokenize=rs reached blobExec" $? +grep -q -- "--memo-dir=$W/memo" "$ARGV"; check "and --memo-dir named this project's memo" $? +grep -q -- '--tokenizer-identity=' "$ARGV"; check "and the identity too, which blobExec requires" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 3: a successful invalidation drops the artifacts derived from the re-fold" +# A re-tokenized blob gives every cregit commit a new sha, so old blame names +# commits that are gone and step 6's clone hits an existing directory. +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +[ ! -f "$W/blame/one.c.blame" ]; check "the stale blame output is gone" $? +[ ! -f "$W/html/index.html" ]; check "the stale html is gone" $? +[ ! -f "$W/proj-original.db" ]; check "the stale git-log db is gone" $? +[ -f "$W/proj-blobmap.db" ]; check "the blob map is KEPT — it is what was reused" $? +[ -f "$W/memo/ab/cd/abcdef" ]; check "and so is the memo" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 4: a REFUSED invalidation (exit 7) fails the run and keeps the blame output" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=7 STUB_ARGV="$ARGV" run_step2 "$W" --retokenize rs 2); RC=$? +[ "$RC" -ne 0 ]; check "the run fails (exit $RC), it does not exit 0" $? +grep -q 'invalidat' <<<"$OUT"; check "and says the invalidation was refused" $? +[ -f "$W/blame/one.c.blame" ]; check "days of blame output survived the refusal" $? +[ -f "$W/proj-blobmap.db" ]; check "so did the blob map" $? +[ ! -f "$W/TOKENIZE-TIMEOUTS" ]; check "no resumable-work marker was written" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 5: --retokenize at step 1 is refused, because step 1 deletes the caches" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs 1); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'FROM_STEP=2' <<<"$OUT"; check "and names the resume it wants" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 6: --retokenize at step 3 is refused, because it would be skipped entirely" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs 3); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'exactly' <<<"$OUT"; check "and says the step must be exactly 2" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 7: --retokenize with --mode sharded is refused" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs --mode sharded --shards 2 2); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +grep -q 'sharded' <<<"$OUT"; check "and says why" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 8: a malformed extension list is refused before the clone" +for BAD in ".rs" "RS" "r s" "" "rs,.c"; do + W=$(fixture) + OUT=$(STUB_RC=0 run_step2 "$W" --retokenize "$BAD" 2); RC=$? + [ "$RC" -ne 0 ]; check "refused [$BAD] (exit $RC)" $? + rm -rf "$W" +done + +# --------------------------------------------------------------------------- +echo "case 9: --mask-widened and --retokenize are not combined" +W=$(fixture) +OUT=$(STUB_RC=0 run_step2 "$W" --retokenize rs --mask-widened --mask '\.c$' 2); RC=$? +[ "$RC" -ne 0 ]; check "refused (exit $RC)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "case 10: --tokenizer-worker reaches blobExec, and only in the pipelined modes" +W=$(fixture); ARGV="$W/argv.log" +OUT=$(STUB_RC=0 STUB_ARGV="$ARGV" run_step2 "$W" --tokenizer-worker 2); RC=$? +grep -q -- "--tokenizer-worker=$REPO/tokenizeByBlobId/tokenWorker.pl" "$ARGV" +check "blobExec was given the worker path" $? +OUT=$(STUB_RC=0 run_step2 "$W" --tokenizer-worker --mode serial 2); RC=$? +[ "$RC" -ne 0 ] && grep -q 'needs --mode pipeline' <<<"$OUT"; check "--mode serial is refused (exit $RC)" $? +rm -rf "$W" + +# --------------------------------------------------------------------------- +echo "" +echo "passed: $PASS failed: $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/t/languages.t b/tests/t/languages.t index a1c62bd62..da03ac112 100644 --- a/tests/t/languages.t +++ b/tests/t/languages.t @@ -1,23 +1,16 @@ #!/usr/bin/env perl -# The extension table has been stable since 2017 and needs no test. These six -# assertions cover what this PR adds: a case-insensitive mask that newly selects -# .C and .H, the override that routes those to C++, and a pipeline default -# derived from the table rather than written out a second time. +# Covers the case-insensitive mask (.C, .H) and the pipeline default derived +# from the table; the table itself is not tested. use strict; use warnings; -use Test::More tests => 6; +use Test::More tests => 2; use FindBin; use lib "$FindBin::Bin/../../tokenize"; use CregitLanguages; -is(CregitLanguages::language_for_ext('C'), 'C++', ".C routes to C++, per the GNU convention"); -is(CregitLanguages::language_for_ext('H'), 'C++', ".H routes to C++"); -is(CregitLanguages::language_for_ext('c'), 'C', ".c still routes to C"); -is(CregitLanguages::language_for_ext('h'), 'C', ".h still routes to C"); - my $mask = CregitLanguages::file_mask(); ok("src/Matrix.C" =~ /$mask/, "the mask selects .C, which the old \\.[ch]\$ mask did not"); diff --git a/tests/t/tokenizeSrcMl.t b/tests/t/tokenizeSrcMl.t index b29781418..6f1a3d7eb 100644 --- a/tests/t/tokenizeSrcMl.t +++ b/tests/t/tokenizeSrcMl.t @@ -13,8 +13,6 @@ my $srcml2token = "$root/tokenize/srcMLtoken/srcml2token"; my $fixtures = "$root/tokenize/srcMLtoken/tests"; my $expected = "$FindBin::Bin/expected"; -plan skip_all => "srcml not on PATH" - unless system("srcml --version >/dev/null 2>&1") == 0; plan skip_all => "ctags not on PATH" unless system("ctags --version >/dev/null 2>&1") == 0; plan skip_all => "srcml2token not built (cd tokenize/srcMLtoken && make)" @@ -79,7 +77,7 @@ sub run_tokenizer { { my $crasher = "$FindBin::Bin/fixtures/srcml-position-crash.c"; - my $rawRc = system("srcml -l C --position '$crasher' >/dev/null 2>&1"); + my $rawRc = system("'$srcml2token' -l C '$crasher' >/dev/null 2>&1"); my $rawSignal = $rawRc & 127; if (not $rawSignal and ($rawRc >> 8) > 128) { @@ -87,12 +85,12 @@ sub run_tokenizer { } SKIP: { - skip "srcml no longer dies on this fixture (rc=$rawRc): the upstream bug " + skip "srcml2token no longer dies on this fixture (rc=$rawRc): the upstream bug " . "appears fixed, so the crash-handling assertions cannot be exercised", 6 unless $rawSignal; is($rawSignal == 11 || $rawSignal == 6, 1, - "fixture kills srcml with SIGSEGV(11) or SIGABRT(6), got signal $rawSignal"); + "fixture kills srcml2token with SIGSEGV(11) or SIGABRT(6), got signal $rawSignal"); my ($status, $out, $err) = run_tokenizer("--position", "'$crasher'"); @@ -104,14 +102,13 @@ sub run_tokenizer { "no tokenization is emitted on stdout"); like($err, qr/FAILED/, "the failure is reported on stderr instead of passing silently"); - like($err, qr/killed by signal $rawSignal/, - "stderr names the signal, so srcml's death is attributed to srcml " - . "and not to srcml2token, which exits 0 on the truncated XML"); + like($err, qr/srcml2token was killed by signal $rawSignal/, + "stderr names the signal"); } } { - my $dies = "$workdir/srcml-that-dies"; + my $dies = "$workdir/srcml2token-that-dies"; open(my $fh, '>', $dies) or die $!; print $fh "#!/bin/sh\nkill -SEGV \$\$\n"; close $fh; @@ -123,10 +120,10 @@ sub run_tokenizer { close $fh; my ($status, $out, $err) = - run_tokenizer("--srcml='$dies'", "--position", "'$victim'"); + run_tokenizer("--srcml2token='$dies'", "--position", "'$victim'"); is($status >> 8, $PARSER_CRASH_EXIT, - "a srcml killed by a signal exits $PARSER_CRASH_EXIT, whatever srcml we ship"); + "a srcml2token killed by a signal exits $PARSER_CRASH_EXIT, whatever srcML we ship"); is($out, "", "and emits no tokenization"); like($err, qr/killed by signal 11/, "and names the signal on stderr"); } @@ -160,6 +157,6 @@ sub run_tokenizer { close $fh; my ($status, $out, $err) = run_tokenizer("--position", "\"$odd\""); - is($status, 0, "a path containing a quote is passed to the shell safely"); + is($status, 0, "a path containing a quote tokenizes"); like($out, qr/begin_unit/, "and really was tokenized"); } diff --git a/tests/t/tokenizerIdentity.t b/tests/t/tokenizerIdentity.t new file mode 100644 index 000000000..49e4d145d --- /dev/null +++ b/tests/t/tokenizerIdentity.t @@ -0,0 +1,181 @@ +#!/usr/bin/env perl + +# Tests for tokenize/tokenizerIdentity.pl: stable when nothing changes, moves +# only the affected extensions, and is fatal (never silent) on a missing part. + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); +use File::Path qw(make_path); +use File::Copy; + +use lib "$FindBin::Bin/../../tokenize"; +use CregitLanguages; + +my $realTokenizeDir = "$FindBin::Bin/../../tokenize"; + +sub fixture { + my $dir = tempdir(CLEANUP => 1); + my $tok = "$dir/tokenize"; + make_path($tok); + for my $f (qw(tokenize.pl CregitLanguages.pm tokenizerIdentity.pl tokenizeSrcMl.pl CregitSrcMl.pm)) { + copy("$realTokenizeDir/$f", "$tok/$f") or die "copy $f: $!"; + } + chmod 0755, "$tok/tokenizerIdentity.pl", "$tok/tokenize.pl", "$tok/tokenizeSrcMl.pl"; + + # The identity digests content, so "changing a tokenizer" is a new string. + make_path("$tok/rustTokenizer/target/release"); + write_file("$tok/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v1\n"); + make_path("$tok/m4Tokenizer"); + write_file("$tok/m4Tokenizer/m4.py", "M4 v1\n"); + write_srcml2token($dir, "v1"); + write_file("$dir/libsrcml", "LIBSRCML v1\n"); + write_file("$dir/ctags", "CTAGS v1\n"); + return $dir; +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die "write $path: $!"; + print $fh $content; + close $fh; + chmod 0755, $path; +} + +sub write_srcml2token { + my ($dir, $version) = @_; + write_file("$dir/srcml2token", "#!/bin/sh\n# SRCML2TOKEN $version\necho '$dir/libsrcml'\n"); +} + +sub identity { + my ($dir, @extra) = @_; + my @cmd = ("perl", "$dir/tokenize/tokenizerIdentity.pl", + "--srcml2token=$dir/srcml2token", + "--ctags=$dir/ctags", @extra); + open(my $ph, '-|', @cmd) or die "run: $!"; + my $out = do { local $/; <$ph> }; + close $ph; + my $status = $?; + $out = '' unless defined $out; + chomp $out; + return ($out, $status); +} + +sub as_map { + my ($spec) = @_; + return map { split(/=/, $_, 2) } split(/,/, $spec); +} + +# --------------------------------------------------------------------------- +my $dir = fixture(); +my ($base, $status) = identity($dir); +is($status, 0, "tokenizerIdentity.pl succeeds on a complete checkout"); + +my %baseMap = as_map($base); + +is_deeply( + [sort keys %baseMap], + [CregitLanguages::masked_extensions_sorted()], + "every masked extension gets an identity, and only those"); + +# The format blobExec's TokenizerIdentity.parse accepts. +for my $ext (sort keys %baseMap) { + like($ext, qr/^[a-z0-9+]+$/, "extension [$ext] is lowercase and dotless"); + like($baseMap{$ext}, qr/^[0-9a-f]{64}$/, "identity for [$ext] is a sha256 hex digest"); +} + +my ($again) = identity($dir); +is($again, $base, "an unchanged checkout yields a byte-identical identity"); + +# --------------------------------------------------------------------------- +write_file("$dir/tokenize/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v2 fixed\n"); +my ($afterRust) = identity($dir); +my %rustMap = as_map($afterRust); + +isnt($rustMap{rs}, $baseMap{rs}, "rebuilding the Rust tokenizer moves the .rs identity"); +for my $ext (grep { $_ ne 'rs' } sort keys %baseMap) { + is($rustMap{$ext}, $baseMap{$ext}, + "and leaves [$ext] alone, so its tokenizations are not invalidated"); +} + +# --------------------------------------------------------------------------- +for my $component (qw(srcml2token libsrcml ctags)) { + my $d = fixture(); + my ($before) = identity($d); + if ($component eq "srcml2token") { write_srcml2token($d, "v2") } + else { write_file("$d/$component", "$component v2\n") } + my ($after) = identity($d); + my %b = as_map($before); + my %a = as_map($after); + isnt($a{c}, $b{c}, "changing $component moves the .c identity"); + is($a{rs}, $b{rs}, "changing $component leaves .rs alone"); +} + +# CregitSrcMl.pm is tokenizeSrcMl.pl's copy in the tokenizer worker. +for my $script (qw(tokenizeSrcMl.pl CregitSrcMl.pm)) { + my $d = fixture(); + my ($before) = identity($d); + write_file("$d/tokenize/$script", "# v2\n"); + my ($after) = identity($d); + my %b = as_map($before); + my %a = as_map($after); + isnt($a{c}, $b{c}, "editing $script moves the .c identity"); + is($a{rs}, $b{rs}, "and leaves .rs alone"); +} + +# --------------------------------------------------------------------------- +# Deliberately every extension: either file can change tokens with no parser +# change (tokenize.pl picks parser and flags, CregitLanguages.pm the language). +for my $shared (qw(tokenize.pl CregitLanguages.pm)) { + my $d = fixture(); + my ($before) = identity($d); + my %b = as_map($before); + open(my $fh, '>>', "$d/tokenize/$shared") or die $!; + print $fh "\n# a change\n"; + close $fh; + my ($after) = identity($d); + my %a = as_map($after); + for my $ext (sort keys %b) { + isnt($a{$ext}, $b{$ext}, "editing $shared moves [$ext] too"); + } +} + +# --------------------------------------------------------------------------- +{ + my $d = fixture(); + unlink("$d/tokenize/rustTokenizer/target/release/rust_tokenizer"); + my @cmd = ("perl", "$d/tokenize/tokenizerIdentity.pl", + "--srcml2token=$d/srcml2token", "--ctags=$d/ctags"); + my $err = `@cmd 2>&1 1>/dev/null`; + isnt($?, 0, "an unbuilt Rust tokenizer is fatal, not omitted"); + like($err, qr/rust_tokenizer/, "and the message names the file"); +} + +{ + my $d = fixture(); + my @cmd = ("perl", "$d/tokenize/tokenizerIdentity.pl", "--ctags=$d/ctags"); + my $err = `@cmd 2>&1 1>/dev/null`; + isnt($?, 0, "omitting --srcml2token is fatal: .c tokens depend on it"); + like($err, qr/--srcml2token/, "and the message names the flag"); +} + +{ + my $d = fixture(); + write_file("$d/srcml2token", "#!/bin/sh\nexit 0\n"); + my ($out, $status) = identity($d); + isnt($status, 0, "an srcml2token that does not name its libsrcml is fatal"); + is($out, "", "and prints no identity"); +} + +# --------------------------------------------------------------------------- +{ + my ($all) = identity($dir, "--all-extensions"); + my %allMap = as_map($all); + is_deeply([sort keys %allMap], [CregitLanguages::extensions_sorted()], + "--all-extensions covers every extension in the table"); + ok(exists $allMap{am}, "including .am, which is routed to m4 but not masked"); +} + +done_testing(); diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index f29377be0..d91861e07 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -11,14 +11,16 @@ import json import sqlite3 +from collections import Counter import pytest from generate_dataset import (DEFAULT_MEMORY_LIMIT, PROJECT_META_FIELDS, SourceReader, is_ws, load_project_meta, - parse_memory_limit, project_meta_sql, + parse_memory_limit, process_blame_file, + project_meta_sql, repair_token_line, skip_comment, skip_literal, skip_token, - sql_literal) + sql_literal, undo_mojibake) def reader(text: str) -> SourceReader: @@ -450,45 +452,208 @@ def test_an_unknown_project_key_stops_the_run_before_phase_1(monkeypatch, tmp_pa # --------------------------------------------------------------------------- # # footer trailers resolve to personids +# --------------------------------------------------------------------------- # -def seed_trailer(value, *, email_rows=()): - """Give the tiny project one Reviewed-by trailer and the emails rows given.""" +def seed_footers(*trailers, people=()): + """Return a `seed` for generate(): trailers into footers, people into + emails/persons. `people` is (personid, emailaddr, personname).""" def seed(cregit_db, persons_db): con = sqlite3.connect(cregit_db) - con.execute("INSERT INTO footers VALUES (?,?,?,?)", - (SHA, 0, "Reviewed-by", value)) + for idx, (key, value) in enumerate(trailers): + con.execute("INSERT INTO footers VALUES (?,?,?,?)", + (SHA, idx, key, value)) con.commit() con.close() - con = sqlite3.connect(persons_db) - for personid, emailaddr, emailname in email_rows: - con.execute( - "INSERT INTO emails (personid, fullemail, emailaddr, emailname, " - "lcemail, userid, domain) VALUES (?,?,?,?,?,?,?)", - (personid, f"{emailname} <{emailaddr}>", emailaddr, emailname, - emailaddr.lower(), personid, emailaddr.partition("@")[2])) - con.execute("INSERT INTO persons VALUES (?,?)", (personid, emailname)) + for i, (pid, addr, pname) in enumerate(people, start=1): + con.execute("INSERT INTO emails (recordid, personid, fullemail, " + "emailaddr, emailname) VALUES (?,?,?,?,?)", + (i, pid, f"{pname} <{addr}>", addr, pname)) + con.execute("INSERT OR IGNORE INTO persons VALUES (?,?)", (pid, pname)) con.commit() con.close() return seed -def footer_personids(duckdb, out): - return duckdb.sql("select footer_personids from read_parquet(?)", - params=[str(out)]).fetchall()[0][0] +def footers_of(duckdb, path): + return duckdb.sql("select footer_personids, footer_person_names, " + f"footer_reviewed_by from read_parquet('{path}')").fetchone() -def test_a_trailer_carrying_a_bare_address_resolves_to_its_person(monkeypatch, tmp_path): +@pytest.mark.parametrize("value", ["bob@example.com", "Bob bob@example.com"]) +def test_a_bare_address_trailer_resolves_to_a_personid(monkeypatch, tmp_path, value): duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path, seed=seed_trailer( - "Bob bob@example.com", email_rows=[("p_bob", "bob@example.com", "Bob")])) - assert footer_personids(duckdb, out) == ["p_bob"] + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", value), + people=[("bob", "bob@example.com", "Bob B")])) + + ids, names, reviewed = footers_of(duckdb, out) + assert list(reviewed) == [value] + assert list(ids) == ["bob"] + assert list(names) == ["Bob B"] + + +def test_the_angle_bracket_form_still_wins_over_a_second_address(monkeypatch, tmp_path): + """The nullif must not change precedence: the bracketed form wins.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", "noreply@example.com writing for Bob B "), + people=[("bob", "bob@example.com", "Bob B"), + ("noreply", "noreply@example.com", "No Reply")])) + + ids, _names, _reviewed = footers_of(duckdb, out) + assert list(ids) == ["bob"] -def test_a_trailer_with_no_address_does_not_match_an_empty_emails_row(monkeypatch, tmp_path): +def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): + """Joining on '' would match the emails row of an author committing as `Name <>`.""" duckdb = pytest.importorskip("duckdb") - out = generate(monkeypatch, tmp_path, seed=seed_trailer( - "Former-commit-id: 0123456789abcdef0123456789abcdef01234567", - email_rows=[("p_empty", "", "Nameless")])) - assert footer_personids(duckdb, out) == [] + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Former-commit-id", "6a32a91a877bc2341810ef674dfdc3be21c500bc"), + people=[("ghost", "", "A Ghost")])) + + ids, names, _reviewed = footers_of(duckdb, out) + assert list(ids) == [] + assert list(names) == [] + + +# --------------------------------------------------------------------------- # +# Token-line repairs before the source walk: srcML mojibake, the Rust position +# prefix and the BOM. Each test checks text and position after the defect. +# --------------------------------------------------------------------------- # + +def walk(tmp_path, name, source: bytes, *token_lines, stats=None): + """Run process_blame_file on one file and return its token_map rows.""" + (tmp_path / name).write_bytes(source) + blame = tmp_path / f"{name}.blame" + blame.write_text("".join(f"{SHA};;\t{t}\n" for t in token_lines), + encoding="utf-8") + con = sqlite3.connect(":memory:") + con.execute("""CREATE TABLE token_map (file_path TEXT, token_index INTEGER, + commit_sha CHAR(40), token_type TEXT, token_value TEXT, + source_text TEXT, source_line INTEGER, source_col INTEGER, + is_structural INTEGER, func_name TEXT)""") + process_blame_file(blame, tmp_path / name, name, con.cursor(), stats) + return con.execute("""SELECT token_type, token_value, source_text, + source_line, source_col, is_structural FROM token_map + ORDER BY token_index""").fetchall() + + +C_SOURCE = "/* Högskolan */\nint x;\n".encode() +C_MOJIBAKE = ("comment|/* Högskolan */", "keyword|int", "name|x", + "operator|;") + + +def test_undo_mojibake_restores_the_utf8_text(): + assert undo_mojibake("Högskolan") == "Högskolan" + assert undo_mojibake("â\x80\x94") == "\u2014" + + +@pytest.mark.parametrize("text", [ + "Högskolan", # correct text: o-umlaut is above the lead-byte range + "é¿", # a lead byte with too few continuation bytes + "Āö", # a character above U+00FF cannot be Latin-1 + "plain ascii", +]) +def test_undo_mojibake_leaves_other_text_alone(text): + assert undo_mojibake(text) == text + + +def test_a_mojibake_token_gets_the_true_text_and_the_walk_stays_aligned(tmp_path): + rows = walk(tmp_path, "a.c", C_SOURCE, *C_MOJIBAKE) + assert rows[0][1] == "/* Högskolan */" + assert rows[0][2].startswith("/* Högskolan */") + assert rows[1][1:5] == ("int", "int ", 2, 1) + assert rows[2][3:5] == (2, 5) + + +def test_a_correct_token_is_not_changed(tmp_path): + rows = walk(tmp_path, "a.c", C_SOURCE, "comment|/* Högskolan */", + *C_MOJIBAKE[1:]) + assert rows[0][1] == "/* Högskolan */" + assert rows[1][3:5] == (2, 1) + + +@pytest.mark.parametrize("name, source", [ + # srcML decodes a BOM file correctly, so the pair is the true text. + ("a.c", b"\xef\xbb\xbf/* H\xc3\x83\xc2\xb6 */\nint x;\n"), + # Not UTF-8: nothing tells what the true text is. + ("a.c", b"/* H\xc3\xb6 \xff */\nint x;\n"), + # The Rust tokenizer does not use srcML. + ("a.rs", "/* Hö */\nint x;\n".encode()), +]) +def test_the_gate_keeps_a_matching_token_as_it_is(tmp_path, name, source): + rows = walk(tmp_path, name, source, "comment|/* Hö */", "keyword|int") + assert rows[0][1] == "/* Hö */" + + +def test_a_rust_line_loses_its_position_prefix(tmp_path): + rows = walk(tmp_path, "m.rs", b"fn main() {}\n", + "-:-\tbegin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", + "1:1\tkeyword|fn", "1:4\tidentifier|main", "1:8\top|(", + "1:9\top|)", "1:11\top|{", "1:12\top|}", "-:-\tend_unit") + assert [r[0] for r in rows] == ["begin_unit", "keyword", "identifier", + "op", "op", "op", "op", "end_unit"] + assert rows[0][5] == 1 and rows[-1][5] == 1 + assert [r[3:5] for r in rows[1:7]] == [(1, 1), (1, 4), (1, 8), (1, 9), + (1, 11), (1, 12)] + assert rows[2][2] == "main" + + +def test_a_decl_line_prefix_is_removed_too(tmp_path): + rows = walk(tmp_path, "m.rs", b"fn f\n", "1:-\tDECL|fn|f", + "1:1\tkeyword|fn") + assert rows[0][0] == "DECL" + assert rows[1][3:5] == (1, 1) + + +def test_a_c_line_that_starts_with_digits_is_kept(tmp_path): + rows = walk(tmp_path, "a.c", b"1\n", "literal|1") + assert rows[0][:2] == ("literal", "1") + + +def test_a_byte_order_mark_does_not_move_the_walk(tmp_path): + rows = walk(tmp_path, "a.c", b"\xef\xbb\xbfint x;\n", "keyword|int", + "name|x", "operator|;") + assert rows[0][2:5] == ("int ", 1, 1) + assert rows[1][2:5] == ("x", 1, 5) + + +def test_windows_line_ends_still_read_as_one_newline(tmp_path): + rows = walk(tmp_path, "a.c", b"int\r\nx;\r\n", "keyword|int", "name|x") + assert rows[1][3:5] == (2, 1) + + +def test_the_repairs_are_counted(tmp_path): + stats = Counter() + walk(tmp_path, "a.c", C_SOURCE, *C_MOJIBAKE, stats=stats) + walk(tmp_path, "m.rs", b"fn\n", "1:1\tkeyword|fn", "-:-\tend_unit", + stats=stats) + walk(tmp_path, "b.c", b"\xef\xbb\xbfint\n", "keyword|int", stats=stats) + assert stats == Counter(mojibake_tokens=1, position_prefix=2, bom_files=1) + + +def test_a_line_from_the_fixed_rust_tokenizer_is_kept(tmp_path): + # The repair removes text; it never shifts a position by a fixed amount. + stats = Counter() + rows = walk(tmp_path, "m.rs", b"fn main\n", + "begin_unit|revision:0.0.1;language:Rust;cregit-version:0.0.1", + "keyword|fn", "identifier|main", "end_unit", stats=stats) + assert [r[0] for r in rows] == ["begin_unit", "keyword", "identifier", + "end_unit"] + assert [r[3:5] for r in rows[1:3]] == [(1, 1), (1, 4)] + assert stats["position_prefix"] == 0 + + +@pytest.mark.parametrize("line, repairable, expected", [ + ("12:5\tkeyword|fn", False, "keyword|fn"), + ("12:-\tDECL|fn|f", False, "DECL|fn|f"), + ("-:-\tend_unit", False, "end_unit"), + ("keyword|fn", False, "keyword|fn"), + ("comment|Hö", True, "comment|Hö"), + ("comment|Hö", False, "comment|Hö"), + ("12:5|keyword|fn", False, "12:5|keyword|fn"), # --position form: a pipe +]) +def test_repair_token_line(line, repairable, expected): + assert repair_token_line(line, repairable, Counter()) == expected diff --git a/tests/test_token_worker.sh b/tests/test_token_worker.sh new file mode 100755 index 000000000..d01f46397 --- /dev/null +++ b/tests/test_token_worker.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Run inside `devenv shell`: it needs perl, python3, srcml, ctags and cargo. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +SRCML2TOKEN="$ROOT/tokenize/srcMLtoken/srcml2token" +[[ -x "$SRCML2TOKEN" ]] || make -C "$ROOT/tokenize/srcMLtoken" +[[ -x "$ROOT/tokenize/rustTokenizer/target/release/rust_tokenizer" ]] || make -C "$ROOT/tokenize/rustTokenizer" + +TEMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/cregit-token-worker.XXXXXX") +trap 'rm -rf "$TEMP_ROOT"' EXIT + +FIXTURES=("$ROOT"/tests/fixtures/* "$ROOT/tests/t/fixtures/srcml-position-crash.c") +python3 "$ROOT/tests/token_worker_driver.py" \ + --worker "$ROOT/tokenizeByBlobId/tokenWorker.pl" \ + --token-by-sha "$ROOT/tokenizeByBlobId/tokenBySha.pl" \ + --tokenize-command "$ROOT/tokenize/tokenize.pl --srcml2token=$SRCML2TOKEN --ctags=$(command -v ctags)" \ + "${FIXTURES[@]/#/--fixture=}" \ + --temp-root "$TEMP_ROOT" diff --git a/tests/token_worker_driver.py b/tests/token_worker_driver.py new file mode 100755 index 000000000..a7b94f943 --- /dev/null +++ b/tests/token_worker_driver.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Checks tokenizeByBlobId/tokenWorker.pl against tokenizeByBlobId/tokenBySha.pl.""" + +import argparse +import hashlib +import os +import pathlib +import shutil +import subprocess +import time + +TIMEOUT_EXIT = 124 +PARSER_CRASH_EXIT = 33 + + +class Worker: + def __init__(self, executable, memo_dir, tokenize_command): + env = dict(os.environ, BFG_MEMO_DIR=str(memo_dir), BFG_TOKENIZE_CMD=tokenize_command) + self.process = subprocess.Popen( + [str(executable)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + ready = self.process.stdout.readline() + if ready != b"READY\n": + raise AssertionError(f"worker did not start: {ready!r}") + + def request(self, filename, contents, timeout_secs=20): + name = os.fsencode(filename) + header = f"REQ {len(contents)} {len(name)} {timeout_secs}\n".encode() + self.process.stdin.write(header + name + contents) + self.process.stdin.flush() + response = self.process.stdout.readline().decode().split() + if len(response) != 4 or response[0] != "RES": + raise AssertionError(f"malformed response header: {response!r}") + exit_code, out_len, err_len = map(int, response[1:]) + return exit_code, self.process.stdout.read(out_len), self.process.stdout.read(err_len) + + def close(self): + self.process.stdin.close() + if self.process.wait(timeout=5) != 0: + raise AssertionError(f"worker exited {self.process.returncode} on EOF") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + print(f"PASS {message}") + + +def token_by_sha(args, memo_dir, fixture): + env = dict( + os.environ, + BFG_FILENAME=fixture.name, + BFG_MEMO_DIR=str(memo_dir), + BFG_TOKENIZE_CMD=args.tokenize_command, + ) + return subprocess.run( + [str(args.token_by_sha)], input=fixture.read_bytes(), capture_output=True, env=env + ) + + +def memo_file(memo_dir, contents): + digest = hashlib.sha1(contents).hexdigest() + return memo_dir / digest[:2] / digest[2:4] / digest + + +def new_dir(args, name): + path = args.temp_root / name + path.mkdir() + return path + + +def write_script(path, body): + path.write_text("#!/usr/bin/env bash\n" + body, encoding="utf-8") + path.chmod(0o755) + return path + + +def no_process_left(marker): + time.sleep(0.2) + return subprocess.run(["pgrep", "-f", marker], capture_output=True).returncode == 1 + + +def timed_request(worker, filename, contents): + started = time.monotonic() + response = worker.request(filename, contents, timeout_secs=1) + return response, time.monotonic() - started + + +def check_same_as_token_by_sha(args): + memo = new_dir(args, "worker-memo") + worker = Worker(args.worker, memo, args.tokenize_command) + for index, fixture in enumerate(args.fixture): + expected = token_by_sha(args, new_dir(args, f"reference-memo-{index}"), fixture) + actual = worker.request(fixture.name, fixture.read_bytes()) + check(actual[0] == expected.returncode, f"{fixture.name}: exit {expected.returncode}") + if expected.returncode == 0: + check(actual[1] == expected.stdout, f"{fixture.name}: same tokens") + check( + memo_file(memo, fixture.read_bytes()).read_bytes() == expected.stdout, + f"{fixture.name}: same memo file", + ) + + first = args.fixture[0] + expected = memo_file(memo, first.read_bytes()).read_bytes() + check(worker.request(first.name, first.read_bytes()) == (0, expected, b""), "a memo hit gives the same bytes") + unknown = worker.request("unknown.xyzzy", b"x") + check(unknown[0] == 255 and b"unknown file extension" in unknown[2], "an unknown extension fails") + check(worker.request(first.name, first.read_bytes())[0] == 0, "the worker serves the next request") + worker.close() + + +def check_timeout(args, marker): + memo = new_dir(args, "timeout-memo") + script = write_script(args.temp_root / "hang.sh", f"exec perl -e 'sleep 100' {marker}\n") + worker = Worker(args.worker, memo, str(script)) + (exit_code, out, _), elapsed = timed_request(worker, "hang.c", b"int x;\n") + check(exit_code == TIMEOUT_EXIT and out == b"" and elapsed < 3, f"a hung tokenizer exits {TIMEOUT_EXIT}") + check(no_process_left(marker), "the timeout kills the tokenizer's process group") + check(not any(memo.rglob("*")), "a timeout memoizes nothing") + worker.close() + + +def in_process_command(args, marker): + options = args.tokenize_command.split() + real = next(o.split("=", 1)[1] for o in options if o.startswith("--srcml2token=")) + srcml2token = write_script( + args.temp_root / "srcml2token.sh", + 'src="${@: -1}"\n' + f'grep -qs HANG_MARKER "$src" && exec perl -e "sleep 100" {marker}\n' + 'grep -qs ABORT_MARKER "$src" && kill -ABRT $$\n' + f'exec {real} "$@"\n', + ) + ctags = write_script( + args.temp_root / "ctags.sh", f'echo CTAGS_STDERR >&2\nexec {shutil.which("ctags")} "$@"\n' + ) + options = [o for o in options if not o.startswith(("--srcml2token=", "--ctags="))] + return " ".join(options + [f"--srcml2token={srcml2token}", f"--ctags={ctags}"]) + + +def check_in_process_failures(args, marker): + memo = new_dir(args, "in-process-memo") + worker = Worker(args.worker, memo, in_process_command(args, marker)) + (exit_code, _, _), elapsed = timed_request(worker, "hang.c", b"int HANG_MARKER;\n") + check(exit_code == TIMEOUT_EXIT and elapsed < 3, f"in process, a hung srcml2token exits {TIMEOUT_EXIT}") + check(no_process_left(marker), "and is killed") + + exit_code, out, err = worker.request("abort.c", b"int ABORT_MARKER;\n") + check(exit_code == PARSER_CRASH_EXIT and out == b"", f"in process, a srcml2token signal death exits {PARSER_CRASH_EXIT}") + check(b"killed by signal 6" in err and b"CTAGS_STDERR" in err, "with the srcml2token and ctags stderr") + check(not any(memo.rglob("*")), "and nothing is memoized") + + fixture = args.fixture[0] + expected = token_by_sha(args, new_dir(args, "in-process-reference"), fixture).stdout + check(worker.request(fixture.name, fixture.read_bytes())[1] == expected, "the worker recovers") + worker.close() + + +def check_stale_srcml2token(args): + stale = write_script(args.temp_root / "srcml2token", "exit 0\n") + command = f"{args.tokenize_command} --srcml2token={stale}" + env = dict(os.environ, BFG_MEMO_DIR=str(new_dir(args, "stale-memo")), BFG_TOKENIZE_CMD=command) + started = subprocess.run([str(args.worker)], input=b"", capture_output=True, env=env) + check(started.returncode != 0 and b"READY" not in started.stdout, "a srcml2token without --libsrcml-path stops the start") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--worker", required=True, type=pathlib.Path) + parser.add_argument("--token-by-sha", required=True, type=pathlib.Path) + parser.add_argument("--tokenize-command", required=True) + parser.add_argument("--fixture", action="append", required=True, type=pathlib.Path) + parser.add_argument("--temp-root", required=True, type=pathlib.Path) + args = parser.parse_args() + + marker = f"cregit-token-worker-{os.getpid()}" + check_same_as_token_by_sha(args) + check_timeout(args, marker) + check_in_process_failures(args, marker) + check_stale_srcml2token(args) + print("ALL TOKEN WORKER TESTS PASSED") + + +if __name__ == "__main__": + main() diff --git a/tests/tokenize_gate.sh b/tests/tokenize_gate.sh index d6ab714fa..2a61ec119 100755 --- a/tests/tokenize_gate.sh +++ b/tests/tokenize_gate.sh @@ -203,11 +203,12 @@ OUT=$(STUB_RC=0 run_step2 "$W" --stall-timeout abc); RC=$? rm -rf "$W" echo "case 10: the runner exits with blobExec's own status, not a flat 1" -for rc in 4 5; do +for rc in 4 5 6; do W=$(fixture) STUB_RC=$rc run_step2 "$W" > /dev/null 2>&1 GOT=$? [ "$GOT" -eq "$rc" ]; check "blobExec $rc makes the runner exit $rc (got $GOT)" $? + [ -n "$(ls "$W"/TOKENIZE-* 2>/dev/null)" ]; check "and the marker is still written" $? rm -rf "$W" done W=$(fixture) diff --git a/tokenize/CregitLanguages.pm b/tokenize/CregitLanguages.pm index b142e5cd8..6d85fd04e 100644 --- a/tokenize/CregitLanguages.pm +++ b/tokenize/CregitLanguages.pm @@ -13,16 +13,45 @@ package CregitLanguages; # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# The only extension table in cregit. tokenize.pl, tokenizeSrcMl.pl, -# tokenizeByBlobId/tokenBySha.pl and fileMask.pl all derive from it. +# THE single source of truth for "which files can cregit tokenize". +# +# Before this module the same knowledge was written out three times and the +# copies did not agree: +# +# tokenize/tokenize.pl %extensions — the dispatcher's gate +# tokenize/tokenizeSrcMl.pl %extensions — the srcML parser's autodetect +# tokenizeByBlobId/tokenBySha.pl %mapLang — the per-blob gate bfg calls +# +# tokenBySha.pl mapped `go`, `md` and `yaml`; tokenize.pl had no parser for any +# of them. So such a blob passed the first gate and died at the second with +# "Unknown parser for extension" — a whole project's run lost to an +# inconsistency between two tables that describe the same capability. They are +# now one table, and t/languages.t holds them to it. +# +# The file mask the pipeline selects blobs with is DERIVED from this table +# (file_mask below), never typed out again: a mask naming an extension the table +# does not know is a run that dies mid-way, and an extension in the table that +# no mask names is source silently left untokenized. use strict; use warnings; -# Lowercase extension, no leading dot -> language, spelled as --language takes it. +# Extension (lowercase, no leading dot) -> language name. The language names are +# the ones %LANG_PARSER_REL keys on and the ones passed as --language. +# +# Verified against the pinned srcML (1.1.0) on 2026-09-19 by tokenizing the same +# C++ fixture under each extension and comparing the token stream to .cpp's: +# .hxx and .tcc are byte-identical to .cpp. `.ixx .inl .cppm .cxxm .ipp` were +# candidates and are NOT here: srcml 1.1.0 does not know those extensions and +# then ignores `-l C++`, emitting an XML declaration with no at all. It +# exits 0 doing so, so the chain produces an EMPTY token file and reports +# success. That is why t/languages.t probes srcML per extension instead of +# trusting a reference page. our %EXT_LANG = ( + # C 'c' => 'C', 'h' => 'C', + # C++ 'c++' => 'C++', 'cc' => 'C++', 'cp' => 'C++', @@ -33,13 +62,41 @@ our %EXT_LANG = ( 'hpp' => 'C++', 'hxx' => 'C++', 'tcc' => 'C++', + # Java 'java' => 'Java', + # M4 (autoconf/automake input). Routed, but NOT in the mask: see + # %MASKED_LANGUAGES for the measurement that keeps it out. 'ac' => 'M4', 'am' => 'M4', + # Rust 'rs' => 'Rust', ); -# M4 is missing on purpose: m4.py raises StopIteration on real autotools quoting. +# Languages the pipeline's file mask is allowed to select, i.e. the ones whose +# parser is known to produce correct tokens for real-world input. +# +# M4 is deliberately ABSENT, on measurement. m4Tokenizer/m4.py was Python 2 and +# did not run at all under the pinned Python 3 (SyntaxError on `print`), so the +# "the parser exists and is executable" check that cleared M4 for inclusion was +# testing the wrong thing. It has since been ported, and it still must not be in +# the mask, because its lexer is wrong on real autotools input: +# +# Lexer::__init__ sets end_quote to a BACKTICK, not the apostrophe that m4 +# uses and that this file's own changequote() defaults to. So `` `x' `` runs +# the string on to the next backtick and swallows whatever is between -- +# measured on a two-line fixture, an AC_MSG_WARN macro call disappeared inside +# a string token. Setting end_quote to an apostrophe is NOT the fix: shell +# backticks in Makefile.am then never close, and the sample below goes from +# 36/38 files parsed to 12/38. +# +# Measured 2026-09-19 over 38 real configure.ac/configure.in/Makefile.am files +# on this machine: 36 parse, 2 die with "generator raised StopIteration" (an +# unterminated string at EOF; under Python 2 the same condition silently +# truncated the token stream and exited 0). +# +# Selecting .am/.ac would therefore fail whole projects at step 2, for 1.4 MB of +# source in 22 projects. Fixing the m4 lexer is its own task with its own +# fixtures; when it is done, add 'M4' here and the mask widens by itself. our %MASKED_LANGUAGES = ( 'C' => 1, 'C++' => 1, @@ -47,6 +104,8 @@ our %MASKED_LANGUAGES = ( 'Rust' => 1, ); +# Language -> parser, relative to the tokenize/ directory. Every value must +# exist and be executable in a built checkout; t/languages.t asserts it. our %LANG_PARSER_REL = ( 'C' => 'tokenizeSrcMl.pl', 'C++' => 'tokenizeSrcMl.pl', @@ -55,40 +114,51 @@ our %LANG_PARSER_REL = ( 'Rust' => 'rustTokenizer/target/release/rust_tokenizer', ); +# The parser every srcML-routed language shares. Named so callers can ask "is +# this extension srcML's?" without string-matching a path. our $SRCML_PARSER_REL = 'tokenizeSrcMl.pl'; -# An uppercase .C or .H is C++ by the GNU convention. Every other extension is -# case-insensitive. -our %CASE_SENSITIVE_EXT = ( 'C' => 'C++', 'H' => 'C++' ); - -# $ext arrives without its dot and with its case intact. -sub language_for_ext { - my ($ext) = @_; - return undef unless defined $ext && $ext ne ''; - return $CASE_SENSITIVE_EXT{$ext} if exists $CASE_SENSITIVE_EXT{$ext}; - return $EXT_LANG{ lc($ext) }; +# Extensions keyed WITH the leading dot, as tokenize.pl and tokenizeSrcMl.pl +# look them up (both lowercase the extension first). +sub extensions_by_dot { + return map { ( ".$_" => $EXT_LANG{$_} ) } keys %EXT_LANG; } +# The same, restricted to the languages srcML parses. tokenizeSrcMl.pl must not +# autodetect a .rs file as Rust: it would hand `-l Rust` to srcml. sub srcml_extensions_by_dot { - return map { ( ".$_" => $EXT_LANG{$_} ) } + return map { ( ".$_" => $EXT_LANG{$_} ) } grep { $LANG_PARSER_REL{ $EXT_LANG{$_} } eq $SRCML_PARSER_REL } keys %EXT_LANG; } +# Language -> absolute parser path, given the tokenize/ directory. sub parsers { my ($basedir) = @_; - die "parsers() needs the tokenize/ directory" unless defined $basedir and $basedir ne ''; + die "parsers() needs the tokenize/ directory" unless defined $basedir and $basedir ne ""; return map { ( $_ => "$basedir/$LANG_PARSER_REL{$_}" ) } keys %LANG_PARSER_REL; } -sub extensions_sorted { return sort keys %EXT_LANG } +# Extensions in a stable order, so the derived mask is byte-stable: it is stored +# in blobExec's meta table and compared string-for-string on every resume. +sub extensions_sorted { + return sort keys %EXT_LANG; +} +# The extensions the mask may name: every extension of every masked language. sub masked_extensions_sorted { return grep { $MASKED_LANGUAGES{ $EXT_LANG{$_} } } extensions_sorted(); } -# Sorted, because blobExec stores this mask and compares it on resume. Anchored -# at the end only: every consumer searches a path, not a bare basename. +# The universal file mask: every extension of every masked language, and nothing +# else. +# +# Case-insensitive, because `.C` and `.H` are real files and the per-language +# masks this replaces missed them. Anchored at the end only: every consumer +# matches it against a repository-relative path, not just a basename, and all +# four regex engines it reaches (Scala/Java in blobExec, Perl in blameRepoFiles +# and prettyPrintFiles, and whatever later reads the recorded string) search +# rather than full-match. sub file_mask { return '(?i)\.(' . join('|', map { quotemeta } masked_extensions_sorted()) . ')$'; } diff --git a/tokenize/CregitSrcMl.pm b/tokenize/CregitSrcMl.pm new file mode 100644 index 000000000..0b61382d7 --- /dev/null +++ b/tokenize/CregitSrcMl.pm @@ -0,0 +1,225 @@ +package CregitSrcMl; + +# tokenizeSrcMl.pl in a persistent process, with one ctags per language. Its +# output must stay byte-identical to tokenizeSrcMl.pl (tests/test_token_worker.sh). + +use strict; +use warnings; +use bytes; +use File::Temp (); +use POSIX qw(_exit); + +our $PARSER_CRASH_EXIT = 33; + +my $EMPTY_STREAM = "the token stream was empty. A successful srcML parse always emits at " + . "least the begin_unit/end_unit wrapper, even for an empty file, so an " + . "empty stream is a failed parse and never a legitimately empty result."; + +sub new { + my ($class, %options) = @_; + my $self = bless { ctags => "ctags-universal", position => 0, %options, + ctags_procs => {}, retired_errors => [] }, $class; + open(my $probe, '-|', $self->{srcml2token}, "--libsrcml-path") + or die "unable to execute srcml2token [$self->{srcml2token}]: $!\n"; + my $library = <$probe> // ""; + close($probe); + die "[$self->{srcml2token}] does not answer --libsrcml-path; rebuild it: make -C tokenize/srcMLtoken\n" + if $? != 0 or $library eq ""; + return $self; +} + +sub tokenize { + my ($self, $language, $inputName, $workDir, $timeoutSecs) = @_; + my $timedOut = 0; + local $SIG{ALRM} = sub { $timedOut = 1; $self->kill_all() }; + alarm($timeoutSecs); + my @result = eval { $self->run($language, $inputName, $workDir) }; + my $error = $@; + alarm(0); + my $helperErr = $self->take_helper_stderr(); + + if ($timedOut) { + $self->shutdown(); + return (124, "", $helperErr . "tokenize command timed out after [$timeoutSecs] seconds\n"); + } + return (255, "", $helperErr . $error) if not @result; + $result[2] = $helperErr . $result[2]; + return @result; +} + +sub run { + my ($self, $language, $inputName, $workDir) = @_; + my $declarations = $self->read_declarations($language, $inputName, "$workDir/$inputName"); + my ($status, $tokens, $stderr) = $self->run_srcml2token($language, $inputName, $workDir); + return parser_crash($inputName, $stderr, srcml2token_failure($status)) if $status != 0; + return parser_crash($inputName, $stderr, $EMPTY_STREAM) if not @$tokens; + return (0, merge($tokens, $declarations, $self->{position}), $stderr); +} + +sub srcml2token_failure { + my ($status) = @_; + my $signal = $status & 127; + return "srcml2token exited " . ($status >> 8) . "." if not $signal; + return "srcml2token was killed by signal $signal. srcML 1.1.0, which it links, " + . "dies on a signal on some C/C++ inputs under --position."; +} + +sub parser_crash { + my ($filename, $stderr, $why) = @_; + return ($PARSER_CRASH_EXIT, "", $stderr + . "cregit: tokenization of [$filename] FAILED: $why\n" + . "cregit: refusing to emit a tokenization for [$filename]; " + . "exiting $PARSER_CRASH_EXIT so the caller can count this blob.\n"); +} + +sub merge { + my ($tokens, $declarations, $position) = @_; + my $lastLine = -1; + my $output = ""; + for my $tokenLine (@$tokens) { + die "unable to parse srcml line [$tokenLine]\n" unless $tokenLine =~ /^([0-9]+|-):([0-9]+|-)\s+(.+)$/; + my ($line, $col, $token) = ($1, $2, $3); + my $lineNumber = $line eq '-' ? 0 : $line; + $output .= declaration_lines($declarations->{$line}, $line, $position) if $lineNumber != $lastLine; + $lastLine = $lineNumber; + $output .= "$line:$col|" if $position; + $output .= "$token\n"; + $output .= ($position ? "-:-|" : "") . "\n" if $token =~ /^end_/; + } + return $output; +} + +# Like tokenizeSrcMl.pl, a name that is twice on one line prints the type of its +# last declaration twice. +sub declaration_lines { + my ($declarations, $line, $position) = @_; + return "" if not $declarations; + return join "", map { ($position ? "$line:-|" : "") . "DECL|$declarations->{types}{$_}|$_\n" } + @{ $declarations->{names} }; +} + +sub read_declarations { + my ($self, $language, $inputName, $path) = @_; + open(my $in, '<:raw', $path) or die "unable to read [$path]: $!\n"; + my $source = do { local $/; <$in> } // ""; + my $ctags = $self->ctags_for($language); + print {$ctags->{in}} qq({"command":"generate-tags","filename":"$inputName","size":) + . length($source) . "}\n", $source; + $ctags->{in}->flush; + + my %declarations; + while (defined(my $tag = readline($ctags->{out}))) { + return \%declarations if $tag =~ /\A\{"_type": "completed"/; + chomp $tag; + if ($tag !~ /^([0-9]+) (.+) @@@ (.*) @@@ (.*)$/) { + $self->retire(delete $self->{ctags_procs}{$language}); + die "unable to parse output [$tag]\n"; + } + my ($line, $name, $type, $signature) = ($1, $2, $3, $4); + $name .= " $signature" if $signature ne "-"; + $declarations{$line}{types}{$name} = $type; + push @{ $declarations{$line}{names} }, $name; + } + $self->retire(delete $self->{ctags_procs}{$language}); + return \%declarations; +} + +# The same -x format as tokenizeSrcMl.pl, so that the names are byte-identical: +# ctags --_interactive drops a JSON field that is not valid UTF-8. +sub ctags_for { + my ($self, $language) = @_; + return $self->{ctags_procs}{$language} //= do { + my $proc = spawn_pipe($self->{ctags}, "--_interactive", "--language-force=$language", "-x", + "--sort=n", "--_xformat=%n %N @@@ %K @@@ %S"); + my $hello = readline($proc->{out}) // "EOF"; + die "ctags did not start: $hello\n" unless $hello =~ /"_type": "program"/; + $proc; + }; +} + +sub run_srcml2token { + my ($self, $language, $inputName, $workDir) = @_; + my $out = File::Temp->new(TEMPLATE => "cregit-srcml2token-out-XXXXX", TMPDIR => 1); + my $err = File::Temp->new(TEMPLATE => "cregit-srcml2token-err-XXXXX", TMPDIR => 1); + my $pid = fork() // die "unable to fork srcml2token: $!\n"; + if ($pid == 0) { + setpgrp(0, 0); + chdir($workDir) and open(STDIN, '<', '/dev/null') and open(STDOUT, '>', $out->filename) + and open(STDERR, '>', $err->filename) or _exit(255); + { no warnings 'exec'; exec $self->{srcml2token}, "-l", $language, $inputName; } + print STDERR "unable to execute srcml2token [$self->{srcml2token}]: $!\n"; + _exit(255); + } + $self->{parser_pid} = $pid; + waitpid($pid, 0); + my $status = $?; + delete $self->{parser_pid}; + seek($out, 0, 0); + chomp(my @tokens = <$out>); + return ($status, \@tokens, read_all($err)); +} + +# A helper writes its stderr to a file: a full stderr pipe would block it while +# this process waits for its stdout. +sub spawn_pipe { + my (@command) = @_; + pipe(my $childIn, my $parentOut) or die "unable to create pipe: $!\n"; + pipe(my $parentIn, my $childOut) or die "unable to create pipe: $!\n"; + my $err = File::Temp->new(TEMPLATE => "cregit-helper-err-XXXXX", TMPDIR => 1); + my $pid = fork() // die "unable to fork [$command[0]]: $!\n"; + if ($pid == 0) { + open(STDIN, '<&', $childIn) and open(STDOUT, '>&', $childOut) + and open(STDERR, '>>', $err->filename) or _exit(255); + { no warnings 'exec'; exec @command; } + print STDERR "unable to execute [$command[0]]: $!\n"; + _exit(255); + } + close($childIn); + close($childOut); + binmode $parentOut; + binmode $parentIn; + return { pid => $pid, in => $parentOut, out => $parentIn, err => $err }; +} + +sub take_helper_stderr { + my ($self) = @_; + my @live = grep { $_ } values(%{ $self->{ctags_procs} }); + my $text = join "", map { read_all($_) } @{ $self->{retired_errors} }, map { $_->{err} } @live; + $self->{retired_errors} = []; + truncate($_->{err}, 0) for @live; + return $text; +} + +sub read_all { + my ($fh) = @_; + seek($fh, 0, 0); + local $/; + return <$fh> // ""; +} + +sub retire { + my ($self, $proc) = @_; + return if not $proc; + push @{ $self->{retired_errors} }, $proc->{err}; + close($proc->{in}); + close($proc->{out}); + kill 'KILL', $proc->{pid}; + waitpid($proc->{pid}, 0); +} + +sub kill_all { + my ($self) = @_; + if (my $pid = $self->{parser_pid}) { + kill 'TERM', -$pid; + select(undef, undef, undef, 0.1); + kill 'KILL', -$pid; + } + kill 'KILL', map { $_->{pid} } grep { $_ } values(%{ $self->{ctags_procs} }); +} + +sub shutdown { + my ($self) = @_; + $self->retire(delete $self->{ctags_procs}{$_}) for keys %{ $self->{ctags_procs} }; +} + +1; diff --git a/tokenize/rustTokenizer/src/main.rs b/tokenize/rustTokenizer/src/main.rs index 67ee08f46..20f5c1246 100644 --- a/tokenize/rustTokenizer/src/main.rs +++ b/tokenize/rustTokenizer/src/main.rs @@ -21,7 +21,7 @@ static KEYWORDS: phf::Set<&'static str> = phf::phf_set! { }; fn main() { - let (path, emit_positions) = parse_args(); + let (path, position) = parse_args(); let src = match fs::read_to_string(&path) { Ok(s) => s, Err(e) => { @@ -29,15 +29,18 @@ fn main() { exit(1); } }; - print_pipe_separated_tokens(&src, emit_positions); + tokenize_source(&src, position); } fn parse_args() -> (String, bool) { let mut path: Option = None; - let mut emit_positions = false; + // Positions are OFF unless asked for, exactly as tokenizeSrcMl.pl gates them. This + // flag used to be swallowed and positions emitted unconditionally; see + // tokenize_source for why that corrupted every downstream row. + let mut position = false; for arg in env::args().skip(1) { if arg == "--position" { - emit_positions = true; + position = true; continue; } if arg.starts_with("--language=") || arg == "--verbose" { @@ -58,19 +61,38 @@ fn parse_args() -> (String, bool) { eprintln!("Usage: rust_tokenizer [--language=Rust] [--position] "); exit(2); }); - (path, emit_positions) + (path, position) } -fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { - let print_positionless_line = |body: &str| { - if emit_positions { +// Emits the cregit FINAL token format. That format is defined by tokenizeSrcMl.pl -- the +// only other tokenizer the pipeline actually routes to (CregitLanguages.pm +// %LANG_PARSER_REL: C/C++/Java -> tokenizeSrcMl.pl, Rust -> here; Go is unrouted and M4 +// is excluded from the file mask) -- and it is PIPE-separated with NO position prefix +// unless --position is given: +// +// tokenize/t/expected/main.c.nopos.token begin_unit|revision:... comment|/* ... */ +// tokenize/t/expected/main.c.token -:-|begin_unit|... 1:1|comment|/* ... */ +// +// Do NOT copy the TAB form in tokenize/srcMLtoken/tests/expected/*.token: that is +// srcml2token's INTERMEDIATE output, which tokenizeSrcMl.pl:120 consumes with +// /^([0-9]+|-):([0-9]+|-)\s+(.+)$/ and re-emits with `|`. It never reaches a consumer. +// +// The consumers both split on `|`: +// generate_dataset/generate_dataset.py:243 re.match(r"^(.+?)\|(.+)$", token_content) +// prettyPrint/prettyPrint-author.pl:976 split('\|', $value) +// The first group is NON-GREEDY, so any extra leading `line:col` field lands whole +// in token_type and shifts token_value, source_text and is_structural by one. +fn tokenize_source(src: &str, position: bool) { + // A line with no source position of its own: `-:-|` under --position, bare otherwise. + let marker = |body: &str| { + if position { println!("-:-|{}", body); } else { println!("{}", body); } }; - print_positionless_line(&format!( + marker(&format!( "begin_unit|revision:{};language:Rust;cregit-version:{}", REVISION, CREGIT_VERSION )); @@ -84,7 +106,7 @@ fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { let slice = &src[byte..end]; if let Some(out) = classify(&tok.kind, slice) { - if emit_positions { + if position { println!("{}:{}|{}", line, col, out); } else { println!("{}", out); @@ -103,8 +125,11 @@ fn print_pipe_separated_tokens(src: &str, emit_positions: bool) { byte = end; } - print_positionless_line("end_unit"); - print_positionless_line(""); + marker("end_unit"); + // tokenizeSrcMl.pl:143 prints a bare marker line after every `end_*` token, so the + // stream ends `end_unit` + blank (or `-:-|end_unit` + `-:-|`). generate_dataset.py + // classifies that blank as token_type `blank`, is_structural 1. + marker(""); } // None skips output (whitespace / Eof); the caller still advances the cursor. Every diff --git a/tokenize/srcMLtoken/Makefile b/tokenize/srcMLtoken/Makefile index 4458f33c0..9f34887bb 100644 --- a/tokenize/srcMLtoken/Makefile +++ b/tokenize/srcMLtoken/Makefile @@ -3,30 +3,30 @@ XERCES_CFLAGS ?= $(shell pkg-config --cflags xerces-c 2>/dev/null) XERCES_LIBS ?= $(shell pkg-config --libs xerces-c 2>/dev/null || echo -lxerces-c-3.2) +ifndef SRCML_PREFIX +$(error SRCML_PREFIX is not set: build inside `devenv shell`, or set it to the prefix of a srcML build with headers) +endif +SRCML_CFLAGS ?= -I$(SRCML_PREFIX)/include +SRCML_LIBS ?= -L$(SRCML_PREFIX)/lib -Wl,-rpath,$(SRCML_PREFIX)/lib -lsrcml + +FIXTURES = StringUtil.java:Java audit.c:C ceiling.cpp:C++ main.c:C srcml2tokenHandlers.cpp:C++ + default: srcml2token srcml2token: srcml2token.cpp srcml2tokenHandlers.cpp srcml2tokenHandlers.hpp srcml2token.hpp - g++ -g -std=c++11 -O -Wall -pedantic $(XERCES_CFLAGS) -o srcml2token srcml2token.cpp srcml2tokenHandlers.cpp $(XERCES_LIBS) + g++ -g -std=c++11 -O -Wall -pedantic $(XERCES_CFLAGS) $(SRCML_CFLAGS) -o srcml2token srcml2token.cpp srcml2tokenHandlers.cpp $(XERCES_LIBS) $(SRCML_LIBS) test: srcml2token mkdir -p tests/out - srcml --position --language Java tests/StringUtil.java | ./srcml2token > tests/out/StringUtil.java.token - diff tests/expected/StringUtil.java.token tests/out/StringUtil.java.token - srcml --position --language C tests/audit.c | ./srcml2token > tests/out/audit.c.token - diff tests/expected/audit.c.token tests/out/audit.c.token - srcml --position --language C++ tests/ceiling.cpp | ./srcml2token > tests/out/ceiling.cpp.token - diff tests/expected/ceiling.cpp.token tests/out/ceiling.cpp.token - srcml --position --language C tests/main.c | ./srcml2token > tests/out/main.c.token - diff tests/expected/main.c.token tests/out/main.c.token - srcml --position --language 'C++' tests/srcml2tokenHandlers.cpp | ./srcml2token > tests/out/srcml2tokenHandlers.cpp.token - diff tests/expected/srcml2tokenHandlers.cpp.token tests/out/srcml2tokenHandlers.cpp.token - + for f in $(FIXTURES); do \ + ./srcml2token -l "$${f#*:}" "tests/$${f%%:*}" > "tests/out/$${f%%:*}.token" \ + && diff "tests/expected/$${f%%:*}.token" "tests/out/$${f%%:*}.token" || exit 1; \ + done + test -f "$$(./srcml2token --libsrcml-path)" + printf 'int x;\n' > tests/out/unknown.xyz + ! ./srcml2token -l C tests/out/unknown.xyz 2> tests/out/unknown.err + grep -q 'does not know the extension' tests/out/unknown.err expected: srcml2token mkdir -p tests/expected - srcml --position --language 'C++' tests/srcml2tokenHandlers.cpp | ./srcml2token > tests/expected/srcml2tokenHandlers.cpp.token - srcml --position --language C tests/main.c | ./srcml2token > tests/expected/main.c.token - srcml --position --language 'C++' tests/ceiling.cpp | ./srcml2token > tests/expected/ceiling.cpp.token - srcml --position --language C tests/audit.c | ./srcml2token > tests/expected/audit.c.token - srcml --position --language Java tests/StringUtil.java | ./srcml2token > tests/expected/StringUtil.java.token - + for f in $(FIXTURES); do ./srcml2token -l "$${f#*:}" "tests/$${f%%:*}" > "tests/expected/$${f%%:*}.token"; done diff --git a/tokenize/srcMLtoken/README.md b/tokenize/srcMLtoken/README.md index 7146f42dc..1ea038fc6 100644 --- a/tokenize/srcMLtoken/README.md +++ b/tokenize/srcMLtoken/README.md @@ -1,31 +1,28 @@ # srcMLtoken -`srcMLtoken` transcodes the output of srcML ([www.srcml.org](www.srcml.org)) to a tokenized view that can be used by Cregit. - -## Requirements - -`srcml` must be installed and in the PATH ([www.srcml.org](www.srcml.org)) +`srcml2token` parses a C, C++ or Java file with libsrcml +([www.srcml.org](https://www.srcml.org)) and prints a tokenized view of its +srcML that Cregit uses. ## How to use -The input of `srml2token` is the output of `srcml`. `srml2token` takes no parameters, and it reads its input from _stdin_ and writes to _stdout_. - ```sh -srcml -l C --position | srcml2token +srcml2token -l C ``` +It parses the file as `srcml -l C --position ` does, and writes the +tokens to _stdout_. `srcml2token --libsrcml-path` prints the libsrcml that it +loads; `tokenize/tokenizerIdentity.pl` digests that file. + ## How to build -It requires `xerces` to be installed. Simply run: +It needs `xerces-c` and the libsrcml headers and library. Inside `devenv shell`, +`SRCML_PREFIX` names the srcML build of `nix/srcml.nix`. Run: ```sh -make +make && make test ``` -## TODO - -- Create a `cmake` configuration file. - ## License The code is mostly derived from the examples of Xerces, hence it is under the @@ -33,3 +30,4 @@ Apache-2.0 Dependencies: - `xerces`: Apache-2.0 + - `libsrcml`: GPL-3.0, so the linked `srcml2token` binary is under GPL-3.0 diff --git a/tokenize/srcMLtoken/srcml2token.cpp b/tokenize/srcMLtoken/srcml2token.cpp index a91f110ef..ca9d2951d 100644 --- a/tokenize/srcMLtoken/srcml2token.cpp +++ b/tokenize/srcMLtoken/srcml2token.cpp @@ -15,96 +15,106 @@ * limitations under the License. */ -// --------------------------------------------------------------------------- -// Includes -// --------------------------------------------------------------------------- -#include -#include -#include "srcml2token.hpp" +#include +#include +#include #include +#include "srcml2token.hpp" -// --------------------------------------------------------------------------- -// Local helper methods -// --------------------------------------------------------------------------- -void usage() +static int usage() { - XERCES_STD_QUALIFIER cout << "\nUsage:\n" - " srcml2token \n\n" - "This program converts the output of srcML into a simplified tokenized version\n" - << XERCES_STD_QUALIFIER endl; + XERCES_STD_QUALIFIER cerr << "Usage: srcml2token -l \n" + " srcml2token --libsrcml-path\n" + "Parses the file with libsrcml and prints its tokens.\n"; + return 2; } +static int fail(const std::string& why) +{ + XERCES_STD_QUALIFIER cerr << "srcml2token: " << why << XERCES_STD_QUALIFIER endl; + return 3; +} -// --------------------------------------------------------------------------- -// Program entry point -// --------------------------------------------------------------------------- -int main(int argC, char* argV[]) +// The tokenizer identity digests the library that is loaded, not the one on disk. +static int print_libsrcml_path() { - // Initialize the XML4C system - try - { - XMLPlatformUtils::Initialize(); - } + Dl_info info; + if (!dladdr(reinterpret_cast(&srcml_version_string), &info) || !info.dli_fname) + return fail("cannot find the loaded libsrcml"); + XERCES_STD_QUALIFIER cout << info.dli_fname << XERCES_STD_QUALIFIER endl; + return 0; +} - catch (const XMLException& toCatch) - { - XERCES_STD_QUALIFIER cerr << "Error during initialization! Message:\n" - << StrX(toCatch.getMessage()) << XERCES_STD_QUALIFIER endl; - return 1; - } +// The same calls as `srcml -l --position `. Like that command, +// refuse an extension that srcML does not know: it would ignore the language. +static int parse_source(const char* language, const char* filename, std::string& xml) +{ + srcml_archive* archive = srcml_archive_create(); + char* buffer = nullptr; + size_t size = 0; + int status = srcml_archive_write_open_memory(archive, &buffer, &size); + if (status == SRCML_STATUS_OK) status = srcml_archive_enable_option(archive, SRCML_OPTION_POSITION); + if (status == SRCML_STATUS_OK) status = srcml_archive_set_language(archive, language); + if (status == SRCML_STATUS_OK) status = srcml_archive_set_tabstop(archive, 8); + if (status == SRCML_STATUS_OK) status = srcml_archive_enable_solitary_unit(archive); + if (status == SRCML_STATUS_OK) status = srcml_archive_disable_hash(archive); + if (status == SRCML_STATUS_OK && !srcml_archive_check_extension(archive, filename)) + status = SRCML_STATUS_UNSET_LANGUAGE; + srcml_unit* unit = status == SRCML_STATUS_OK ? srcml_unit_create(archive) : nullptr; + if (unit) { + status = srcml_unit_set_language(unit, language); + if (status == SRCML_STATUS_OK) status = srcml_unit_set_filename(unit, filename); + if (status == SRCML_STATUS_OK) status = srcml_unit_parse_filename(unit, filename); + if (status == SRCML_STATUS_OK) status = srcml_archive_write_unit(archive, unit); + srcml_unit_free(unit); + } + srcml_archive_close(archive); + srcml_archive_free(archive); + if (status == SRCML_STATUS_OK && buffer) + xml.assign(buffer, size); + srcml_memory_free(buffer); + return status; +} +static int tokenize(const std::string& xml) +{ + XMLPlatformUtils::Initialize(); SAX2XMLReader* parser = XMLReaderFactory::createXMLReader(); - srcml2tokenHandlers handler; parser->setContentHandler(&handler); parser->setErrorHandler(&handler); - - int errorCount = 0; - // create a faux scope so that 'src' destructor is called before - // XMLPlatformUtils::Terminate - { - // - // Kick off the parse and catch any exceptions. Create a standard - // input input source and tell the parser to parse from that. - // - try - { - if (argC < 2) { - StdInInputSource src; - parser->parse(src); - } else { - parser->parse(argV[1]); - } - } - catch (const OutOfMemoryException&) - { - XERCES_STD_QUALIFIER cerr << "OutOfMemoryException" << XERCES_STD_QUALIFIER endl; - errorCount = 2; - return 4; - } - catch (const XMLException& e) - { - XERCES_STD_QUALIFIER cerr << "\nError during parsing: \n" - << StrX(e.getMessage()) - << "\n" << XERCES_STD_QUALIFIER endl; - errorCount = 1; - return 4; - } - + int exitCode = 0; + try { + MemBufInputSource source(reinterpret_cast(xml.data()), xml.size(), "srcml"); + parser->parse(source); + } + catch (const OutOfMemoryException&) { + XERCES_STD_QUALIFIER cerr << "OutOfMemoryException" << XERCES_STD_QUALIFIER endl; + exitCode = 4; + } + catch (const XMLException& e) { + XERCES_STD_QUALIFIER cerr << "\nError during parsing: \n" << StrX(e.getMessage()) << "\n" << XERCES_STD_QUALIFIER endl; + exitCode = 4; } - - // - // Delete the parser itself. Must be done prior to calling Terminate, below. - // delete parser; - XMLPlatformUtils::Terminate(); - - if (errorCount > 0) - return 4; - else - return 0; + return exitCode; } +int main(int argC, char* argV[]) +{ + if (argC == 2 && std::string(argV[1]) == "--libsrcml-path") + return print_libsrcml_path(); + if (argC != 4 || std::string(argV[1]) != "-l") + return usage(); + std::string xml; + int status = parse_source(argV[2], argV[3], xml); + if (status == SRCML_STATUS_UNSET_LANGUAGE) + return fail("srcML does not know the extension of [" + std::string(argV[3]) + "]"); + if (status != SRCML_STATUS_OK) + return fail("libsrcml could not parse [" + std::string(argV[3]) + "] as " + argV[2] + + " (status " + std::to_string(status) + ")"); + return tokenize(xml); +} diff --git a/tokenize/tokenize.pl b/tokenize/tokenize.pl old mode 100644 new mode 100755 index 6538d03e4..ea4737b48 --- a/tokenize/tokenize.pl +++ b/tokenize/tokenize.pl @@ -17,11 +17,16 @@ use File::Basename; use FindBin; use lib $FindBin::Bin; +# The extension table and the language->parser table both come from here, and so +# does tokenizeByBlobId/tokenBySha.pl's. They used to be separate literals that +# disagreed; see CregitLanguages.pm. use CregitLanguages; my %declarations; my %listDeclarations; +my %extensions = CregitLanguages::extensions_by_dot(); + my $basedir = dirname($0); $basedir = "." if ($basedir eq ""); @@ -75,7 +80,7 @@ # autodetect Usage("File has no extension. You must provide one [$filename]") unless $filename =~ /(\.[a-z0-9\+]+)$/i; my $ext = lc($1); - $language = CregitLanguages::language_for_ext( substr($1, 1) ); + $language = $extensions{$ext}; Usage("Unknown extension [$ext] in file [$filename]. You must provide language using --language option") unless defined $language and $language ne ""; Usage("Unknown parser for extension [$ext] in file [$filename]. You must provide language using --language option") unless defined($parsers{$language}); } else { @@ -147,6 +152,11 @@ sub Tokenize { my $status = execute_command(@command); return if $status == 0; + # Propagate the parser's own exit status instead of die-ing, which would + # collapse every failure onto 255. tokenizeSrcMl.pl signals "srcML died / the + # tokenization is unusable" with a specific status, and blobExec counts that + # status separately from an ordinary error -- so flattening it here would put + # the crash back into the untraceable bucket it came from. if ($status == -1) { die "Unable to execute command @command: $!"; } diff --git a/tokenize/tokenizeSrcMl.pl b/tokenize/tokenizeSrcMl.pl index d7227a8e9..101e9552c 100755 --- a/tokenize/tokenizeSrcMl.pl +++ b/tokenize/tokenizeSrcMl.pl @@ -15,11 +15,18 @@ use strict; use File::Basename; -use File::Temp qw(tempfile); use FindBin; use lib $FindBin::Bin; use CregitLanguages; +# Exit status meaning "this parse did not produce a usable tokenization": +# srcml2token died on a signal or exited non-zero, or the token stream came back +# empty (which a healthy srcML parse never is -- see Verify_Parse). +# +# Picked to be distinguishable from every other outcome in the chain: below 128, +# so it can never be confused with a shell's 128+signal encoding; clear of GNU +# timeout's 124 and 137, which blobExec already reads as "we killed it"; and clear +# of blobExec's own 2 (abort), 3 (mask changed) and 4 (timed out). our $PARSER_CRASH_EXIT = 33; my %declarations; @@ -43,7 +50,6 @@ Options: --srcml2token= - --srcml= --language= --ctags=- --position @@ -52,15 +58,13 @@ my $basedir = dirname($0); $basedir = "." if ($basedir eq ""); -my $srcml = "srcml"; my $srcml2token = "$basedir/srcMLtoken/srcml2token"; my $ctags = "ctags-universal"; my $language = ""; my $verbose; my $position = 0; -GetOptions ("srcml=s" => \$srcml, - "srcml2token=s" => \$srcml2token, +GetOptions ("srcml2token=s" => \$srcml2token, "language=s" => \$language, "ctags=s" => \$ctags, "position" => \$position, @@ -84,8 +88,7 @@ # autodetect Usage("File has no extension. You must provide one [$filename]") unless $filename =~ /(\.[a-z\+]+)$/i; my $ext = lc($1); - $language = CregitLanguages::language_for_ext( substr($1, 1) ); - $language = undef unless defined $language and exists $extensions{ lc($1) }; + $language = $extensions{$ext}; Usage("Unknown extension [$ext] in file [$filename]. You must provide language using --language option") unless defined $language and $language ne ""; } @@ -106,27 +109,14 @@ sub Tokenize { - my $saveDir = `pwd`; - chomp $saveDir; my ($filename) = @_; - - my ($statusFh, $statusFile) = - tempfile("cregit-pipestatus-XXXXXX", TMPDIR => 1, UNLINK => 1); - close $statusFh; - - my $pipeline = sprintf( - '%s -l %s --position %s | %s; printf %%s\\ %%s "${PIPESTATUS[0]}" "${PIPESTATUS[1]}" > %s', - Shell_Quote($srcml), Shell_Quote($language), - Shell_Quote($filename), Shell_Quote($srcml2token), - Shell_Quote($statusFile)); - - open(parser, "-|", "bash", "-c", $pipeline) - or die "Unable to execute srcml pipeline on file [$filename]: $!"; + open(my $parser, "-|", $srcml2token, "-l", $language, $filename) + or die "Unable to execute srcml2token on file [$filename]: $!"; my $lastLine = -1; my $tokensRead = 0; - while () { + while (<$parser>) { $tokensRead++; # print STDERR; chomp; @@ -159,68 +149,29 @@ sub Tokenize } } - my $closed = close parser; - my $closeStatus = $?; - chdir($saveDir); - - Verify_Parse($filename, $tokensRead, $statusFile, $closed, $closeStatus); -} - -sub Shell_Quote -{ - my ($s) = @_; - $s = '' unless defined $s; - $s =~ s/'/'\\''/g; - return "'$s'"; + close $parser; + Verify_Parse($filename, $tokensRead, $?); } +# Fail closed. Anything other than "srcml2token exited 0 with a non-empty token +# stream" is a defect, and must be reported with a status the caller can count +# rather than silently becoming a 0-byte tokenization. sub Verify_Parse { - my ($filename, $tokensRead, $statusFile, $closed, $closeStatus) = @_; - - my ($srcmlStatus, $tokenStatus) = Read_Pipe_Status($statusFile); - - if (defined $srcmlStatus and $srcmlStatus > 128) { - my $signal = $srcmlStatus - 128; - Parser_Crash($filename, - "srcml was killed by signal $signal (shell status $srcmlStatus)."); - } - if (defined $srcmlStatus and $srcmlStatus != 0) { - Parser_Crash($filename, "srcml exited $srcmlStatus."); - } - if (defined $tokenStatus and $tokenStatus > 128) { - my $signal = $tokenStatus - 128; - Parser_Crash($filename, - "srcml2token was killed by signal $signal (shell status $tokenStatus)."); - } - if (defined $tokenStatus and $tokenStatus != 0) { - Parser_Crash($filename, "srcml2token exited $tokenStatus."); - } - - if (not defined $srcmlStatus or not defined $tokenStatus) { - if (not $closed or $closeStatus != 0) { - Parser_Crash_Unknown($filename, $closeStatus); - } - } - - if ($tokensRead == 0) { - Parser_Crash($filename, - "the token stream was empty. A successful srcML parse always emits at " - . "least the begin_unit/end_unit wrapper, even for an empty file, so an " - . "empty stream is a failed parse and never a legitimately empty result."); - } -} - -sub Read_Pipe_Status -{ - my ($statusFile) = @_; - open(my $fh, '<', $statusFile) or return (undef, undef); - my $line = <$fh>; - close $fh; - return (undef, undef) unless defined $line; - chomp $line; - return (undef, undef) unless $line =~ /^([0-9]+)\s+([0-9]+)$/; - return ($1, $2); + my ($filename, $tokensRead, $status) = @_; + my $signal = $status & 127; + Parser_Crash($filename, + "srcml2token was killed by signal $signal. srcML 1.1.0, which it links, " + . "dies on a signal on some C/C++ inputs under --position.") if $signal; + Parser_Crash($filename, "srcml2token exited " . ($status >> 8) . ".") if $status != 0; + + # Even a zero-byte source file yields the begin_unit/end_unit wrapper, so zero + # tokens always means the parse failed. + Parser_Crash($filename, + "the token stream was empty. A successful srcML parse always emits at " + . "least the begin_unit/end_unit wrapper, even for an empty file, so an " + . "empty stream is a failed parse and never a legitimately empty result.") + if $tokensRead == 0; } sub Parser_Crash @@ -232,17 +183,6 @@ sub Parser_Crash exit($PARSER_CRASH_EXIT); } -sub Parser_Crash_Unknown -{ - my ($filename, $closeStatus) = @_; - print STDERR "cregit: tokenization of [$filename] FAILED: the srcml pipeline " - . "reported a non-zero close status ($closeStatus) and bash did not report " - . "PIPESTATUS, so the failing stage is unknown.\n"; - exit($PARSER_CRASH_EXIT); -} - - - sub Declarations_Test { foreach my $line (sort {$a <=> $b} keys %declarations) { diff --git a/tokenize/tokenizerIdentity.pl b/tokenize/tokenizerIdentity.pl new file mode 100755 index 000000000..af630f376 --- /dev/null +++ b/tokenize/tokenizerIdentity.pl @@ -0,0 +1,128 @@ +#!/usr/bin/env perl + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Prints blobExec's --tokenizer-identity, ext=,...: a digest of each +# extension's parser, tokenize.pl, CregitLanguages.pm and, for srcML languages, +# srcml2token, libsrcml, ctags and CregitSrcMl.pm. A change refuses old caches. + +use strict; +use warnings; + +use Digest::SHA qw(sha256_hex); +use File::Basename; +use FindBin; +use lib $FindBin::Bin; +use CregitLanguages; +use Getopt::Long; + +my $usage = " +Usage: $0 [options] + +Prints blobExec's --tokenizer-identity value for this checkout: + ext=,ext=,... + +Options: + --srcml2token= the built srcML tokenizer (required for C/C++/Java) + --ctags= the ctags binary (required for C/C++/Java) + --all-extensions cover every extension in the table, not only the + masked ones (.am/.ac are routed but not masked) +"; + +my $srcml2tokenPath = ""; +my $ctagsPath = ""; +my $allExtensions = 0; + +GetOptions( + "srcml2token=s" => \$srcml2tokenPath, + "ctags=s" => \$ctagsPath, + "all-extensions" => \$allExtensions, +) or die($usage); + +my $basedir = $FindBin::Bin; + +my @shared = ("$basedir/tokenize.pl", "$basedir/CregitLanguages.pm"); + +my %parsers = CregitLanguages::parsers($basedir); + +my @extensions = $allExtensions + ? CregitLanguages::extensions_sorted() + : CregitLanguages::masked_extensions_sorted(); + +# A missing component is fatal: an identity without it would compare equal +# across a change in it. +sub digest_of { + my ($path) = @_; + open(my $fh, '<', $path) + or die "tokenizerIdentity: cannot read [$path]: $!\n" + . " It is part of a tokenizer's identity, and an identity computed without it\n" + . " would compare equal across a change in it. Build the checkout first\n" + . " (./run_pipeline_process.sh --ensure-artifacts) and try again.\n"; + binmode $fh; + my $sha = Digest::SHA->new(256); + $sha->addfile($fh); + close $fh; + return $sha->hexdigest; +} + +sub libsrcml_of { + my ($srcml2token) = @_; + open(my $ph, '-|', $srcml2token, "--libsrcml-path") + or die "tokenizerIdentity: cannot run [$srcml2token]: $!\n"; + my $path = <$ph> // ""; + close $ph; + chomp $path; + die "tokenizerIdentity: [$srcml2token] does not answer --libsrcml-path, so it is\n" + . " not the srcml2token that parses with libsrcml. Rebuild it: make -C tokenize/srcMLtoken\n" + if $? != 0 or $path eq ""; + return $path; +} + +my %digestCache; +sub cached_digest { + my ($path) = @_; + $digestCache{$path} //= digest_of($path); + return $digestCache{$path}; +} + +my @pairs; +my $libsrcml; +for my $ext (@extensions) { + my $language = $CregitLanguages::EXT_LANG{$ext}; + die "tokenizerIdentity: no language for extension [$ext]\n" unless defined $language; + my $parser = $parsers{$language}; + die "tokenizerIdentity: no parser for language [$language]\n" unless defined $parser; + + my @components = (@shared, $parser); + + if ($CregitLanguages::LANG_PARSER_REL{$language} eq $CregitLanguages::SRCML_PARSER_REL) { + for my $pair (["--srcml2token", $srcml2tokenPath], + ["--ctags", $ctagsPath]) { + my ($flag, $path) = @$pair; + die "tokenizerIdentity: $flag is required, because extension [$ext] is parsed by\n" + . " $CregitLanguages::SRCML_PARSER_REL and its tokens depend on that binary.\n$usage" + if $path eq ""; + push @components, $path; + } + $libsrcml //= libsrcml_of($srcml2tokenPath); + push @components, $libsrcml, "$basedir/CregitSrcMl.pm"; + } + + my $combined = sha256_hex(join("\n", map { cached_digest($_) } @components)); + push @pairs, "$ext=$combined"; +} + +die "tokenizerIdentity: no extensions to report\n" unless @pairs; + +print join(",", @pairs), "\n"; diff --git a/tokenizeByBlobId/WORKER_PROTOCOL.md b/tokenizeByBlobId/WORKER_PROTOCOL.md new file mode 100644 index 000000000..a427c122d --- /dev/null +++ b/tokenizeByBlobId/WORKER_PROTOCOL.md @@ -0,0 +1,41 @@ +# Tokenizer worker protocol + +`tokenWorker.pl` is a persistent `tokenBySha.pl`. With +`--tokenizer-worker=`, blobExec starts one worker per processor and +sends each blob to an idle worker. A worker handles one request at a time. + +The worker reads `BFG_MEMO_DIR` and `BFG_TOKENIZE_CMD` from its environment, +as `tokenBySha.pl` does, and gives the same tokens, memo files and exit statuses. + +For C, C++ and Java, the worker does the work of `tokenizeSrcMl.pl` in its own +process (`tokenize/CregitSrcMl.pm`) if `BFG_TOKENIZE_CMD` is `tokenize.pl` with +only the options `--srcml2token=`, `--ctags=` and `--position`: one +`ctags --_interactive` per language stays alive and only `srcml2token` starts per +file. Otherwise it starts `BFG_TOKENIZE_CMD` per blob. It refuses to start if +`srcml2token` does not answer `--libsrcml-path`. + +## Messages + +At start, the worker writes `READY\n` to stdout. Then, for each request: + +``` +JVM -> worker: REQ \n +worker -> JVM: RES \n +``` + +- `name` is the file name of the blob. Its extension selects the language. +- Lengths are byte counts. The strings are raw bytes. +- `exit` is the status that `tokenBySha.pl` gives: `0` (the tokens are in + `out`), `33` (parser crash) or another status (tokenizer error, `out` is + empty). `124` means that the tokenizer did not finish in `timeoutSeconds`. +- The worker memoizes only a successful tokenization. + +## Failures + +- The worker runs the tokenizer (or `srcml2token`) in its own process group. At the + timeout it kills the group and the helpers, and answers `124`. +- If no answer comes in `timeoutSeconds + 5` seconds, blobExec kills the + worker and its children, starts a new worker, and reports a timeout. +- If the worker stops in a request, blobExec reports a tokenizer error for + that blob and starts a new worker. +- At EOF on stdin, the worker exits 0. A malformed request makes it exit 2. diff --git a/tokenizeByBlobId/tokenBySha.pl b/tokenizeByBlobId/tokenBySha.pl old mode 100644 new mode 100755 index 7099e9a0f..93d91c7d0 --- a/tokenizeByBlobId/tokenBySha.pl +++ b/tokenizeByBlobId/tokenBySha.pl @@ -44,6 +44,8 @@ make_path($buildDir) if not -d $buildDir; +my %mapLang = %CregitLanguages::EXT_LANG; + if (not defined($ENV{BFG_MEMO_DIR}) || $ENV{BFG_MEMO_DIR} eq "") { die "You must define the environment variable BFG_MEMO_DIR equal to the directory where to memoize" @@ -84,14 +86,12 @@ die "BFG_FILENAME environment variable not set " if $blobFN eq ""; my $fileExt; -my $blobLang; if ($blobFN =~ /\.([^.]+)$/) { $fileExt = lc($1); - $blobLang = CregitLanguages::language_for_ext($1); } -if (not defined($blobLang)) { +if (not defined($mapLang{$fileExt})) { die "unknown file extension [$fileExt]"; } @@ -117,7 +117,7 @@ my ($fout, $outfile) = mkstemp( "$buildDir/tmpfile-out-XXXXX" ); - my $langOp = "--language=" . $blobLang; + my $langOp = "--language=" . $mapLang{$fileExt}; open(PROC, "cd \"$wd\" && $tokenizeCmd $langOp \"$inName\" |") or die "unable to execute $tokenizeCmd (verify variable BFG_TOKENIZE_CMD) [$tokenizeCmd]"; diff --git a/tokenizeByBlobId/tokenWorker.pl b/tokenizeByBlobId/tokenWorker.pl new file mode 100755 index 000000000..f44c708e8 --- /dev/null +++ b/tokenizeByBlobId/tokenWorker.pl @@ -0,0 +1,160 @@ +#!/usr/bin/env perl + +# A persistent tokenBySha.pl with the same output and memo. The protocol is in +# WORKER_PROTOCOL.md. C, C++ and Java go through CregitSrcMl in this process. + +use strict; +use warnings; +use bytes; +use Digest::SHA qw(sha1_hex); +use File::Basename qw(basename dirname); +use File::Copy qw(move); +use File::Path qw(make_path); +use File::Temp qw(tempfile); +use FindBin qw($RealBin); +use POSIX qw(_exit); +use lib "$RealBin/../tokenize"; +use CregitLanguages; +use CregitSrcMl; + +binmode STDIN; +binmode STDOUT; +$| = 1; + +my $memoDir = $ENV{BFG_MEMO_DIR} // ""; +die "You must define BFG_MEMO_DIR\n" if $memoDir eq ""; +die "Sha dir [$memoDir] does not exist\n" if not -d $memoDir; +my @tokenizeCommand = split ' ', $ENV{BFG_TOKENIZE_CMD} // ""; +die "Tokenize command not defined. Use BFG_TOKENIZE_CMD\n" if not @tokenizeCommand; +my $buildDir = "$RealBin/build"; +make_path($buildDir); +my $inProcess = in_process_tokenizer(@tokenizeCommand); + +print "READY\n"; +while (defined(my $header = )) { + my ($bodyLength, $nameLength, $timeoutSecs) = $header =~ /\AREQ (\d+) (\d+) (\d+)\n\z/ + or protocol_error("malformed request header [$header]"); + my $filename = read_exact($nameLength); + my $contents = read_exact($bodyLength); + my ($exit, $out, $err) = eval { tokenize_blob($filename, $contents, $timeoutSecs) }; + ($exit, $out, $err) = (255, "", $@) if not defined $exit; + print "RES $exit ", length($out), " ", length($err), "\n", $out, $err; +} +exit 0; + +sub read_exact { + my ($length) = @_; + my $value = ""; + while (length($value) < $length) { + my $read = read(STDIN, $value, $length - length($value), length($value)); + protocol_error(defined $read ? "the request ended early" : "cannot read the request: $!") + if not $read; + } + return $value; +} + +sub protocol_error { + print STDERR "tokenWorker.pl: @_\n"; + exit 2; +} + +sub tokenize_blob { + my ($filename, $contents, $timeoutSecs) = @_; + my ($ext) = $filename =~ /\.([^.]+)$/; + $ext = lc($ext // ""); + my $language = $CregitLanguages::EXT_LANG{$ext}; + return (255, "", "unknown file extension [$ext]\n") if not defined $language; + + my $sha1 = sha1_hex($contents); + my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . "/$sha1"; + return (0, read_file($memoFile), "") if -f $memoFile; + + my $workDir = File::Temp->newdir("tokdir-XXXXX", DIR => $buildDir); + write_file("$workDir/input.$ext", $contents); + my ($exit, $out, $err) = tokenize_file($language, "input.$ext", "$workDir", $timeoutSecs); + return ($exit, "", $err) if $exit != 0; + write_memo($memoFile, $out); + return (0, $out, $err); +} + +sub tokenize_file { + my ($language, $inputName, $workDir, $timeoutSecs) = @_; + return $inProcess->tokenize($language, $inputName, $workDir, $timeoutSecs) + if $inProcess and $CregitLanguages::LANG_PARSER_REL{$language} eq $CregitLanguages::SRCML_PARSER_REL; + return run_tokenizer($workDir, $timeoutSecs, @tokenizeCommand, "--language=$language", $inputName); +} + +# Only a tokenize.pl command with the options that it gives to tokenizeSrcMl.pl +# can run in this process. Any other command starts once per blob. +sub in_process_tokenizer { + my ($program, @options) = @_; + return undef if basename($program) ne "tokenize.pl"; + my %config = (srcml2token => dirname($program) . "/srcMLtoken/srcml2token"); + for (@options) { + if (/\A--(srcml2token|ctags)=(.+)\z/) { $config{$1} = $2 } + elsif ($_ eq "--position") { $config{position} = 1 } + else { return undef } + } + return CregitSrcMl->new(%config); +} + +sub run_tokenizer { + my ($workDir, $timeoutSecs, @command) = @_; + my $pid = fork() // die "cannot fork the tokenizer: $!\n"; + exec_in_group($workDir, @command) if $pid == 0; + + my $timedOut = 0; + local $SIG{ALRM} = sub { $timedOut = 1; kill_group($pid) }; + alarm($timeoutSecs); + waitpid($pid, 0); + my $status = $?; + alarm(0); + + my $err = read_file("$workDir/.stderr"); + return (124, "", $err . "tokenize command timed out after [$timeoutSecs] seconds\n") if $timedOut; + return (255, "", $err . "tokenize command was killed by signal " . ($status & 127) . "\n") if $status & 127; + return ($status >> 8, read_file("$workDir/.stdout"), $err); +} + +# The child's own process group lets a timeout kill srcml and everything else +# the tokenizer started. +sub exec_in_group { + my ($workDir, @command) = @_; + setpgrp(0, 0); + chdir($workDir) and open(STDIN, '<', '/dev/null') and open(STDOUT, '>', '.stdout') + and open(STDERR, '>', '.stderr') or _exit(255); + { no warnings 'exec'; exec { $command[0] } @command; } + print STDERR "cannot execute the tokenizer [$command[0]]: $!\n"; + _exit(255); +} + +sub kill_group { + my ($pid) = @_; + kill 'TERM', -$pid; + select(undef, undef, undef, 0.1); + kill 'KILL', -$pid; +} + +sub read_file { + my ($path) = @_; + open(my $in, '<:raw', $path) or die "cannot read [$path]: $!\n"; + local $/; + return <$in> // ""; +} + +sub write_file { + my ($path, $contents) = @_; + open(my $out, '>:raw', $path) or die "cannot write [$path]: $!\n"; + print {$out} $contents; + close($out) or die "cannot close [$path]: $!\n"; +} + +sub write_memo { + my ($memoFile, $output) = @_; + make_path(dirname($memoFile)); + my ($out, $temp) = tempfile("tmpfile-out-XXXXX", DIR => $buildDir); + binmode $out; + print {$out} $output; + close($out) or die "cannot close [$temp]: $!\n"; + move($temp, $memoFile) or die "cannot move the tokenization to [$memoFile]: $!\n"; +}