From 7777535ed979723d11e472e3aa4654b5d95d792f Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Tue, 6 Oct 2026 04:34:22 +0000 Subject: [PATCH] fix(blobExec): denylist only the 4 hanging blobs srcML 1.1.0 did not terminate on 8 Java blobs with annotated array dimensions (srcML/srcML#2361). #104 moved srcML to develop d38271916. Develop parses all 8 blobs. Each one gives exit 0 and a token stream of 1,154 to 2,588 lines in 0.07 s or less. The denylist kept them out of the dataset for no reason. The 4 StatTimes.h blobs stay on the list. Develop still does not end on them within 120 s, and the minimal reproducer still hangs. Their reason now names both srcML versions. BlobDenylistSpec now asserts the 4 entries, and a new test asserts that the 8 Java blobs are not on the list. The lookup tests use a C entry. The WalkerIntegrationSpec fixture no longer cites #2361. DATASET.md said that the denylist holds 209 blobs in three srcML 1.1.0 defects. #102 removed the 197 crash entries, and this commit removes the 8 Java entries. README.md gave 12 entries. Both now give the 4 StatTimes.h blobs, which srcML 1.1.0 and develop d38271916 do not end on. --- README.md | 6 ++--- .../cregit/blobexec/BlobDenylistEntries.scala | 25 +++++-------------- .../cregit/blobexec/BlobDenylistSpec.scala | 22 ++++++++++------ .../blobexec/WalkerIntegrationSpec.scala | 6 ++--- generate_dataset/DATASET.md | 10 ++++---- 5 files changed, 31 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index ea89528a..dfae65fc 100644 --- a/README.md +++ b/README.md @@ -186,9 +186,9 @@ returns without invoking `srcml` at all. Put it out of reach with `--memo-dir`, names neither repository nor extension). The runner refuses a step-1 wipe that would delete a memo of 10,000 entries or more; `--force-clean` overrides it. -12 blobs on which srcML 1.1.0 does not terminate (8 Java, srcML/srcML#2361; 4 C) are -on a **blob denylist** (`BlobDenylistEntries.scala`). They never reach the tokenizer, -are dropped from the rewritten trees, and are counted as `blobsDenylisted`. +4 C blobs on which srcML does not terminate (`eden/fs/utils/StatTimes.h`, no upstream +issue) are on a **blob denylist** (`BlobDenylistEntries.scala`). They never reach the +tokenizer, are dropped from the rewritten trees, and are counted as `blobsDenylisted`. A blob whose tokenizer times out, reports a parser crash or exits non-zero is excluded the same way without an entry: it is named on an `EXCLUDED failed blob` diff --git a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala index 236feae7..a68075cb 100644 --- a/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala +++ b/blobExec/src/main/scala/cregit/blobexec/BlobDenylistEntries.scala @@ -1,33 +1,20 @@ package cregit.blobexec -/** Blobs on which srcML 1.1.0 does not terminate, keyed by git blob id and grouped - * by defect. A tokenizer crash needs no entry: the walk already excludes that blob. */ +/** Blobs on which srcML 1.1.0 and develop d38271916 do not terminate, keyed by git + * blob id and grouped by defect. A tokenizer crash needs no entry: the walk already + * excludes that blob. */ private[blobexec] object BlobDenylistEntries { import BlobDenylist.Entry - private val JavaAnnotatedArrays = Entry( - "srcML/srcML#2361", - "srcML 1.1.0 does not terminate on annotated array dimensions, minimal reproducer " + - "`class C { int @A [] a @A []; }`; OpenJDK langtools regression tests " + - "TestNewCastArray (@bug 8005681) and CheckErrorsForSource7" - ) - private val CppInCHeader = Entry( "srcml-nontermination-analysis.md#class-2-the-c-parser-on-c-inside-a-h", - "srcML 1.1.0's C parser does not terminate on C++ in a .h file (no upstream issue), " + - "minimal reproducer `namespace n{struct T&f(const struct S);}`; eden/fs/utils/StatTimes.h" + "srcML's C parser (1.1.0 and develop d38271916) does not terminate on C++ in a .h file " + + "(no upstream issue), minimal reproducer `namespace n{struct T&f(const struct S);}`; " + + "eden/fs/utils/StatTimes.h" ) val NonTerminating: Map[String, Entry] = Map( - "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65" -> JavaAnnotatedArrays, - "303c1a109f1fdc4b9d84e15111c56931202844ac" -> JavaAnnotatedArrays, - "504aaf9109fc01d6d62d99ac529ded99a8dfa8ca" -> JavaAnnotatedArrays, - "c91924d1e87cb82d372a73b24579ccc34609efaa" -> JavaAnnotatedArrays, - "56c8adf588e697a3e42b600b2e5ec54710592c61" -> JavaAnnotatedArrays, - "5e9c9138ed9c43c20591d1814cb72ad014d1a9e5" -> JavaAnnotatedArrays, - "6bf666550f6cba72f1602ce1b7a6b0c9025fc8c7" -> JavaAnnotatedArrays, - "afbd81e5789ba060147b0de3be4f01759963299d" -> JavaAnnotatedArrays, "5460d43f1884e3be230e02215151b9b963844a48" -> CppInCHeader, "720914b27e170b73b38ef662fb04239273dd0a34" -> CppInCHeader, "99c191e9a208547f5c9b1224c290f106a640de59" -> CppInCHeader, diff --git a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala index dc033b8b..2c28d8dc 100644 --- a/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/BlobDenylistSpec.scala @@ -8,6 +8,7 @@ import org.scalatest.matchers.should.Matchers class BlobDenylistSpec extends AnyFunSuite with Matchers { // TestNewCastArray.java (4) and CheckErrorsForSource7.java (4): srcML/srcML#2361. + // srcML develop d38271916 parses them, so they must not be on the list. private val JavaAnnotationShas = Set( "2ee2673ad0a8ff2cef0254e7bfdc488cc1d61a65", "303c1a109f1fdc4b9d84e15111c56931202844ac", @@ -29,10 +30,16 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { private val shipped = BlobDenylistEntries.NonTerminating - test("the shipped list holds exactly the twelve known hangs, so an addition is deliberate") { + test("the shipped list holds exactly the four known hangs, so an addition is deliberate") { // Also catches a sha written twice: a Map literal silently keeps the last. - shipped.keySet shouldEqual JavaAnnotationShas ++ CParserShas - BlobDenylist.shipped.size shouldEqual 12 + shipped.keySet shouldEqual CParserShas + BlobDenylist.shipped.size shouldEqual 4 + } + + test("the Java blobs that srcML develop parses are not denylisted") { + JavaAnnotationShas.foreach { sha => + withClue(s"$sha: ") { BlobDenylist.shipped.entryFor(sha) shouldEqual None } + } } test("every key is a 40-character lowercase hex git blob id") { @@ -44,10 +51,9 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { test("every entry carries a one-line reason and the citation of its own defect") { shipped.foreach { case (sha, e) => withClue(s"$sha: ") { - if (JavaAnnotationShas.contains(sha)) e.citation shouldEqual "srcML/srcML#2361" - else e.citation should include("srcml-nontermination-analysis.md") + e.citation should include("srcml-nontermination-analysis.md") e.reason should not include "\n" - e.reason should include("srcML 1.1.0") + e.reason should include("1.1.0 and develop d38271916") } } } @@ -64,7 +70,7 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { } test("lookup is by git blob id, in either case, and misses everything else") { - val sha = JavaAnnotationShas.head + val sha = CParserShas.head BlobDenylist.shipped.entryFor(sha) should not be empty BlobDenylist.shipped.entryFor(sha.toUpperCase) should not be empty BlobDenylist.shipped.entryFor("0" * 40) shouldEqual None @@ -78,6 +84,6 @@ class BlobDenylistSpec extends AnyFunSuite with Matchers { test("an empty list matches nothing, so a test fixture cannot exclude by accident") { BlobDenylist.empty.isEmpty shouldBe true - BlobDenylist.empty.entryFor(JavaAnnotationShas.head) shouldEqual None + BlobDenylist.empty.entryFor(CParserShas.head) shouldEqual None } } diff --git a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala index 7b1ab37f..34994d55 100644 --- a/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala +++ b/blobExec/src/test/scala/cregit/blobexec/WalkerIntegrationSpec.scala @@ -804,7 +804,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte } // -- DENYLISTED mask-matched blob ------------------------------------------- - // srcML 1.1.0 never terminates on these (srcML/srcML#2361). A denylisted blob is + // srcML never terminates on these (eden/fs/utils/StatTimes.h). A denylisted blob is // excluded and does not gate the exit status; an unexplained timeout still does. /** The git blob id of `content`, computed the way git computes it, so a fixture @@ -815,7 +815,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte private def checkDenylistExclusion(label: String, pipeline: Boolean, pipelineTrees: Boolean): Unit = { val dir = freshWorkDir("denylisted-" + label) val keep = "int main(){}\n" - // Stands in for TestNewCastArray.java. Its content is irrelevant: the point is + // Stands in for a blob that hangs srcML. Its content is irrelevant: the point is // that the tokenizer is never asked about it, so a fixture that would hang is // not needed and would make the test itself hang. val hangs = "String @A [] [] s = new String @A [2] [2];\n" @@ -827,7 +827,7 @@ class WalkerIntegrationSpec extends AnyFunSuite with Matchers with BeforeAndAfte val denied = blobIdOf(hangs) val denylist = BlobDenylist(Map( - denied.name -> BlobDenylist.Entry("srcML/srcML#2361", "srcML 1.1.0 does not terminate on it"))) + denied.name -> BlobDenylist.Entry("no upstream issue", "srcML does not terminate on it"))) // Every invocation of the tokenizer records the blob it was given. "Excluded in // milliseconds, never handed to the tokenizer" is only checkable from the diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index ca045e7e..aa115d40 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -304,11 +304,11 @@ produces no blame and no dataset row rather than rows of unparsed text: | **oversized** blobs (at or above JGit's stream-file threshold) | `blobsOversized`, plus one `EXCLUDED oversized blob` line each | no rows for those blobs | | a blob the tokenizer **timed out**, **crashed** or **failed** on | `blobsTimedOut`, `blobsParserCrashed` or `blobsTokenizerFailed`, plus one `EXCLUDED failed blob` line each with `reason=timeout`, `reason=parser-crash` or `reason=tokenizer-error` | no rows for those blobs. The run still exits 0 | -The denylist holds 209 blob ids, keyed by content, in three srcML 1.1.0 defects: -8 Java blobs that do not terminate (upstream -[`srcML/srcML#2361`](https://github.com/srcML/srcML/issues/2361)), 4 C blobs that do -not terminate (no upstream issue), and 197 C and C++ blobs on which srcML crashes -under `--position`. The header of the file carries the reproducers and citations. +The denylist holds 4 blob ids, keyed by content: 4 blobs of +`eden/fs/utils/StatTimes.h`. srcML's C parser does not terminate on the C++ in this +`.h` file (no upstream issue). This is true for srcML 1.1.0 and for develop +d38271916, which devenv builds. The entry in the file carries the reproducer and +the citation. Also outside the dataset, by mask rather than by exclusion: M4 (`.am`, `.ac`), whose tokenizer's lexer is not fit for real autotools input, and `.ixx`, `.inl`,