From 73c6df48200eac008151febfa050a2c69f540e78 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Thu, 8 Oct 2026 21:02:14 +0000 Subject: [PATCH] fix(srcml): patch the two non-termination defects srcML never finishes on two kinds of file, so the census excludes them after the 600 s blob timeout: 64 versions of NSS certdata.c and 17 boost/hana headers in the finished runs. The patch, against the pinned source d38271916, carries: - the lexer ends an ordinary C, C++ or Objective-C string at its closing quote, so "a""b" is two literals and not one open string that flips string and code for the rest of the file; - a statement leaves function-name mode when the function-name rule consumes no token twice at the same token, which is the only way that rule can loop. With it, all 81 excluded blobs finish, in at most 1.1 s of CPU. So do the 4 C++-in-a-C-header blobs on the blob denylist. The 8 Java blobs there (srcML/srcML#2361) already finish at d38271916. Measured over 8492 C, C++ and Java blobs from the census originals, NSS at foxhound HEAD, boost/hana headers from sonic-pi and files picked for the changed rules: 8454 token streams are byte-identical, 22 are files that never finished before, and 16 are files with adjacent string literals. In 12 of them the lexer fix corrects the markup. The other 4 are copies of sqlite3.c, where an #error string continued over a backslash-newline already puts the lexer out of step, in both builds; the fix only splits that wrong text differently. srcML's own parser test suite passes 53496/53496 with the patch. Without it, the suite fails the 66 adjacent-string cases and does not finish the new macro-before-template case. The tokenizer identity includes libsrcml, so a work directory with cached tokens needs `--retokenize` for the srcML extensions. --- nix/srcml-nontermination.patch | 129 +++++++++++++++++++++++++++++++++ nix/srcml.nix | 18 +++++ 2 files changed, 147 insertions(+) create mode 100644 nix/srcml-nontermination.patch diff --git a/nix/srcml-nontermination.patch b/nix/srcml-nontermination.patch new file mode 100644 index 00000000..7a444dae --- /dev/null +++ b/nix/srcml-nontermination.patch @@ -0,0 +1,129 @@ +diff --git a/src/parser/CommentTextLexer.g b/src/parser/CommentTextLexer.g +index d6d0c138a..5bf81cf32 100644 +--- a/src/parser/CommentTextLexer.g ++++ b/src/parser/CommentTextLexer.g +@@ -232,7 +232,17 @@ COMMENT_TEXT { + in_squotes = true; + } + (options { greedy = true; } : +- { !skip_dquote_processing && (prevLA != '\\' || noescape) }? ++ // in C, C++, and Objective-C, "a""b" is two adjacent string literals: ++ // the quote that ends a string must not take the quote that starts the next one ++ { ++ !skip_dquote_processing ++ && (prevLA != '\\' || noescape) ++ && !( ++ mode == STRING_END ++ && !noescape ++ && (inLanguage(LANGUAGE_C) || inLanguage(LANGUAGE_CXX) || inLanguage(LANGUAGE_OBJECTIVE_C)) ++ ) ++ }? + '\042' + { + ++dquote_count; +diff --git a/src/parser/srcMLParser.g b/src/parser/srcMLParser.g +index bc1b573d3..d1b18da94 100644 +--- a/src/parser/srcMLParser.g ++++ b/src/parser/srcMLParser.g +@@ -766,6 +766,11 @@ public: + bool in_template_param = false; + int start_count = 0; + ++ // The token at which the function-name rule last matched nothing; see ++ // statement_part's function-identifier alternative. A reference, not a ++ // pointer, keeps that token alive, so no later token can reuse its address. ++ antlr::RefToken function_name_stuck_at; ++ + static const antlr::BitSet keyword_name_token_set; + static const antlr::BitSet keyword_token_set; + static const antlr::BitSet macro_call_token_set; +@@ -5387,6 +5392,7 @@ statement_part[] { + CALL_TYPE type = NOCALL; + bool isempty = false; + int call_count = 0; ++ antlr::RefToken function_name_start; + ENTRY_DEBUG + } : + { inMode(MODE_EAT_TYPE) }? +@@ -5610,7 +5616,21 @@ statement_part[] { + + // function identifier + { inMode(MODE_FUNCTION_NAME) }? +- function_header[0] | ++ { function_name_start = LT(1); } ++ function_header[0] ++ { ++ // function_header[0] can match nothing, e.g., at auto after a type that a ++ // leading macro made too long. Once is harmless: another rule can still ++ // consume that token. Twice at the same token is a loop, because start() ++ // calls this rule again in the same state, so leave function-name mode. ++ if (LT(1).get() != function_name_start.get()) ++ function_name_stuck_at = antlr::RefToken(); ++ else if (function_name_stuck_at.get() == function_name_start.get()) { ++ function_name_stuck_at = antlr::RefToken(); ++ clearMode(MODE_FUNCTION_NAME); ++ } else ++ function_name_stuck_at = function_name_start; ++ } | + + // function identifier + { inMode(MODE_FUNCTION_PARAMETER) }? +diff --git a/test/parser/testsuite/string_adjacent.cpp.xml b/test/parser/testsuite/string_adjacent.cpp.xml +new file mode 100644 +index 000000000..5e85d85bc +--- /dev/null ++++ b/test/parser/testsuite/string_adjacent.cpp.xml +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++"a""b"; ++ ++ ++ ++""""; ++ ++ ++ ++"a\"""b"; ++ ++ ++ +diff --git a/test/parser/testsuite/suite.txt b/test/parser/testsuite/suite.txt +index 3811cb22f..9df797c1e 100644 +--- a/test/parser/testsuite/suite.txt ++++ b/test/parser/testsuite/suite.txt +@@ -1,5 +1,6 @@ + LANGUAGE_ALL_NO_PYTHON comment + LANGUAGE_ALL_NO_PYTHON string ++LANGUAGE_C_FAMILY_NO_SHARP string_adjacent + LANGUAGE_ALL_BASE variable_names + LANGUAGE_ALL_NO_PYTHON expression + LANGUAGE_ALL_NO_PYTHON decl_simple +@@ -46,6 +47,7 @@ LANGUAGE_CXX enum + LANGUAGE_C_FAMILY for + #LANGUAGE_C_FAMILY auto + LANGUAGE_CXX auto ++LANGUAGE_CXX template_after_macro + #LANGUAGE_C_FAMILY block_lambda + LANGUAGE_CXX block_lambda + #LANGUAGE_C_FAMILY cuda +diff --git a/test/parser/testsuite/template_after_macro.cpp.xml b/test/parser/testsuite/template_after_macro.cpp.xml +new file mode 100644 +index 000000000..eff461c78 +--- /dev/null ++++ b/test/parser/testsuite/template_after_macro.cpp.xml +@@ -0,0 +1,12 @@ ++ ++ ++ ++ ++X template <class R> auto a; ++ ++ ++ ++X template <class R> const a; ++ ++ ++ diff --git a/nix/srcml.nix b/nix/srcml.nix index 614c46b4..2c2c57c1 100644 --- a/nix/srcml.nix +++ b/nix/srcml.nix @@ -1,6 +1,22 @@ # srcML develop at d38271916: it fixes the heap corruption on a GCC attribute # after an expression in parentheses (srcML/srcML#2425), which 1.1.0 dies on. # No release carries the fix yet; move the pin to the first one that does. +# +# srcml-nontermination.patch adds two fixes that upstream does not have yet. +# Without them srcML never finishes on some files, so the census drops them: +# - a C string that ends right before the next one, as in "a""b" (NSS +# certdata.c ran over 600 s; it takes about 1 s); +# - no function name after a type that a macro made too long, as in +# "X template auto a;" (17 boost/hana headers looped for ever). +# The patch is `git diff d38271916 543fed6af` on branch +# fix/cregit-nontermination of a local srcML clone. Drop it when upstream has both. +# +# Measured over 8492 C, C++ and Java blobs: 8454 token streams are +# byte-identical, 22 are files that never finished before, and 16 hold adjacent +# string literals. In 12 of those the patch corrects the markup. The other 4 are +# sqlite3.c copies, where an #error string continued over a backslash-newline +# (a defect the patch does not touch) already puts the lexer out of step; the +# patch only changes how that already wrong text is split. { lib , stdenv , fetchFromGitHub @@ -59,6 +75,8 @@ stdenv.mkDerivation { hash = "sha256-SRUvvsMLenVadNOG3S49Rwe0eDEf3AidTxvo3RtZPHg="; }; + patches = [ ./srcml-nontermination.patch ]; + nativeBuildInputs = [ cmake ninja jdk_headless ]; buildInputs = [ libxml2 libxslt libarchive curl ]