From c73173d6fc8bdd59faa9ef4da2fe3b14ab0b01e7 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Fri, 9 Oct 2026 20:31:37 +0000 Subject: [PATCH 01/10] fix(dataset): read the trailers from the original log DB The footer_* columns came from the footers table of the cregit DB (step 4). That DB comes from the rewritten repository. There, blobExec appends "\n" + "Former-commit-id: X" to a message whose last paragraph has trailers. A git CLI message ends in "\n", so a blank line then comes before Former-commit-id, and JGit finds no other trailer. Only messages with no final newline (for example, from the GitHub web UI) kept their trailers. The curl census Parquet has 0 trailers, but git finds them on 520 of its 12,072 commits. Step 10 now reads the footers table of the original log DB (step 3), keyed by the original commit SHA. The key is the same as for original_commit_sha: commitmap.originalcid, else the cregit SHA. blobExec and the rewritten history do not change, so the commit SHAs and the blob map stay the same, and a resume stays valid. generate_dataset.py has a new required argument, --original-db. If the file is missing or has no footers table, the run stops before Phase 1, with the reason. It does not read the trailers from the cregit DB. run_pipeline_process.sh passes the argument and stops step 10 if the step 3 DB is missing. The column names, types, order, the key match and the idx order do not change. Only the 15 footer_* columns change: on libexpat and iputils, every other column is identical row by row. (cherry picked from commit acd08b4f548b27536d7c9124f932adec605b06b4) --- generate_dataset/DATASET.md | 23 +++- generate_dataset/generate_dataset.py | 64 +++++++++- run_pipeline_process.sh | 3 + tests/test_generate_dataset.py | 184 ++++++++++++++++++++++++--- 4 files changed, 252 insertions(+), 22 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index aa115d40..2002d79a 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -119,8 +119,18 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or ### Commit trailers (footers) Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with -`len(footer_reviewed_by) = 0`. Source is `cregit.db`'s `footers` table, grouped by -commit, keys matched case-insensitively, each list in `footers.idx` order. +`len(footer_reviewed_by) = 0`. Source is the `footers` table of the original +log DB (`-original.db`, Step 3 output), grouped by commit and joined on +`original_commit_sha`. Keys are matched case-insensitively, and each list is in +`footers.idx` order. The trailers are JGit's `getFooterLines` of the original +commit message. + +The source is not `cregit.db`. In the cregit repository, blobExec appends +`"\n" + "Former-commit-id: "` to a message whose last paragraph has +trailers. A git CLI message ends in `"\n"`, so the result has a blank line +before `Former-commit-id`, and JGit then finds no other trailer. A message with +no final newline (for example, from the GitHub web UI) kept its trailers. +Datasets made before this change have this defect. | Column | Trailer key | |--------|-------------| @@ -244,6 +254,7 @@ tables) maps multiple email addresses to a single `personid`. ``` token_map.commit_sha → commits.cid → commitmap.cid → commitmap.originalcid (original commit SHA) + → original DB footers.cid (trailers) → emails.emailname → persons.personid → persons.personname \_ (matched on autname + autemail) ``` @@ -341,6 +352,9 @@ source files ───────────┤ cregit.db ──────────────┤ (commits, commitmap) │ │ +-original.db ─────┤ +(footers) │ + │ persons.db ─────────────┤ (emails, persons) │ │ @@ -364,9 +378,10 @@ For each `.blame` file: ### Phase 2: DuckDB JOIN → Parquet -Join `token_map` with the three SQLite databases: +Join `token_map` with the four SQLite databases: - `commits` — commit metadata (author, committer, dates, summary) - `commitmap` — cregit → original commit SHA mapping +- `footers` of the original log DB — commit trailers, by original commit SHA - `emails` / `persons` — person identity resolution Output is written as ZSTD-compressed Parquet. @@ -378,6 +393,7 @@ uv run python generate_dataset/generate_dataset.py \ --blame-dir ../cregit-files/blame \ --source-dir ../cregit-files/jq-original \ --cregit-db ../cregit-files/jq-cregit.db \ + --original-db ../cregit-files/jq-original.db \ --persons-db ../cregit-files/jq-persons.db \ --output ../cregit-files/jq-dataset.parquet \ --repo-name jq \ @@ -391,6 +407,7 @@ uv run python generate_dataset/generate_dataset.py \ | `--blame-dir` | yes | Directory containing `.blame` files (Step 8 output). | | `--source-dir` | yes | Root of the original source tree. | | `--cregit-db` | yes | Path to `cregit.db` (Step 5 output). | +| `--original-db` | yes | Path to the git log DB of the original repository (Step 3 output). The `footer_*` columns come from its `footers` table. If the file is missing or has no `footers` table, the run stops before Phase 1. | | `--persons-db` | yes | Path to `persons.db` (Step 6 output). | | `--output` | yes | Output Parquet file path. | | `--repo-name` | no | Repository name. Default: inferred from output filename. | diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index 1712cef7..d227dd36 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -20,6 +20,7 @@ --blame-dir ../cregit-files/blame \ --source-dir ../cregit-files/jq-original \ --cregit-db ../cregit-files/jq-cregit.db \ + --original-db ../cregit-files/jq-original.db \ --persons-db ../cregit-files/jq-persons.db \ --output ../cregit-files/jq-dataset.parquet """ @@ -514,6 +515,33 @@ def project_meta_sql(project_meta) -> str: ) +# The columns that the trailer query reads from the footers table. +FOOTER_COLUMNS = ("cid", "idx", "key", "value") + + +def footers_table_problem(db_path) -> str: + """Return why db_path cannot supply the trailers, or "" when it can. + + The trailers come from the footers table that slickGitLog writes for the + original repository. Check that table before Phase 1, because Phase 1 can + take half an hour, and a bad input must stop the run before that work. + """ + try: + con = sqlite3.connect(f"file:{Path(db_path).resolve()}?mode=ro", uri=True) + try: + names = {row[1] for row in con.execute("PRAGMA table_info(footers)")} + finally: + con.close() + except sqlite3.Error as exc: + return f"it is not a readable SQLite database ({exc})" + if not names: + return "it has no footers table" + missing = [c for c in FOOTER_COLUMNS if c not in names] + if missing: + return f"its footers table has no column {', '.join(missing)}" + return "" + + def main(): parser = argparse.ArgumentParser( description="Generate unified Parquet dataset from CreGit pipeline outputs" @@ -527,6 +555,15 @@ def main(): parser.add_argument( "--cregit-db", required=True, help="Path to cregit.db (Step 5 output)" ) + parser.add_argument( + "--original-db", + required=True, + help="Path to the git log DB of the original repository (Step 3 output). " + "The footer_* columns come from its footers table, keyed by the original " + "commit SHA. The cregit DB cannot supply them: its commit messages carry " + "the Former-commit-id trailer that blobExec adds, and when a blank line " + "comes before that trailer, JGit sees no other trailer.", + ) parser.add_argument( "--persons-db", required=True, help="Path to persons.db (Step 6 output)" ) @@ -596,11 +633,22 @@ def main(): print(f"ERROR: {label} not found: {p}", file=sys.stderr) sys.exit(1) - for p, label in [(args.cregit_db, "cregit-db"), (args.persons_db, "persons-db")]: + for p, label in [(args.cregit_db, "cregit-db"), (args.original_db, "original-db"), + (args.persons_db, "persons-db")]: if not Path(p).is_file(): print(f"ERROR: {label} not found: {p}", file=sys.stderr) + if label == "original-db": + print(" Step 3 writes this file. Run the pipeline again from step 3 " + "or earlier. The trailers do not come from the cregit DB.", + file=sys.stderr) sys.exit(1) + problem = footers_table_problem(args.original_db) + if problem: + print(f"ERROR: original-db {args.original_db} cannot supply the trailers: " + f"{problem}", file=sys.stderr) + sys.exit(1) + if not args.repo_name: args.repo_name = output_path.stem.replace("-dataset", "") @@ -727,6 +775,10 @@ def main(): con.execute(f"CALL sqlite_attach('{sync_db_path}')") con.execute(f"CALL sqlite_attach('{args.cregit_db}')") con.execute(f"CALL sqlite_attach('{args.persons_db}')") + # A named catalog, not sqlite_attach: the original DB has the same table + # names as the cregit DB, and sqlite_attach puts every table in one schema. + original_db = str(args.original_db).replace(chr(39), chr(39) * 2) + con.execute(f"ATTACH '{original_db}' AS original_log (TYPE sqlite, READ_ONLY)") meta_sql = project_meta_sql(project_meta) @@ -804,7 +856,13 @@ def main(): list(DISTINCT coalesce(fpn.personname, fe.personid) ORDER BY coalesce(fpn.personname, fe.personid)) FILTER (WHERE coalesce(fpn.personname, fe.personid) IS NOT NULL) AS footer_person_names - FROM footers f + -- The trailers come from the original repository, keyed by + -- the original commit SHA. In the cregit repository, blobExec + -- appends "\\n" + "Former-commit-id: X" to a message whose last + -- paragraph has trailers. When that message ends in "\\n" (the + -- git CLI writes it so), a blank line comes before the new + -- trailer, and JGit then sees only Former-commit-id. + FROM original_log.footers f -- regexp_extract returns '' rather than NULL on no match, so -- without nullif a no-address trailer joins on '' and matches -- every `Name <>` row. Bracketed form first: correct precedence. @@ -815,7 +873,7 @@ def main(): ) LEFT JOIN persons fpn ON fe.personid = fpn.personid GROUP BY f.cid - ) ftr ON c.cid = ftr.cid + ) ftr ON coalesce(m.originalcid, t.commit_sha) = ftr.cid ORDER BY t.file_path, t.token_index ) TO '{output_path}' diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 7ac200e4..de073b65 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -172,6 +172,7 @@ build_dataset_argv() { --blame-dir "$WORK/blame" --source-dir "$REPO_PATH_ORIGINAL" --cregit-db "$DB_PATH_CREGIT" + --original-db "$DB_PATH_ORIGINAL" --persons-db "$DB_PATH_PERSONS" --output "$DATASET_PATH" --repo-name "$REPO_NAME" @@ -1020,6 +1021,8 @@ if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then DATASET_SCRIPT="$CREGIT/generate_dataset/generate_dataset.py" [ -f "$DATASET_SCRIPT" ] || die "dataset generator not found: $DATASET_SCRIPT" if [ -n "$PYTHON" ] && "$PYTHON" -c 'import duckdb' 2>/dev/null; then +# The footer_* columns come from the original log DB (see generate_dataset.py). +[ -f "$DB_PATH_ORIGINAL" ] || die "step 3 did not produce $DB_PATH_ORIGINAL, and step 10 reads the trailers from it. Resume from step 3 or earlier." build_dataset_argv "$PYTHON" "$DATASET_SCRIPT" "${DATASET_ARGV[@]}" log "dataset written: $DATASET_PATH" diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index e7addb25..191f929a 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -100,6 +100,16 @@ def test_is_ws_treats_none_as_not_whitespace(): # there: filling a tmpfs also exhausts system memory. # --------------------------------------------------------------------------- # +def make_original_db(path): + """The git log DB of the original repository, as slickGitLog writes it in + step 3. Only its footers table is read.""" + con = sqlite3.connect(path) + con.execute("CREATE TABLE footers (cid CHAR(40), idx INTEGER, key TEXT, value TEXT)") + con.commit() + con.close() + return path + + def run_main(monkeypatch, tmp_path, argv_extra=()): """Drive main() far enough to reach the scratch allocation, then stop. @@ -116,6 +126,7 @@ def run_main(monkeypatch, tmp_path, argv_extra=()): (src / "a.c").write_text("int x;\n") for name in ("cregit.db", "persons.db"): (tmp_path / name).write_text("") + make_original_db(tmp_path / "original.db") out = tmp_path / "out" / "proj-dataset.parquet" seen = {} @@ -130,6 +141,7 @@ def fake_mkstemp(*a, **kw): "--blame-dir", str(blame), "--source-dir", str(tmp_path / "src"), "--cregit-db", str(tmp_path / "cregit.db"), + "--original-db", str(tmp_path / "original.db"), "--persons-db", str(tmp_path / "persons.db"), "--output", str(out), *argv_extra, @@ -308,7 +320,8 @@ def test_a_key_absent_from_the_sidecar_fails_loudly(tmp_path): # --------------------------------------------------------------------------- # TOTAL_COLUMNS = 67 # 38 token/commit columns + 29 metadata -SHA = "a" * 40 +SHA = "a" * 40 # the commit in the cregit repository +ORIGINAL_SHA = "b" * 40 # the same commit in the original repository def build_tiny_project(tmp_path): @@ -333,9 +346,12 @@ def build_tiny_project(tmp_path): con.execute("INSERT INTO commits VALUES (?,?,?,?,?,?,?,?,0)", (SHA, "A Dev", "a@example.com", "2020-01-01", "A Dev", "a@example.com", "2020-01-01", "first")) + con.execute("INSERT INTO commitmap VALUES (?,?,'')", (SHA, ORIGINAL_SHA)) con.commit() con.close() + original_db = make_original_db(tmp_path / "original.db") + persons_db = tmp_path / "persons.db" con = sqlite3.connect(persons_db) con.executescript(""" @@ -347,26 +363,27 @@ def build_tiny_project(tmp_path): """) con.commit() con.close() - return blame, src, cregit_db, persons_db + return blame, src, cregit_db, original_db, persons_db def generate(monkeypatch, tmp_path, repo_name="proj", argv_extra=(), seed=None): """Run main() for real and return the Parquet path. - `seed(cregit_db, persons_db)` runs between the fixture and main(), for the - tests that need a trailer or a person in the input. + `seed(cregit_db, original_db, persons_db)` runs between the fixture and + main(), for the tests that need a trailer or a person in the input. """ import generate_dataset as gd - blame, src, cregit_db, persons_db = build_tiny_project(tmp_path) + blame, src, cregit_db, original_db, persons_db = build_tiny_project(tmp_path) if seed is not None: - seed(cregit_db, persons_db) + seed(cregit_db, original_db, persons_db) out = tmp_path / "out" / "proj-dataset.parquet" monkeypatch.setattr(gd.sys, "argv", [ "generate_dataset.py", "--blame-dir", str(blame), "--source-dir", str(src), "--cregit-db", str(cregit_db), + "--original-db", str(original_db), "--persons-db", str(persons_db), "--output", str(out), "--repo-name", repo_name, @@ -455,16 +472,21 @@ def test_an_unknown_project_key_stops_the_run_before_phase_1(monkeypatch, tmp_pa # --------------------------------------------------------------------------- # -def seed_footers(*trailers, people=()): - """Return a `seed` for generate(): trailers into footers, people into - emails/persons. `people` is (personid, emailaddr, personname).""" - def seed(cregit_db, persons_db): - con = sqlite3.connect(cregit_db) - for idx, (key, value) in enumerate(trailers): - con.execute("INSERT INTO footers VALUES (?,?,?,?)", - (SHA, idx, key, value)) - con.commit() - con.close() +def seed_footers(*trailers, people=(), cregit_trailers=()): + """Return a `seed` for generate(): trailers into the footers table of the + original DB, keyed by the original commit SHA, and people into + emails/persons. `people` is (personid, emailaddr, personname). + `cregit_trailers` go into the footers table of the cregit DB, keyed by the + cregit commit SHA, as slickGitLog writes them in step 4.""" + def seed(cregit_db, original_db, persons_db): + for db, sha, rows in ((original_db, ORIGINAL_SHA, trailers), + (cregit_db, SHA, cregit_trailers)): + con = sqlite3.connect(db) + for idx, (key, value) in enumerate(rows): + con.execute("INSERT INTO footers VALUES (?,?,?,?)", + (sha, idx, key, value)) + con.commit() + con.close() con = sqlite3.connect(persons_db) for i, (pid, addr, pname) in enumerate(people, start=1): con.execute("INSERT INTO emails (recordid, personid, fullemail, " @@ -518,6 +540,136 @@ def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): assert list(names) == [] +# --------------------------------------------------------------------------- # +# The trailers come from the original repository. In the cregit repository, +# blobExec appends "\n" + "Former-commit-id: X" to a message whose last +# paragraph has trailers. A git CLI message ends in "\n", so a blank line then +# comes before Former-commit-id, and JGit (step 4) sees no other trailer. +# --------------------------------------------------------------------------- # + +CLI_MESSAGE = ("Fix the parser\n" + "\n" + "The parser stopped at the first comment.\n" + "\n" + "Signed-off-by: A Dev \n" + "Reviewed-by: Bob B \n") + + +def rewritten_by_blobexec(message: str, original_sha: str) -> str: + """FormerCommitFooter.append for a last paragraph that has trailers.""" + return message + "\n" + "Former-commit-id: " + original_sha + + +def git_trailer_keys(message: str) -> list[str]: + import subprocess + out = subprocess.run(["git", "interpret-trailers", "--parse"], input=message, + capture_output=True, text=True, check=True).stdout + return [line.split(":", 1)[0] for line in out.splitlines()] + + +def test_the_rewritten_cli_message_hides_its_trailers_from_a_trailer_parser(): + """The defect itself, with git as the parser: the original message has two + trailers, and the rewritten message has only Former-commit-id.""" + import shutil + if shutil.which("git") is None: + pytest.skip("git is not on PATH") + rewritten = rewritten_by_blobexec(CLI_MESSAGE, ORIGINAL_SHA) + assert "\n\nFormer-commit-id: " in rewritten + assert git_trailer_keys(CLI_MESSAGE) == ["Signed-off-by", "Reviewed-by"] + assert git_trailer_keys(rewritten) == ["Former-commit-id"] + + +def test_a_cli_message_keeps_its_signed_off_by_and_reviewed_by(monkeypatch, tmp_path): + """The two DBs hold what slickGitLog finds for CLI_MESSAGE: the original DB + has both trailers, and the cregit DB has only Former-commit-id.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Reviewed-by", "Bob B "), + cregit_trailers=[("Former-commit-id", ORIGINAL_SHA)], + people=[("adev", "a@example.com", "A Dev"), + ("bob", "bob@example.com", "Bob B")])) + + row = duckdb.sql("select original_commit_sha, footer_signed_off_by, " + "footer_reviewed_by, footer_personids, footer_person_names " + f"from read_parquet('{out}')").fetchone() + sha, signed, reviewed, ids, names = row + assert sha == ORIGINAL_SHA + assert list(signed) == ["A Dev "] + assert list(reviewed) == ["Bob B "] + assert list(ids) == ["adev", "bob"] + assert list(names) == ["A Dev", "Bob B"] + + +def test_the_trailers_of_the_cregit_db_are_not_read(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + cregit_trailers=[("Signed-off-by", "A Dev ")])) + + (signed,) = duckdb.sql("select footer_signed_off_by " + f"from read_parquet('{out}')").fetchone() + assert list(signed) == [] + + +def test_the_trailer_key_falls_back_to_the_cregit_sha_with_no_map_row(monkeypatch, tmp_path): + """The same fallback as original_commit_sha: no commitmap row, so the + original commit SHA is the cregit SHA.""" + duckdb = pytest.importorskip("duckdb") + + def seed(cregit_db, original_db, _persons_db): + con = sqlite3.connect(cregit_db) + con.execute("DELETE FROM commitmap") + con.commit() + con.close() + con = sqlite3.connect(original_db) + con.execute("INSERT INTO footers VALUES (?,0,'Signed-off-by','A Dev ')", + (SHA,)) + con.commit() + con.close() + + out = generate(monkeypatch, tmp_path, seed=seed) + sha, signed = duckdb.sql("select original_commit_sha, footer_signed_off_by " + f"from read_parquet('{out}')").fetchone() + assert sha == SHA + assert list(signed) == ["A Dev "] + + +def test_a_missing_original_db_stops_the_run_before_phase_1(monkeypatch, tmp_path, capsys): + """An old resume can lack the step 3 DB. The run must stop and say so, not + read the trailers from the cregit DB.""" + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, + argv_extra=("--original-db", str(tmp_path / "absent.db"))) + err = capsys.readouterr().err + assert "original-db not found" in err + assert "step 3" in err + + +@pytest.mark.parametrize("content, reason", [ + (b"", "no footers table"), + (b"not a database, only text" * 40, "not a readable SQLite database"), +], ids=["empty-file", "not-sqlite"]) +def test_an_original_db_with_no_footers_table_stops_the_run_before_phase_1( + monkeypatch, tmp_path, capsys, content, reason): + bad = tmp_path / "bad.db" + bad.write_bytes(content) + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, argv_extra=("--original-db", str(bad))) + assert reason in capsys.readouterr().err + + +def test_a_call_without_the_original_db_is_refused(monkeypatch, tmp_path): + """An old caller that does not pass --original-db must fail, not fall back.""" + import generate_dataset as gd + blame, src, cregit_db, _original_db, persons_db = build_tiny_project(tmp_path) + monkeypatch.setattr(gd.sys, "argv", [ + "generate_dataset.py", "--blame-dir", str(blame), "--source-dir", str(src), + "--cregit-db", str(cregit_db), "--persons-db", str(persons_db), + "--output", str(tmp_path / "out" / "proj-dataset.parquet")]) + with pytest.raises(SystemExit): + gd.main() + + # --------------------------------------------------------------------------- # # Token-line repairs before the source walk: srcML mojibake, the Rust position # prefix and the BOM. Each test checks text and position after the defect. From 974193b48817563aca3e77efcfe6327f5bf757aa Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Fri, 9 Oct 2026 20:32:47 +0000 Subject: [PATCH 02/10] fix(dataset): do not repeat a trailer for each emails row The typed footer_* lists and the person columns came from one subquery, which joins each trailer to emails on its address. The persons DB has one emails row for each name that an address committed with. So a trailer with such an address came out once for each row: on libexpat, 1 of 1,752 commits had its Co-authored-by twice, and on iputils, 1 of 388 commits had its Reviewed-by twice. git shows each of them once. The typed lists now come from a subquery over footers alone. The footer_personids and footer_person_names subquery is the same as before, so those two columns do not change. A trailer line that the message repeats stays repeated. (cherry picked from commit 9079a9c63f4656b8bf76a56159d4f62d817fea8b) --- generate_dataset/DATASET.md | 4 ++-- generate_dataset/generate_dataset.py | 27 ++++++++++++++++++--------- tests/test_generate_dataset.py | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 2002d79a..d01a96d5 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -122,8 +122,8 @@ Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with `len(footer_reviewed_by) = 0`. Source is the `footers` table of the original log DB (`-original.db`, Step 3 output), grouped by commit and joined on `original_commit_sha`. Keys are matched case-insensitively, and each list is in -`footers.idx` order. The trailers are JGit's `getFooterLines` of the original -commit message. +`footers.idx` order, with one value for each trailer line. The trailers are +JGit's `getFooterLines` of the original commit message. The source is not `cregit.db`. In the cregit repository, blobExec appends `"\n" + "Former-commit-id: "` to a message whose last paragraph has diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index d227dd36..d964586b 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -826,8 +826,8 @@ def main(): coalesce(ftr.footer_mentored_by, []) AS footer_mentored_by, coalesce(ftr.footer_assisted_by, []) AS footer_assisted_by, coalesce(ftr.footer_thanks_to, []) AS footer_thanks_to, - coalesce(ftr.footer_personids, []) AS footer_personids, - coalesce(ftr.footer_person_names, []) AS footer_person_names + coalesce(fpr.footer_personids, []) AS footer_personids, + coalesce(fpr.footer_person_names, []) AS footer_person_names FROM token_map t JOIN commits c ON t.commit_sha = c.cid @@ -850,12 +850,7 @@ def main(): list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'helped-by') AS footer_helped_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'mentored-by') AS footer_mentored_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'assisted-by') AS footer_assisted_by, - list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'thanks-to') AS footer_thanks_to, - list(DISTINCT fe.personid ORDER BY fe.personid) - FILTER (WHERE fe.personid IS NOT NULL) AS footer_personids, - list(DISTINCT coalesce(fpn.personname, fe.personid) - ORDER BY coalesce(fpn.personname, fe.personid)) - FILTER (WHERE coalesce(fpn.personname, fe.personid) IS NOT NULL) AS footer_person_names + list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'thanks-to') AS footer_thanks_to -- The trailers come from the original repository, keyed by -- the original commit SHA. In the cregit repository, blobExec -- appends "\\n" + "Former-commit-id: X" to a message whose last @@ -863,6 +858,20 @@ def main(): -- git CLI writes it so), a blank line comes before the new -- trailer, and JGit then sees only Former-commit-id. FROM original_log.footers f + GROUP BY f.cid + ) ftr ON coalesce(m.originalcid, t.commit_sha) = ftr.cid + -- A separate subquery, so that the emails join cannot repeat a + -- trailer in the lists above: one address can have many emails + -- rows, one for each name that it committed with. + LEFT JOIN ( + SELECT + f.cid, + list(DISTINCT fe.personid ORDER BY fe.personid) + FILTER (WHERE fe.personid IS NOT NULL) AS footer_personids, + list(DISTINCT coalesce(fpn.personname, fe.personid) + ORDER BY coalesce(fpn.personname, fe.personid)) + FILTER (WHERE coalesce(fpn.personname, fe.personid) IS NOT NULL) AS footer_person_names + FROM original_log.footers f -- regexp_extract returns '' rather than NULL on no match, so -- without nullif a no-address trailer joins on '' and matches -- every `Name <>` row. Bracketed form first: correct precedence. @@ -873,7 +882,7 @@ def main(): ) LEFT JOIN persons fpn ON fe.personid = fpn.personid GROUP BY f.cid - ) ftr ON coalesce(m.originalcid, t.commit_sha) = ftr.cid + ) fpr ON coalesce(m.originalcid, t.commit_sha) = fpr.cid ORDER BY t.file_path, t.token_index ) TO '{output_path}' diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index 191f929a..90992ecf 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -601,6 +601,22 @@ def test_a_cli_message_keeps_its_signed_off_by_and_reviewed_by(monkeypatch, tmp_ assert list(names) == ["A Dev", "Bob B"] +def test_an_address_with_two_names_does_not_repeat_the_trailer(monkeypatch, tmp_path): + """The persons DB has one emails row for each name that an address + committed with. The join to emails must not repeat the trailer.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", "Bob B "), + ("Reviewed-by", "Bob B "), + people=[("bob", "bob@example.com", "Bob B"), + ("bob", "bob@example.com", "bob-laptop")])) + + ids, names, reviewed = footers_of(duckdb, out) + assert list(reviewed) == ["Bob B "] * 2 # two lines, two values + assert list(ids) == ["bob"] + assert list(names) == ["Bob B"] + + def test_the_trailers_of_the_cregit_db_are_not_read(monkeypatch, tmp_path): duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( From b2a07bb52c368e17351a6df0100a965b40d52327 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Fri, 9 Oct 2026 22:44:16 +0000 Subject: [PATCH 03/10] fix(dataset): keep one entry per person in each trailer list A commit message can name one person twice in one trailer key: with the same text, or with another address. In the Linux Parquet, 4,695 of 839,023 commits have one Signed-off-by value twice, and 7,135 have one name with another text, for example "Borislav Petkov (AMD) " and "Borislav Petkov ". git log shows these repeats in the messages. Each of the 13 typed footer_* lists now holds one entry per person. In one list, two values are one person when their addresses resolve to one personid. The address is the one that footer_personids uses: the address in <...>, else a bare address. If the address is not in emails, the lowercased address is the key. A value with no address is never merged. For each person, the list keeps the value with the lowest idx, at its place, so the list is in the order of the first line of each person. footer_personids and footer_person_names do not change. (cherry picked from commit 1f176633675eb836e322bf66bcf87343e98c98e1) --- generate_dataset/DATASET.md | 22 ++++- generate_dataset/generate_dataset.py | 51 +++++++++-- tests/test_generate_dataset.py | 122 ++++++++++++++++++++++++++- 3 files changed, 184 insertions(+), 11 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index d01a96d5..6674302c 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -121,10 +121,28 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with `len(footer_reviewed_by) = 0`. Source is the `footers` table of the original log DB (`-original.db`, Step 3 output), grouped by commit and joined on -`original_commit_sha`. Keys are matched case-insensitively, and each list is in -`footers.idx` order, with one value for each trailer line. The trailers are +`original_commit_sha`. Keys are matched case-insensitively. The trailers are JGit's `getFooterLines` of the original commit message. +Each of the 13 typed lists holds **one entry per person**, not the raw trailer +lines. A message can name one person twice, with the same text or with another +address (for example `Borislav Petkov (AMD) ` and +`Borislav Petkov `). In one list, two values are the same person +when: + +- their addresses resolve to the same `personid`. The address is the one that + `footer_personids` uses: the address inside `<…>`, else a bare address. It + is matched against `emails.emailaddr`. If the `emails` rows of one address + name two persons, the lowest `personid` is used. +- their addresses are not in `emails`, and the lowercased addresses are equal. + +A value with no address is kept as it is and is never merged, also when its +text repeats. For each person, the list keeps the value with the lowest +`footers.idx`, at the place of that idx, so the list is in the order of the +first trailer line of each person. The rule works in each list separately: one +person can be in `footer_signed_off_by` and in `footer_reviewed_by`. +`footer_personids` and `footer_person_names` do not change: they are sets. + The source is not `cregit.db`. In the cregit repository, blobExec appends `"\n" + "Former-commit-id: "` to a message whose last paragraph has trailers. A git CLI message ends in `"\n"`, so the result has a blank line diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index d964586b..d47921ab 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -851,13 +851,50 @@ def main(): list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'mentored-by') AS footer_mentored_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'assisted-by') AS footer_assisted_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'thanks-to') AS footer_thanks_to - -- The trailers come from the original repository, keyed by - -- the original commit SHA. In the cregit repository, blobExec - -- appends "\\n" + "Former-commit-id: X" to a message whose last - -- paragraph has trailers. When that message ends in "\\n" (the - -- git CLI writes it so), a blank line comes before the new - -- trailer, and JGit then sees only Former-commit-id. - FROM original_log.footers f + -- One entry for each person in each list: the trailer line with + -- the lowest idx. A message can name one person twice, with the + -- same text or with another address of that person. + FROM ( + SELECT + f.cid, f.idx, f.key, f.value, + row_number() OVER (PARTITION BY f.cid, LOWER(f.key), f.who + ORDER BY f.idx) AS nth, + f.who + FROM ( + -- who: the personid of the address, else the lowercased + -- address, else NULL (a value with no address is kept). + -- The address is the one that footer_personids uses. + SELECT + f.cid, f.idx, f.key, f.value, + CASE WHEN fa.personid IS NOT NULL THEN 'p:' || fa.personid + WHEN f.addr IS NOT NULL THEN 'a:' || LOWER(f.addr) + END AS who + FROM ( + SELECT + f.cid, f.idx, f.key, f.value, + coalesce( + nullif(regexp_extract(f.value, '<([^>]+)>', 1), ''), + nullif(regexp_extract(f.value, '([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+[.][A-Za-z]{{2,}})', 1), '') + ) AS addr + -- The trailers come from the original repository, keyed by + -- the original commit SHA. In the cregit repository, blobExec + -- appends "\\n" + "Former-commit-id: X" to a message whose last + -- paragraph has trailers. When that message ends in "\\n" (the + -- git CLI writes it so), a blank line comes before the new + -- trailer, and JGit then sees only Former-commit-id. + FROM original_log.footers f + ) f + -- One personid for each address. min() if the emails rows of + -- one address name two persons. + LEFT JOIN ( + SELECT emailaddr, min(personid) AS personid + FROM emails + WHERE personid IS NOT NULL + GROUP BY emailaddr + ) fa ON fa.emailaddr = f.addr + ) f + ) f + WHERE f.who IS NULL OR f.nth = 1 GROUP BY f.cid ) ftr ON coalesce(m.originalcid, t.commit_sha) = ftr.cid -- A separate subquery, so that the emails join cannot repeat a diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index 90992ecf..33575632 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -606,17 +606,135 @@ def test_an_address_with_two_names_does_not_repeat_the_trailer(monkeypatch, tmp_ committed with. The join to emails must not repeat the trailer.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( - ("Reviewed-by", "Bob B "), ("Reviewed-by", "Bob B "), people=[("bob", "bob@example.com", "Bob B"), ("bob", "bob@example.com", "bob-laptop")])) ids, names, reviewed = footers_of(duckdb, out) - assert list(reviewed) == ["Bob B "] * 2 # two lines, two values + assert list(reviewed) == ["Bob B "] assert list(ids) == ["bob"] assert list(names) == ["Bob B"] +# --------------------------------------------------------------------------- # +# One entry for each person in each typed list. Two values are the same person +# when their addresses resolve to one personid; an address that is not in +# emails merges on its lowercased text; a value with no address never merges. +# The entry kept is the value with the lowest idx, at the place of that idx. +# --------------------------------------------------------------------------- # + +def typed_lists_of(duckdb, path, *cols): + row = duckdb.sql(f"select {', '.join(cols)} from read_parquet('{path}')").fetchone() + return [list(v) for v in row] + + +def test_an_identical_repeated_trailer_gives_one_entry(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["A Dev "] + + +def test_two_addresses_of_one_person_give_the_first_value(monkeypatch, tmp_path): + """The Linux case: "Borislav Petkov (AMD) " and + "Borislav Petkov " are one personid.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "Borislav Petkov (AMD) "), + ("Signed-off-by", "Borislav Petkov "), + people=[("bp", "bp@alien8.de", "Borislav Petkov"), + ("bp", "bp@suse.de", "Borislav Petkov")])) + + signed, ids = typed_lists_of(duckdb, out, "footer_signed_off_by", "footer_personids") + assert signed == ["Borislav Petkov (AMD) "] + assert ids == ["bp"] + + +def test_two_addresses_of_two_persons_stay_two_entries(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev"), + ("adev2", "a@other.example.com", "A Dev")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["A Dev ", "A Dev "] + + +def test_an_unresolved_address_merges_across_letter_case(monkeypatch, tmp_path): + """No emails row for the address: the lowercased address is the key.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "Carol "), + ("Reported-by", "carol "))) + + reported, ids = typed_lists_of(duckdb, out, "footer_reported_by", "footer_personids") + assert reported == ["Carol "] + assert ids == [] + + +def test_a_bare_address_and_a_bracketed_one_are_one_person(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "carol@example.com"), + ("Reported-by", "Carol "))) + + [reported] = typed_lists_of(duckdb, out, "footer_reported_by") + assert reported == ["carol@example.com"] + + +def test_a_value_with_no_address_is_never_merged(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "Daniel Stenberg"), + ("Reported-by", "Daniel Stenberg"), + ("Reported-by", "daniel stenberg"))) + + [reported] = typed_lists_of(duckdb, out, "footer_reported_by") + assert reported == ["Daniel Stenberg", "Daniel Stenberg", "daniel stenberg"] + + +def test_the_entries_keep_the_order_of_their_first_idx(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), # idx 0: a, kept + ("Signed-off-by", "Bob B "), # idx 1: bob, kept + ("Signed-off-by", "No Address"), # idx 2: kept + ("Signed-off-by", "A Dev (work) "), # idx 3: a again, dropped + ("Signed-off-by", "Carol "), # idx 4: unresolved, kept + ("Signed-off-by", "Bob B "), # idx 5: bob again, dropped + people=[("adev", "a@example.com", "A Dev"), + ("adev", "a@work.example", "A Dev"), + ("bob", "bob@example.com", "Bob B")])) + + signed, ids, names = typed_lists_of(duckdb, out, "footer_signed_off_by", + "footer_personids", "footer_person_names") + assert signed == ["A Dev ", "Bob B ", "No Address", + "Carol "] + assert ids == ["adev", "bob"] + assert names == ["A Dev", "Bob B"] + + +def test_one_person_in_two_lists_stays_in_both(monkeypatch, tmp_path): + """The rule works inside each typed list, not across the lists.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Reviewed-by", "A Dev "), + ("signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev")])) + + signed, reviewed = typed_lists_of(duckdb, out, "footer_signed_off_by", + "footer_reviewed_by") + assert signed == ["A Dev "] + assert reviewed == ["A Dev "] + + def test_the_trailers_of_the_cregit_db_are_not_read(monkeypatch, tmp_path): duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( From 00b739adaf479b3e2c7d67044af91a79ee7cabe1 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Sun, 11 Oct 2026 18:55:39 +0000 Subject: [PATCH 04/10] fix(blame): limit -C100 time, fail on a git error git blame -C100 takes 3 to 38 hours on one huge generated file (protobuf Java, tree-sitter parser.c, ICU stubdata.cpp). Plain blame takes about 3 minutes on the same file. formatBlame.pl now runs git with fork and exec, not through a shell, so it knows the pid and the exit status of git. After --c100-limit seconds (default CREGIT_C100_LIMIT, else 3600; 0 = no limit), it kills -C100, discards the partial output, and runs git blame --porcelain without -C and with no limit. It prints one line to STDERR: "PLAIN-BLAME after s of -C100". The pipeline log keeps it. formatBlame.pl now checks the exit status of git. Before, a git that stopped early (killed, out of memory) gave an empty .blame file and exit 0, so the tokens of the file were lost with no error. Now the script removes the temp file, writes no .blame file, and exits 1. blameRepoFiles.pl counts the error, and the blame step fails. A file that ends within the limit gets the same bytes as before: Read_Record and copy_file do not change. blameRepoFiles.pl does not change: its children inherit CREGIT_C100_LIMIT. (cherry picked from commit 57551d7ad82a784615c1490623d72cb9dc994ba1) --- blameRepo/formatBlame.pl | 78 +++++++++++++++++++++++--- generate_dataset/DATASET.md | 25 +++++++++ tests/t/formatBlame.t | 108 +++++++++++++++++++++++++++++++++++- 3 files changed, 202 insertions(+), 9 deletions(-) diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index 7f028348..76d8a379 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -22,6 +22,7 @@ use Getopt::Long; use Pod::Usage; use FindBin qw($RealBin); +use POSIX qw(_exit); # build/ tempdir anchored to the script's dir, not the caller's CWD. my $buildDir = "$RealBin/build"; @@ -31,13 +32,21 @@ my $help = 0; my $man = 0; my $verbose = 0; +# Wall-clock limit in seconds for blame -C100. 0 means no limit. +# blameRepoFiles.pl starts this script, so CREGIT_C100_LIMIT reaches it with no new flag. +my $c100Limit = (defined $ENV{CREGIT_C100_LIMIT} and $ENV{CREGIT_C100_LIMIT} ne '') + ? $ENV{CREGIT_C100_LIMIT} : 3600; GetOptions ("blameExtension=s" => \$blameExtension, + "c100-limit=s" => \$c100Limit, "help" => \$help, # string "verbose" => \$verbose, "man" => \$man) # flag or die("Error in command line arguments\n"); +die "Error: the -C100 limit must be a whole number of seconds, not [$c100Limit]\n" + unless $c100Limit =~ /^[0-9]+$/; + if ($man) { pod2usage(-verbose=>2); exit(1) @@ -62,23 +71,68 @@ print STDERR "$0 processing repo [$repo] file [$file] [$dest]\n"; } +# Suppressed headers still owe field 2 a filename. +my %filenameOfCommit; + # -C100: credit a line moved in from another file to its author, not the mover. # --porcelain: same shas, but the header once per commit instead of once per line. -open(IN, "git -C '$repo' blame -C100 --porcelain '$file'|" ) or "unable to execute git "; -while (my $l = Read_Record()) { - print $fh $l; - print $fh "\n"; +# A huge generated file can take hours with -C100. After $c100Limit seconds, the +# script stops it and runs plain blame on the same file, with no limit. +my $started = time(); +my $status = Blame_To_Temp($c100Limit, '-C100', '--porcelain'); +if (not defined $status) { + printf STDERR "PLAIN-BLAME %s after %d s of -C100\n", $file, time() - $started; + $status = Blame_To_Temp(0, '--porcelain'); } -close IN; close $fh; +# A git that stops early (killed, out of memory) gives a short or empty stream. +# Do not keep it: no .blame file, and a non-zero exit that blameRepoFiles.pl counts. +if ($status != 0) { + unlink($temp); + print STDERR "Error: git blame of [$file] in [$repo] failed with status [$status]; no blame file written\n"; + exit(1); +} copy_file($temp, $dest, $file . $blameExtension); if ($verbose) { print STDERR "...completed\n"; } - -# Suppressed headers still owe field 2 a filename. -my %filenameOfCommit; +# Runs git blame with @options on $file and writes the records to the temp file. +# Returns the wait status of git, or undef if $limit (seconds, 0 = none) passed first. +# On undef, git is killed and the temp file is empty again. +sub Blame_To_Temp { + my ($limit, @options) = @_; + %filenameOfCommit = (); + seek($fh, 0, 0) and truncate($fh, 0) or die "unable to empty [$temp]: $!"; + # fork + exec, not a shell, so the pid and the exit status are those of git. + my $pid = open(IN, '-|'); + die "unable to start git blame: $!" unless defined $pid; + if ($pid == 0) { + exec('git', '-C', $repo, 'blame', @options, $file); + print STDERR "unable to execute git: $!\n"; + _exit(127); + } + my $timedOut = 0; + eval { + local $SIG{ALRM} = sub { die "c100-limit\n" }; + alarm($limit); + while (my $l = Read_Record()) { + print $fh $l; + print $fh "\n"; + } + alarm(0); + }; + alarm(0); + if ($@) { + die $@ unless $@ eq "c100-limit\n"; + $timedOut = 1; + kill('KILL', $pid); + } + close IN; + my $waitStatus = $?; + return undef if $timedOut; + return $waitStatus; +} sub Read_Record { my $f ; @@ -155,6 +209,10 @@ =head1 SYNOPSIS Options: --blame-extension=s extension to append to output files. Default is .blame + --c100-limit=N seconds for git blame -C100. After that, the + script kills it and runs git blame without -C. + Default is $CREGIT_C100_LIMIT, else 3600. + 0 means no limit. --help brief help message --man full documentation @@ -177,4 +235,8 @@ =head1 DESCRIPTION with the same full path as the original file, with the .blame extension (which can be changed with the --blame option). +If git blame -C100 does not end within the limit, the script runs +git blame without -C and prints "PLAIN-BLAME after s of -C100" +to STDERR. If git fails, the script writes no output file and exits 1. + =cut diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 6674302c..324ba3dd 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -344,6 +344,31 @@ whose tokenizer's lexer is not fit for real autotools input, and `.ixx`, `.inl`, `.cppm`, `.cxxm`, `.ipp`, for which srcML emits **zero tokens and exits 0**. `file_mask` on every row records the mask that decided this. +### Files blamed without `-C100` + +Step 7 blames each file with `git blame -C100`. This option credits a line that +moved in from another file to the commit that wrote it, not to the commit that +moved it. On a huge generated file (protobuf or thrift Java, a tree-sitter +`parser.c`, ICU `stubdata.cpp`), `-C100` can take 3 to 38 hours for one file. + +Thus `blameRepo/formatBlame.pl` has a wall-clock limit for `-C100`: 3600 seconds +by default. The option `--c100-limit=SECONDS` or the environment variable +`CREGIT_C100_LIMIT` sets a different limit. The value 0 removes the limit. When +the limit passes, the script stops `-C100` and blames the same file with plain +`git blame`, with no limit. The pipeline log then holds one line +`PLAIN-BLAME after s of -C100`. + +In such a file, a line that moved in from another file has the commit that moved +it. The rows of the file do not record this. To find these files, search the +pipeline log for `PLAIN-BLAME`. A file that ends within the limit gets the same +blame bytes as before. + +If git fails or stops early for a different reason (for example, the kernel +kills it for memory), `formatBlame.pl` writes no `.blame` file and exits 1. +`blameRepoFiles.pl` counts the error, and the blame step fails. Before this +change, the script wrote an empty `.blame` file and exited 0. The tokens of that +file were lost, and no error showed it. + ### Computed and then dropped, or never typed - `func_name` is computed in Phase 1 for `DECL` tokens and is **not** carried into diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 5c13a737..7b056d5c 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,9 +2,10 @@ use strict; use warnings; -use Test::More tests => 20; +use Test::More tests => 41; use FindBin; use File::Temp qw(tempdir); +use File::Basename qw(dirname); my $root = "$FindBin::Bin/../.."; # tests/t -> repo root @@ -132,6 +133,111 @@ my $cid2 = commit_as($repo, "Bob", "second"); "all six moved lines are credited to the author, naming origin.c"); } +# The -C100 time limit and the git exit status, on the same moved.c repository. +# A fake git on PATH is slow or fails for "blame"; all other commands go to the real git. +{ + my $repo2 = "$workdir/moved"; + my $mover = git($repo2, "rev-parse HEAD"); + my $realGit = `command -v git`; + chomp $realGit; + my $fakeBin = "$workdir/fakebin"; + mkdir $fakeBin or die $!; + write_file("$fakeBin/git", <<"FAKE"); +#!/bin/sh +blame=0; c100=0 +for a in "\$@"; do + [ "\$a" = blame ] && blame=1 + [ "\$a" = -C100 ] && c100=1 +done +if [ \$blame = 1 ]; then + case "\$FAKE_GIT_MODE" in + slow-c100) [ \$c100 = 1 ] && exec sleep 60 ;; + slow-c100-plain-fails) [ \$c100 = 1 ] && exec sleep 60; exit 3 ;; + fail) echo "fatal: fake failure" >&2; exit 3 ;; + killed) printf '%040d 1 1 1\\n' 0; kill -9 \$\$ ;; + esac +fi +exec '$realGit' "\$@" +FAKE + chmod 0755, "$fakeBin/git" or die $!; + + my $buildDir = dirname($script) . "/build"; + my @tempsBefore = glob("$buildDir/tmpfile-*"); + + my $slurp = sub { + my ($f) = @_; + open(my $fh, '<', $f) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; + }; + my $run = sub { + my ($mode, $opts, %env) = @_; + my $dest = tempdir(CLEANUP => 1); + local $ENV{PATH} = "$fakeBin:$ENV{PATH}"; + local $ENV{FAKE_GIT_MODE} = $mode; + local $ENV{CREGIT_C100_LIMIT} = $env{limit} if defined $env{limit}; + my $t0 = time(); + my $status = system("perl '$script' $opts '$repo2' moved.c '$dest' 2>'$workdir/stderr'"); + return ($status, $dest, time() - $t0, $slurp->("$workdir/stderr")); + }; + + # (a) A blame that ends within the limit: the same bytes as with no limit. + my ($s1, $d1) = $run->('none', ''); + my ($s2, $d2) = $run->('none', '--c100-limit=0'); + my ($s3, $d3) = $run->('none', '--c100-limit=3600'); + is($s1 + $s2 + $s3, 0, "the default, no limit and 3600 s runs all succeed"); + my $bytes = $slurp->("$d1/moved.c.blame"); + is($slurp->("$d2/moved.c.blame"), $bytes, "no limit gives the same bytes as the default limit"); + is($slurp->("$d3/moved.c.blame"), $bytes, "a 3600 s limit gives the same bytes as the default limit"); + + # (b) -C100 passes the limit: plain blame output and the PLAIN-BLAME line. + my ($s4, $d4, $sec4, $err4) = $run->('slow-c100', '--c100-limit=1'); + is($s4, 0, "a -C100 run that passes the limit still succeeds"); + cmp_ok($sec4, '<', 30, "the script kills the slow -C100 git and does not wait for it"); + like($err4, qr/^PLAIN-BLAME moved\.c after [0-9]+ s of -C100$/m, + "the script prints the PLAIN-BLAME line to STDERR"); + my @plain = split /\n/, ($slurp->("$d4/moved.c.blame") // ''); + is(scalar(@plain), 10, "the plain blame file has ten lines"); + my @byMover = grep { /^\Q$mover\E;;/ } @plain[2 .. 7]; + is(scalar(@byMover), 6, "plain blame credits the moved lines to the mover, so no -C100 ran"); + my $plainRef = `git -C '$repo2' blame --porcelain moved.c | grep -c '^ '`; + chomp $plainRef; + is($plainRef, 10, "git blame without -C gives the same line count"); + + # The environment variable sets the limit when there is no flag. + my ($s5, $d5, $sec5, $err5) = $run->('slow-c100', '', limit => 1); + is($s5, 0, "CREGIT_C100_LIMIT=1 run succeeds"); + like($err5, qr/^PLAIN-BLAME moved\.c after [0-9]+ s of -C100$/m, + "CREGIT_C100_LIMIT sets the -C100 limit"); + is($slurp->("$d5/moved.c.blame"), $slurp->("$d4/moved.c.blame"), + "the flag and the environment variable give the same bytes"); + + # (c) git fails: a non-zero exit and no blame file. + my ($s6, $d6, undef, $err6) = $run->('fail', ''); + isnt($s6, 0, "a failing git blame gives a non-zero exit"); + ok(!-e "$d6/moved.c.blame", "a failing git blame writes no blame file"); + like($err6, qr/^Error: git blame of \[moved\.c\]/m, "a failing git blame prints an error"); + + # git killed after part of its output: the defect that wrote an empty .blame. + my ($s7, $d7) = $run->('killed', ''); + isnt($s7, 0, "a killed git blame gives a non-zero exit"); + ok(!-e "$d7/moved.c.blame", "a killed git blame writes no blame file"); + + # -C100 passes the limit and plain blame then fails. + my ($s8, $d8, undef, $err8) = $run->('slow-c100-plain-fails', '--c100-limit=1'); + isnt($s8, 0, "a failing plain blame after the limit gives a non-zero exit"); + ok(!-e "$d8/moved.c.blame", "a failing plain blame writes no blame file"); + + my @tempsAfter = glob("$buildDir/tmpfile-*"); + is(scalar(@tempsAfter), scalar(@tempsBefore), "the failed runs leave no temp file in build/"); + + # A limit that is not a whole number of seconds is rejected. + my ($s9) = $run->('none', '--c100-limit=abc'); + isnt($s9, 0, "a limit that is not a number is rejected"); +} + # Two non-contiguous groups from one commit, in a renamed file: --porcelain # suppresses the second group's header, and field 2 must survive that. { From 2d8af90bc9c4059dc8e1465fc00d94a96fdc358c Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Wed, 14 Oct 2026 14:48:20 +0000 Subject: [PATCH 05/10] fix(blame): use plain blame after a git exit 128 of -C100 git blame -C100 can stop with "fatal: unable to generate diff" and exit 128. This occurs when a commit moves a blob over 1 GiB: git reads the blob as a copy candidate, xdiff refuses a buffer over 1 GiB, and git does not skip the candidate. The census found this in one project (gitlab.inria.fr/cedar/connectionlens, a 1.33 GB model file). Since 57551d7, a git failure stops the project, so the project failed. After a -C100 exit 128, formatBlame.pl now runs plain blame, the same path as after the -C100 time limit. It prints "PLAIN-BLAME after git exit 128 of -C100" to STDERR, so the file can be listed in LIMITATIONS. A git that a signal killed, or a plain blame that also fails, still gives a non-zero exit and no blame file. Tests: formatBlame.t has 5 new tests (46 in total); 3 of them fail on 57551d7. (cherry picked from commit 23e4ccb2919e435dc6636c958d7fd69147836a50) --- blameRepo/formatBlame.pl | 5 +++++ tests/t/formatBlame.t | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index 76d8a379..acb1e7e8 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -83,6 +83,11 @@ if (not defined $status) { printf STDERR "PLAIN-BLAME %s after %d s of -C100\n", $file, time() - $started; $status = Blame_To_Temp(0, '--porcelain'); +} elsif (($status & 127) == 0 and ($status >> 8) == 128) { + # git dies (exit 128) when -C100 must diff a candidate blob over 1 GiB (xdiff + # limit, "unable to generate diff"); a killed git keeps its signal and fails below. + printf STDERR "PLAIN-BLAME %s after git exit 128 of -C100\n", $file; + $status = Blame_To_Temp(0, '--porcelain'); } close $fh; # A git that stops early (killed, out of memory) gives a short or empty stream. diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 7b056d5c..54a949c0 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 41; +use Test::More tests => 46; use FindBin; use File::Temp qw(tempdir); use File::Basename qw(dirname); @@ -154,6 +154,8 @@ if [ \$blame = 1 ]; then slow-c100) [ \$c100 = 1 ] && exec sleep 60 ;; slow-c100-plain-fails) [ \$c100 = 1 ] && exec sleep 60; exit 3 ;; fail) echo "fatal: fake failure" >&2; exit 3 ;; + c100-exit128) [ \$c100 = 1 ] && { echo "fatal: unable to generate diff (fake)" >&2; exit 128; } ;; + exit128) echo "fatal: unable to generate diff (fake)" >&2; exit 128 ;; killed) printf '%040d 1 1 1\\n' 0; kill -9 \$\$ ;; esac fi @@ -230,6 +232,19 @@ FAKE isnt($s8, 0, "a failing plain blame after the limit gives a non-zero exit"); ok(!-e "$d8/moved.c.blame", "a failing plain blame writes no blame file"); + # (d) -C100 ends with git exit 128 (xdiff refuses a candidate blob over 1 GiB): plain blame. + my ($s9, $d9, undef, $err9) = $run->('c100-exit128', ''); + is($s9, 0, "a -C100 git exit 128 falls back to plain blame and succeeds"); + like($err9, qr/^PLAIN-BLAME moved\.c after git exit 128 of -C100$/m, + "the script prints the PLAIN-BLAME line for the git exit 128"); + is($slurp->("$d9/moved.c.blame"), $slurp->("$d4/moved.c.blame"), + "the exit 128 fallback gives the same bytes as the time-limit fallback"); + + # Plain blame also ends with exit 128: a non-zero exit and no blame file. + my ($s10, $d10) = $run->('exit128', ''); + isnt($s10, 0, "a plain blame exit 128 after the fallback gives a non-zero exit"); + ok(!-e "$d10/moved.c.blame", "a failing fallback writes no blame file"); + my @tempsAfter = glob("$buildDir/tmpfile-*"); is(scalar(@tempsAfter), scalar(@tempsBefore), "the failed runs leave no temp file in build/"); From eac32d927585b9f3a8ee2db23262cf0cde9bea61 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Wed, 14 Oct 2026 21:51:09 +0000 Subject: [PATCH 06/10] fix(blame): lock each output file, skip a finished file Two blameRepoFiles.pl runs can now use the same output directory. formatBlame.pl takes an exclusive lock on build/locks/.lock and holds it until it exits. When the lock is held and the .blame file already exists, the script exits 0 and does not start git. Otherwise the blame runs as before, with the same bytes. The second run can then blame the files that the first run has not reached yet, and the first run skips each file that the second run finished. The locks stay in build/, out of the output directory. (cherry picked from commit 365a47a1c81995876200bbc5e503325843f097a2) --- blameRepo/formatBlame.pl | 17 +++++++++ tests/t/formatBlame.t | 77 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index acb1e7e8..db2a9648 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -23,6 +23,9 @@ use Pod::Usage; use FindBin qw($RealBin); use POSIX qw(_exit); +use Cwd qw(abs_path); +use Digest::SHA qw(sha1_hex); +use Fcntl qw(:flock); # build/ tempdir anchored to the script's dir, not the caller's CWD. my $buildDir = "$RealBin/build"; @@ -67,6 +70,20 @@ Usage( "Error [$repo] should be a git repo\n\nUsage $0 ") unless -d "$repo/.git"; Usage( "Error [$dest] should be a directory\n\nUsage $0 ") unless -d $dest and $dest ne ""; +# Two runners can blame into the same output directory. One lock per output file, +# held until exit; the second runner skips a file that the first one finished. +my $output = abs_path($dest) . "/" . $file . $blameExtension; +my $lockDir = "$buildDir/locks"; +make_path($lockDir) if not -d $lockDir; +open(my $lockFh, '>>', "$lockDir/" . sha1_hex($output) . ".lock") + or die "unable to open the lock of [$output]: $!"; +flock($lockFh, LOCK_EX) or die "unable to lock [$output]: $!"; +if (-f $output) { + close $fh; + unlink($temp); + exit(0); +} + if ($verbose) { print STDERR "$0 processing repo [$repo] file [$file] [$dest]\n"; } diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 54a949c0..8c623c98 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,10 +2,13 @@ use strict; use warnings; -use Test::More tests => 46; +use Test::More tests => 56; use FindBin; use File::Temp qw(tempdir); use File::Basename qw(dirname); +use Cwd qw(abs_path); +use Digest::SHA qw(sha1_hex); +use POSIX (); my $root = "$FindBin::Bin/../.."; # tests/t -> repo root @@ -285,3 +288,75 @@ FAKE is($lines[2], "$first;old.c;\tint three;", "the second group of the same commit still names the old filename"); } + +# Two runners on one output directory: a lock per output file, and a skip when +# the file is already there. A fake git counts the blames and makes them slow. +{ + my $realGit = `command -v git`; + chomp $realGit; + my $lockBin = "$workdir/lockbin"; + mkdir $lockBin or die $!; + my $counter = "$workdir/blame-count"; + write_file("$lockBin/git", <<"FAKE"); +#!/bin/sh +for a in "\$@"; do + if [ "\$a" = blame ]; then echo x >> '$counter'; sleep 2; fi +done +exec '$realGit' "\$@" +FAKE + chmod 0755, "$lockBin/git" or die $!; + my $count = sub { + return 0 unless -f $counter; + my @l = slurp_lines($counter); + unlink $counter; + return scalar(@l); + }; + my $slurp = sub { + open(my $fh, '<', $_[0]) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; + }; + local $ENV{PATH} = "$lockBin:$ENV{PATH}"; + + my $ref = tempdir(CLEANUP => 1); + system("perl '$script' '$repo' g.c '$ref' 2>/dev/null") == 0 or die "reference run failed"; + $count->(); + my $refBytes = $slurp->("$ref/g.c.blame"); + + # (a) The output file exists: no blame, and the file keeps its bytes. + my $done = tempdir(CLEANUP => 1); + write_file("$done/g.c.blame", "keep\n"); + my $s0 = system("perl '$script' '$repo' g.c '$done' 2>/dev/null"); + is($s0, 0, "a file that is already blamed gives exit 0"); + is($slurp->("$done/g.c.blame"), "keep\n", "an existing blame file keeps its bytes"); + is($count->(), 0, "an existing blame file starts no git blame"); + + # (b) Two processes on the same file at the same time: one blames, one skips. + my $both = tempdir(CLEANUP => 1); + my @pids; + for my $i (1, 2) { + my $pid = fork(); + die "fork: $!" unless defined $pid; + if ($pid == 0) { + exec("perl '$script' '$repo' g.c '$both' 2>/dev/null") or POSIX::_exit(127); + } + push @pids, $pid; + select(undef, undef, undef, 0.3); + } + my @st = map { waitpid($_, 0); $? } @pids; + is($st[0], 0, "the first of two runners gives exit 0"); + is($st[1], 0, "the second of two runners gives exit 0"); + is($count->(), 1, "two runners on one file start one git blame"); + is($slurp->("$both/g.c.blame"), $refBytes, "the blame file of two runners equals a single run"); + ok(-s "$both/g.c.blame", "the blame file of two runners is not empty"); + + # (c) The locks stay in build/, out of the output directory. + my $lock = dirname($script) . "/build/locks/" . sha1_hex(abs_path($both) . "/g.c.blame") . ".lock"; + ok(-f $lock, "the lock file is in build/locks/"); + opendir(my $dh, $both) or die $!; + my @entries = sort grep { !/^\.\.?$/ } readdir($dh); + closedir $dh; + is_deeply(\@entries, ["g.c.blame"], "the output directory holds only the blame file"); +} From 487899902d69116ab81ce52baef309e79cacdd20 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Tue, 20 Oct 2026 20:56:53 +0000 Subject: [PATCH 07/10] fix(blame): read raw paths, end options with -- blameRepoFiles.pl read `git ls-files` without -z. git C-quotes a path with a byte >= 0x80, '"', '\' or a control byte (core.quotePath), so the quoted name did not match the mask and the file was silently not blamed: it had no token rows. Now it reads `ls-files -z`. blameRepoFiles.pl gave the path to formatBlame.pl, and formatBlame.pl gave it to git blame, with no '--' before it. A path that starts with '-' was read as an option, and step 7 exited 1. Now both pass '--'. git blame --porcelain also C-quotes "filename". formatBlame.pl now compares the raw bytes with the blamed path, so a quoted file does not name itself in field 2. Field 2 keeps git's quoted form, as before. The libexpat gate Parquet is unchanged (14eb5d72, 2,196,248 bytes). tests/t/blamePaths.t covers a non-ASCII byte, '"', '\', a tab, a newline, a single quote, a leading '-' file and folder, and renames. (cherry picked from commit 14b8db468e4f6565fab5692a1ab1c3066d12f371) --- blameRepo/blameRepoFiles.pl | 12 ++- blameRepo/formatBlame.pl | 29 +++++-- tests/t/blamePaths.t | 159 ++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 9 deletions(-) create mode 100644 tests/t/blamePaths.t diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index 7036ef5a..544cf63d 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -62,9 +62,12 @@ -open(my $files, '-|', 'git', '-C', $repoDir, 'ls-files') +# -z: raw paths, each ended by a NUL. Without -z, git C-quotes a path that has a +# byte >= 0x80, '"', '\' or a control byte; the quoted name does not match the +# mask, and the file is silently not blamed. +open(my $files, '-|', 'git', '-C', $repoDir, 'ls-files', '-z') or die "unable to traverse git repo [$repoDir] $!"; -my @trackedFiles = <$files>; +my @trackedFiles = do { local $/ = "\0"; map { s/\0\z//r } <$files> }; close($files) or die "unable to traverse git repo [$repoDir]"; my $count = 0; @@ -75,7 +78,7 @@ foreach (@trackedFiles) { # next unless /^kernel/; - chomp; + # No chomp: the NUL is gone, and a newline can be part of a raw path. if ($fileRegExpr ne "") { next unless /$fileRegExpr/; @@ -124,7 +127,8 @@ $count++; print STDERR ("$count: $name\n"); - start_command($name, $blameCommand, "--blameExtension=$blameExtension", $repoDir, $name, $outputDir); + # '--' ends the options: a path that starts with '-' is not read as one. + start_command($name, $blameCommand, "--blameExtension=$blameExtension", '--', $repoDir, $name, $outputDir); } reap_command() while %running; diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index db2a9648..84e8e676 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -130,7 +130,8 @@ sub Blame_To_Temp { my $pid = open(IN, '-|'); die "unable to start git blame: $!" unless defined $pid; if ($pid == 0) { - exec('git', '-C', $repo, 'blame', @options, $file); + # '--': a path that starts with '-' is a path, not an option. + exec('git', '-C', $repo, 'blame', @options, '--', $file); print STDERR "unable to execute git: $!\n"; _exit(127); } @@ -167,17 +168,20 @@ sub Read_Record { $f = "$1;"; $sawFilename = 0; } elsif ($_ =~ /^(filename) (.+)$/) { - $filenameOfCommit{$cid} = $2 if defined $cid; + # As git prints it: C-quoted if needed, so never a raw tab or newline. + # Field 2 keeps this form; only the compare uses the raw bytes. + my $name = $2; + $filenameOfCommit{$cid} = $name if defined $cid; $sawFilename = 1; - if ($2 ne $file) { - $f .= $2 . ";"; + if (Unquote_Path($name) ne $file) { + $f .= $name . ";"; } else { $f .= ";"; } } elsif (/^ (.*)/) { #actual line if (not $sawFilename) { my $known = defined $cid ? $filenameOfCommit{$cid} : undef; - if (defined $known and $known ne $file) { + if (defined $known and Unquote_Path($known) ne $file) { $f .= $known . ";"; } else { $f .= ";"; @@ -192,6 +196,21 @@ sub Read_Record { return $f; } +# git blame --porcelain prints "filename" C-quoted (core.quotePath) when the path +# has a byte >= 0x80, '"', '\' or a control byte. Gives the raw bytes back, as +# the path is on the command line, for the compare with $file. A path that git +# did not quote never starts with '"' (git quotes every path that has one), so +# it comes back unchanged. +sub Unquote_Path { + my ($s) = @_; + return $s unless $s =~ /^"(.*)"$/s; + my $q = $1; + my %escape = ('a' => "\a", 'b' => "\b", 't' => "\t", 'n' => "\n", + 'v' => "\x0b", 'f' => "\f", 'r' => "\r", '"' => '"', '\\' => '\\'); + $q =~ s/\\([0-3][0-7][0-7]|[abtnvfr"\\])/length($1) == 3 ? chr(oct($1)) : $escape{$1}/ge; + return $q; +} + sub Usage { my ($m) = @_; print STDERR "$m\n"; diff --git a/tests/t/blamePaths.t b/tests/t/blamePaths.t new file mode 100644 index 00000000..fee49aa7 --- /dev/null +++ b/tests/t/blamePaths.t @@ -0,0 +1,159 @@ +#!/usr/bin/env perl + +# Paths that git quotes (a byte >= 0x80, '"', '\', a tab, a newline) and paths +# that start with '-' must be blamed like any other path. Before the fix, +# blameRepoFiles.pl read `git ls-files` without -z (a quoted path did not match +# the mask, so the file was silently not blamed) and passed the path to +# formatBlame.pl and git without '--' (a leading '-' was read as an option). + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $root = "$FindBin::Bin/../.."; # tests/t -> repo root +my $script = "$root/blameRepo/blameRepoFiles.pl"; +my $format = "$root/blameRepo/formatBlame.pl"; +my $workdir = tempdir(CLEANUP => 1); + +$ENV{GIT_CONFIG_NOSYSTEM} = 1; +$ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; +$ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_COMMITTER_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_AUTHOR_NAME} = 'Alice'; +$ENV{GIT_AUTHOR_EMAIL} = 'alice@example.com'; +$ENV{GIT_COMMITTER_NAME} = 'Alice'; +$ENV{GIT_COMMITTER_EMAIL} = 'alice@example.com'; + +# List form, no shell: the paths below hold quotes, tabs and newlines. +sub git { + my ($repo, @args) = @_; + system('git', '-C', $repo, @args) == 0 or die "git failed: @args"; +} + +sub git_out { + my ($repo, @args) = @_; + open(my $fh, '-|', 'git', '-C', $repo, @args) or die "git failed: @args"; + local $/; + my $out = <$fh>; + close $fh or die "git failed: @args"; + return $out; +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die "$path: $!"; + print $fh $content; + close $fh; +} + +sub slurp { + my ($path) = @_; + open(my $fh, '<', $path) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; +} + +# Bytes, not characters: git and the scripts see raw bytes. +my $eAcute = "\xc3\xa9"; +my $uUml = "\xc3\xbc"; +my @hard = ( + "caf${eAcute}.c", # non-ASCII byte + "dir ${uUml}/x.h", # non-ASCII byte in a folder name + "q\"uote.c", # double quote + "back\\slash.c", # backslash + "tab\there.c", # tab + "new\nline.c", # newline + "it's.c", # single quote (no quoting by git) + "-dash.c", # leading '-' + "---- Books & Notes ----/notes.c", # leading '-' folder + "a/-inner.c", # '-' in a later segment +); +my @plain = ("normal.c"); + +my $repo = "$workdir/repo"; +mkdir $repo or die $!; +git($repo, 'init', '-q', '-b', 'main'); +my $n = 0; +foreach my $p (@hard, @plain) { + my $full = "$repo/$p"; + (my $dir = $full) =~ s{/[^/]*$}{}; + system('mkdir', '-p', '--', $dir) == 0 or die "mkdir $dir"; + $n++; + write_file($full, "int v$n;\nint w$n;\n"); +} +write_file("$repo/notes.txt", "not source\n"); +git($repo, 'add', '-A'); +git($repo, 'commit', '-q', '-m', 'first'); + +# Sanity: git really quotes the hard paths in plain ls-files output. +my $listed = git_out($repo, 'ls-files'); +like($listed, qr/^"caf\\303\\251\.c"$/m, "git ls-files quotes a non-ASCII path (the defect input)"); +like($listed, qr/^"tab\\there\.c"$/m, "git ls-files quotes a tab"); + +my $out = "$workdir/blame-out"; +mkdir $out or die $!; +my $stdout = `perl '$script' '$repo' '$out' '\\.(c|h)\$' 2>'$workdir/stderr'`; +my $status = $?; +my $stderr = slurp("$workdir/stderr"); +is($status, 0, "blameRepoFiles.pl succeeds on hard paths") or diag($stderr); +my $total = scalar(@hard) + scalar(@plain); +like($stdout, qr/Newly processed \[$total\] Already done \[0\] files Error \[0\]/, + "every mask-matched file is processed, none fails"); + +foreach my $p (@hard, @plain) { + (my $shown = $p) =~ s/([^\x20-\x7e])/sprintf('\\x%02x', ord($1))/ge; + my $b = slurp("$out/$p.blame"); + ok(defined $b, "blame file exists for [$shown]"); + my @lines = defined $b ? split(/\n/, $b) : (); + is(scalar(@lines), 2, "two blame lines for [$shown]"); + # Field 2 is empty: the lines come from this file, not from another one. + like($lines[0] // '', qr/^[0-9a-f]{40};;\tint v[0-9]+;$/, "record format for [$shown]"); +} +ok(!-e "$out/notes.txt.blame", "the mask still filters out notes.txt"); + +# A second run finds every hard path as done (the -f check uses the raw path). +my $again = `perl '$script' '$repo' '$out' '\\.(c|h)\$' 2>/dev/null`; +like($again, qr/Newly processed \[0\] Already done \[$total\] files Error \[0\]/, + "a second run skips every hard path as done"); + +# formatBlame.pl alone, with a path that starts with '-'. +{ + my $o2 = "$workdir/fb-out"; + mkdir $o2 or die $!; + system('perl', $format, '--', $repo, '-dash.c', $o2) == 0 + or fail("formatBlame.pl -- -dash.c fails"); + ok(-s "$o2/-dash.c.blame", "formatBlame.pl blames a path that starts with '-'"); +} + +# git blame --porcelain quotes "filename" too. The compare with the blamed path +# uses the raw bytes (above: a hard path never names itself in field 2). Field 2 +# keeps git's quoted form, as before the fix: it never holds a raw newline or tab, +# so a record stays on one line. +foreach my $case (["old${eAcute}.c", "\"old\\303\\251.c\""], + ["old\nname.c", "\"old\\nname.c\""]) { + my ($old, $quoted) = @$case; + (my $shown = $old) =~ s/([^\x20-\x7e])/sprintf('\\x%02x', ord($1))/ge; + my $r = "$workdir/renamed-" . length($old) . "-" . ord(substr($old, 3, 1)); + mkdir $r or die $!; + git($r, 'init', '-q', '-b', 'main'); + write_file("$r/$old", join('', map { "int line$_;\n" } 1 .. 6)); + git($r, 'add', '-A'); + git($r, 'commit', '-q', '-m', 'add'); + git($r, 'mv', '--', $old, "new${uUml}.c"); + git($r, 'commit', '-q', '-m', 'rename'); + my $o3 = "$r-out"; + mkdir $o3 or die $!; + my $s = system('perl', $script, $r, $o3, '\.c$'); + is($s, 0, "blameRepoFiles.pl succeeds on the repo renamed from [$shown]"); + my $b = slurp("$o3/new${uUml}.c.blame") // ''; + my @lines = split(/\n/, $b); + is(scalar(@lines), 6, "six blame lines, one per record, after a rename from [$shown]"); + is(scalar(grep { /^[0-9a-f]{40};\Q$quoted\E;\tint line[0-9];$/ } @lines), 6, + "field 2 is git's quoted form of [$shown]"); +} + +done_testing(); From 4a3d518cf87eb3c3933d33098f8ebc969cae4789 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Wed, 28 Oct 2026 20:21:41 +0000 Subject: [PATCH 08/10] test(blame): find git without the command builtin formatBlame.t found the real git with `command -v git` in backticks. Perl does not use a shell for a command with no shell characters, so it runs the program "command" directly. On the Ubuntu CI runner, "command" is only a shell builtin. The exec failed, $realGit was empty, each blame through the fake git failed, and the file stopped after 46 of its 56 tests. Amazon Linux has /usr/bin/command (a script from the bash package), so the test passed there. Find git with a PATH search in Perl (File::Spec) instead. --- tests/t/formatBlame.t | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 8c623c98..843387a5 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -6,6 +6,7 @@ use Test::More tests => 56; use FindBin; use File::Temp qw(tempdir); use File::Basename qw(dirname); +use File::Spec; use Cwd qw(abs_path); use Digest::SHA qw(sha1_hex); use POSIX (); @@ -54,6 +55,17 @@ sub slurp_lines { return @lines; } +# Find a program on PATH in Perl. Do not run `command -v`: Perl runs a command +# with no shell characters directly, and `command` is often only a shell builtin. +sub find_on_path { + my ($name) = @_; + for my $dir (File::Spec->path) { + my $file = File::Spec->catfile($dir, $name); + return $file if -f $file && -x _; + } + die "$name is not on PATH"; +} + my $repo = "$workdir/repo"; mkdir $repo or die $!; git($repo, "init -q -b main"); @@ -141,8 +153,7 @@ my $cid2 = commit_as($repo, "Bob", "second"); { my $repo2 = "$workdir/moved"; my $mover = git($repo2, "rev-parse HEAD"); - my $realGit = `command -v git`; - chomp $realGit; + my $realGit = find_on_path('git'); my $fakeBin = "$workdir/fakebin"; mkdir $fakeBin or die $!; write_file("$fakeBin/git", <<"FAKE"); @@ -292,8 +303,7 @@ FAKE # Two runners on one output directory: a lock per output file, and a skip when # the file is already there. A fake git counts the blames and makes them slow. { - my $realGit = `command -v git`; - chomp $realGit; + my $realGit = find_on_path('git'); my $lockBin = "$workdir/lockbin"; mkdir $lockBin or die $!; my $counter = "$workdir/blame-count"; From 205a55dac7fb9e0f29c46880be9a5c8fbc7e9b96 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Fri, 13 Nov 2026 20:44:13 +0000 Subject: [PATCH 09/10] fix(blame): write each .blame file whole or not at all Check every write, close, copy and rename. On failure, print a message, remove the partial output and exit 1. Copy to a temporary name in the output folder and rename it, so an existing output file is complete. Add --overwrite, and pass it from blameRepoFiles.pl, so a re-blame replaces a file. Run the plain-blame fallback only for the xdiff failure: exit 128 with "unable to generate diff" on stderr. --- blameRepo/blameRepoFiles.pl | 2 +- blameRepo/formatBlame.pl | 70 +++++++++++++++++++++++++++++-------- tests/t/formatBlame.t | 38 +++++++++++++++++++- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index 544cf63d..cd64e00a 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -128,7 +128,7 @@ print STDERR ("$count: $name\n"); # '--' ends the options: a path that starts with '-' is not read as one. - start_command($name, $blameCommand, "--blameExtension=$blameExtension", '--', $repoDir, $name, $outputDir); + start_command($name, $blameCommand, "--blameExtension=$blameExtension", ($overwrite ? ('--overwrite') : ()), '--', $repoDir, $name, $outputDir); } reap_command() while %running; diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index 84e8e676..a2f15f13 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -35,6 +35,7 @@ my $help = 0; my $man = 0; my $verbose = 0; +my $overwrite = 0; # Wall-clock limit in seconds for blame -C100. 0 means no limit. # blameRepoFiles.pl starts this script, so CREGIT_C100_LIMIT reaches it with no new flag. my $c100Limit = (defined $ENV{CREGIT_C100_LIMIT} and $ENV{CREGIT_C100_LIMIT} ne '') @@ -42,6 +43,7 @@ GetOptions ("blameExtension=s" => \$blameExtension, "c100-limit=s" => \$c100Limit, + "overwrite" => \$overwrite, "help" => \$help, # string "verbose" => \$verbose, "man" => \$man) # flag @@ -78,7 +80,7 @@ open(my $lockFh, '>>', "$lockDir/" . sha1_hex($output) . ".lock") or die "unable to open the lock of [$output]: $!"; flock($lockFh, LOCK_EX) or die "unable to lock [$output]: $!"; -if (-f $output) { +if (-f $output and not $overwrite) { close $fh; unlink($temp); exit(0); @@ -90,6 +92,7 @@ # Suppressed headers still owe field 2 a filename. my %filenameOfCommit; +my $gitStderr = ''; # -C100: credit a line moved in from another file to its author, not the mover. # --porcelain: same shas, but the header once per commit instead of once per line. @@ -100,21 +103,25 @@ if (not defined $status) { printf STDERR "PLAIN-BLAME %s after %d s of -C100\n", $file, time() - $started; $status = Blame_To_Temp(0, '--porcelain'); -} elsif (($status & 127) == 0 and ($status >> 8) == 128) { +} elsif (($status & 127) == 0 and ($status >> 8) == 128 + and $gitStderr =~ /unable to generate diff/) { # git dies (exit 128) when -C100 must diff a candidate blob over 1 GiB (xdiff - # limit, "unable to generate diff"); a killed git keeps its signal and fails below. + # limit, "unable to generate diff"). Any other exit 128, and a killed git, + # fail below. printf STDERR "PLAIN-BLAME %s after git exit 128 of -C100\n", $file; $status = Blame_To_Temp(0, '--porcelain'); } -close $fh; +close($fh) or Fail("unable to close [$temp]: $!"); # A git that stops early (killed, out of memory) gives a short or empty stream. # Do not keep it: no .blame file, and a non-zero exit that blameRepoFiles.pl counts. if ($status != 0) { unlink($temp); + unlink("$temp.err"); print STDERR "Error: git blame of [$file] in [$repo] failed with status [$status]; no blame file written\n"; exit(1); } copy_file($temp, $dest, $file . $blameExtension); +unlink($temp); if ($verbose) { print STDERR "...completed\n"; } @@ -127,9 +134,14 @@ sub Blame_To_Temp { %filenameOfCommit = (); seek($fh, 0, 0) and truncate($fh, 0) or die "unable to empty [$temp]: $!"; # fork + exec, not a shell, so the pid and the exit status are those of git. + # The stderr of git goes to a file, so the script can tell the xdiff failure + # from another exit 128. It is printed again to STDERR after git ends. + my $errFile = "$temp.err"; + $gitStderr = ''; my $pid = open(IN, '-|'); die "unable to start git blame: $!" unless defined $pid; if ($pid == 0) { + open(STDERR, '>', $errFile) or _exit(127); # '--': a path that starts with '-' is a path, not an option. exec('git', '-C', $repo, 'blame', @options, '--', $file); print STDERR "unable to execute git: $!\n"; @@ -140,8 +152,8 @@ sub Blame_To_Temp { local $SIG{ALRM} = sub { die "c100-limit\n" }; alarm($limit); while (my $l = Read_Record()) { - print $fh $l; - print $fh "\n"; + print $fh $l or Fail("unable to write [$temp]: $!"); + print $fh "\n" or Fail("unable to write [$temp]: $!"); } alarm(0); }; @@ -153,6 +165,14 @@ sub Blame_To_Temp { } close IN; my $waitStatus = $?; + if (open(my $efh, '<', $errFile)) { + local $/; + $gitStderr = <$efh>; + $gitStderr = '' unless defined $gitStderr; + close $efh; + print STDERR $gitStderr; + } + unlink($errFile); return undef if $timedOut; return $waitStatus; } @@ -218,23 +238,41 @@ sub Usage { exit(1); } +# Removes the partial output and stops with a non-zero exit. +sub Fail { + my ($m) = @_; + unlink($temp, "$temp.err"); + print STDERR "Error: $m; no blame file written\n"; + exit(1); +} + sub copy_file { my ($from, $toDir, $toName) = @_; - + my $to = "${toDir}/$toName"; $toDir = dirname($to); -# printf ("copy [$from] to [$to] [$toDir][$toName]\n"); - die "from file does not exist in copy_file [$from]" if not -f $from; if (not -d $toDir) { printf("Creating directory [$toDir]\n"); - make_path($toDir) or "die unable to create directory $to"; - } - move($from, $to) or - (unlink($to), "unable to move [$from] to [$to]"); + make_path($toDir); + Fail("unable to create directory [$toDir]") unless -d $toDir; + } + # Same folder as the output, so the rename is atomic: an existing output + # file is always complete. + my $partial = "$to.partial.$$"; + if (not copy($from, $partial)) { + my $err = $!; + unlink($partial); + Fail("unable to copy [$from] to [$partial]: $err"); + } + if (not rename($partial, $to)) { + my $err = $!; + unlink($partial); + Fail("unable to move [$partial] to [$to]: $err"); + } } __END__ @@ -254,6 +292,8 @@ =head1 SYNOPSIS script kills it and runs git blame without -C. Default is $CREGIT_C100_LIMIT, else 3600. 0 means no limit. + --overwrite replace an existing output file. Without it, the + script skips a file that already exists. --help brief help message --man full documentation @@ -278,6 +318,8 @@ =head1 DESCRIPTION If git blame -C100 does not end within the limit, the script runs git blame without -C and prints "PLAIN-BLAME after s of -C100" -to STDERR. If git fails, the script writes no output file and exits 1. +to STDERR. The script does this only after the xdiff failure of git +(exit 128 with "unable to generate diff" on STDERR); any other exit 128 is a +failure. If git fails, the script writes no output file and exits 1. =cut diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 843387a5..dc1f539c 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 56; +use Test::More tests => 67; use FindBin; use File::Temp qw(tempdir); use File::Basename qw(dirname); @@ -169,6 +169,7 @@ if [ \$blame = 1 ]; then slow-c100-plain-fails) [ \$c100 = 1 ] && exec sleep 60; exit 3 ;; fail) echo "fatal: fake failure" >&2; exit 3 ;; c100-exit128) [ \$c100 = 1 ] && { echo "fatal: unable to generate diff (fake)" >&2; exit 128; } ;; + c100-exit128-other) [ \$c100 = 1 ] && { echo "fatal: bad object (fake)" >&2; exit 128; } ;; exit128) echo "fatal: unable to generate diff (fake)" >&2; exit 128 ;; killed) printf '%040d 1 1 1\\n' 0; kill -9 \$\$ ;; esac @@ -259,6 +260,12 @@ FAKE isnt($s10, 0, "a plain blame exit 128 after the fallback gives a non-zero exit"); ok(!-e "$d10/moved.c.blame", "a failing fallback writes no blame file"); + # An exit 128 of -C100 that is not the xdiff failure is an error, not a fallback. + my ($s11, $d11, undef, $err11) = $run->('c100-exit128-other', ''); + isnt($s11, 0, "a -C100 exit 128 with another message gives a non-zero exit"); + ok(!-e "$d11/moved.c.blame", "a -C100 exit 128 with another message writes no blame file"); + unlike($err11, qr/PLAIN-BLAME/, "a -C100 exit 128 with another message prints no PLAIN-BLAME line"); + my @tempsAfter = glob("$buildDir/tmpfile-*"); is(scalar(@tempsAfter), scalar(@tempsBefore), "the failed runs leave no temp file in build/"); @@ -267,6 +274,35 @@ FAKE isnt($s9, 0, "a limit that is not a number is rejected"); } +# An existing output file: kept without --overwrite, replaced with it. A failed +# write leaves no partial file and a non-zero exit. +{ + my $repo2 = "$workdir/moved"; + my $dest = tempdir(CLEANUP => 1); + write_file("$dest/moved.c.blame", "stale\n"); + is(system("perl '$script' '$repo2' moved.c '$dest' 2>/dev/null"), 0, + "a run with an existing output file succeeds"); + is(join('', slurp_lines("$dest/moved.c.blame")), "stale", + "without --overwrite an existing output file is kept"); + is(system("perl '$script' --overwrite '$repo2' moved.c '$dest' 2>/dev/null"), 0, + "a run with --overwrite succeeds"); + my @fresh = slurp_lines("$dest/moved.c.blame"); + is(scalar(@fresh), 10, "--overwrite replaces the existing output file"); + my @partials = glob("$dest/*.partial.*"); + is(scalar(@partials), 0, "no partial file stays next to the output"); + + SKIP: { + skip "root can write to a read-only folder", 3 if $> == 0; + my $ro = tempdir(CLEANUP => 1); + chmod 0555, $ro or die $!; + my $out = `perl '$script' '$repo2' moved.c '$ro' 2>&1`; + isnt($? , 0, "an output folder that cannot be written gives a non-zero exit"); + ok(!-e "$ro/moved.c.blame", "no blame file is written to it"); + like($out, qr/^Error: unable to copy/m, "the script says that the copy failed"); + chmod 0755, $ro; + } +} + # Two non-contiguous groups from one commit, in a renamed file: --porcelain # suppresses the second group's header, and field 2 must survive that. { From 5e64983b93101b3107c71b766745246822335814 Mon Sep 17 00:00:00 2001 From: Ellian Carlos Date: Fri, 13 Nov 2026 20:46:39 +0000 Subject: [PATCH 10/10] test(dataset): use made-up identities, test the lowest personid Replace the real names and addresses of the examples in tests and DATASET.md. Add a test that fails when the lowest personid rule is removed. State the plain-blame rule, the atomic write and --overwrite in DATASET.md. --- generate_dataset/DATASET.md | 23 ++++++++++++++++--- tests/t/formatBlame.t | 26 ++++++++++++++++++++- tests/test_generate_dataset.py | 41 ++++++++++++++++++++++++---------- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index 324ba3dd..5d5e1414 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -38,7 +38,7 @@ ORDER BY tokens DESC; -- Tokens authored by a specific person SELECT file_path, source_line, source_text, token_type, token_value FROM 'dataset.parquet' -WHERE person_name = 'Linus Torvalds' +WHERE person_name = 'Ada Example' ORDER BY file_path, token_index; ``` @@ -126,8 +126,8 @@ JGit's `getFooterLines` of the original commit message. Each of the 13 typed lists holds **one entry per person**, not the raw trailer lines. A message can name one person twice, with the same text or with another -address (for example `Borislav Petkov (AMD) ` and -`Borislav Petkov `). In one list, two values are the same person +address (for example `Pat Example (Corp) ` and +`Pat Example `). In one list, two values are the same person when: - their addresses resolve to the same `personid`. The address is the one that @@ -363,6 +363,23 @@ it. The rows of the file do not record this. To find these files, search the pipeline log for `PLAIN-BLAME`. A file that ends within the limit gets the same blame bytes as before. +Git also exits 128 when `-C100` must diff a candidate blob over 1 GiB (xdiff +limit). The script then reads the stderr of git. Only when it holds +`unable to generate diff` does it run plain `git blame` and print +`PLAIN-BLAME after git exit 128 of -C100`. Any other exit 128 is a +failure, as below. + +The script writes each `.blame` file under a temporary name in the output +folder and renames it into place, so an existing `.blame` file is complete. A +failed write, close, copy or rename removes the partial file, prints an error +and exits 1. Without `--overwrite`, the script skips a file that exists; with +it, the script replaces the file. `blameRepoFiles.pl --overwrite` passes the +option on. + +Field 2 of a `.blame` record is the old file name as git prints it, with no +escape. A semicolon in the old name adds a field for a reader that splits the +record on `;`. The dataset has this form, so it stays. + If git fails or stops early for a different reason (for example, the kernel kills it for memory), `formatBlame.pl` writes no `.blame` file and exits 1. `blameRepoFiles.pl` counts the error, and the blame step fails. Before this diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index dc1f539c..ceb00bdd 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 67; +use Test::More tests => 70; use FindBin; use File::Temp qw(tempdir); use File::Basename qw(dirname); @@ -303,6 +303,30 @@ FAKE } } +# Field 2 is the old file name as git prints it. It is not escaped, so a +# semicolon in the old name adds a field when a reader splits on ";". This +# test records the behavior; the dataset was built with it. +{ + my $repo4 = "$workdir/semicolon"; + mkdir $repo4 or die $!; + git($repo4, "init -q -b main"); + write_file("$repo4/old;name.c", "int one;\n"); + git($repo4, "add ."); + my $first = commit_as($repo4, "Alice", "write the file"); + git($repo4, "mv 'old;name.c' new.c"); + commit_as($repo4, "Carol", "rename the file"); + + my $dest = tempdir(CLEANUP => 1); + is(system("perl '$script' '$repo4' new.c '$dest' 2>/dev/null"), 0, + "formatBlame.pl succeeds when the old name has a semicolon"); + my @lines = slurp_lines("$dest/new.c.blame"); + is($lines[0], "$first;old;name.c;\tint one;", + "field 2 keeps the semicolon of the old name, unescaped"); + my ($head) = split /\t/, $lines[0], 2; + is(scalar(split /;/, $head, -1), 4, + "a split on a semicolon gives one more field than a name without one"); +} + # Two non-contiguous groups from one commit, in a renamed file: --porcelain # suppresses the second group's header, and field 2 must survive that. { diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index 33575632..fba697aa 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -640,18 +640,35 @@ def test_an_identical_repeated_trailer_gives_one_entry(monkeypatch, tmp_path): def test_two_addresses_of_one_person_give_the_first_value(monkeypatch, tmp_path): - """The Linux case: "Borislav Petkov (AMD) " and - "Borislav Petkov " are one personid.""" + """The Linux case: "Pat Example (Corp) " and + "Pat Example " are one personid.""" duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( - ("Signed-off-by", "Borislav Petkov (AMD) "), - ("Signed-off-by", "Borislav Petkov "), - people=[("bp", "bp@alien8.de", "Borislav Petkov"), - ("bp", "bp@suse.de", "Borislav Petkov")])) + ("Signed-off-by", "Pat Example (Corp) "), + ("Signed-off-by", "Pat Example "), + people=[("pat", "pat@corp.example.com", "Pat Example"), + ("pat", "pat@home.example.com", "Pat Example")])) signed, ids = typed_lists_of(duckdb, out, "footer_signed_off_by", "footer_personids") - assert signed == ["Borislav Petkov (AMD) "] - assert ids == ["bp"] + assert signed == ["Pat Example (Corp) "] + assert ids == ["pat"] + + +def test_an_address_with_two_personids_uses_the_lowest(monkeypatch, tmp_path): + """Two emails rows name one address for two persons. The lowest personid is + the key, so the other address of that lowest person is the same person. + The higher personid is listed first in emails, so a rule that took the + first row, or the highest one, would give two entries.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "Sam One "), + ("Signed-off-by", "Sam Two "), + people=[("zed", "sam@example.com", "Sam One"), + ("abe", "sam@example.com", "Sam One"), + ("abe", "sam@other.example.com", "Sam Two")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["Sam One "] def test_two_addresses_of_two_persons_stay_two_entries(monkeypatch, tmp_path): @@ -691,12 +708,12 @@ def test_a_bare_address_and_a_bracketed_one_are_one_person(monkeypatch, tmp_path def test_a_value_with_no_address_is_never_merged(monkeypatch, tmp_path): duckdb = pytest.importorskip("duckdb") out = generate(monkeypatch, tmp_path, seed=seed_footers( - ("Reported-by", "Daniel Stenberg"), - ("Reported-by", "Daniel Stenberg"), - ("Reported-by", "daniel stenberg"))) + ("Reported-by", "Dana Example"), + ("Reported-by", "Dana Example"), + ("Reported-by", "dana example"))) [reported] = typed_lists_of(duckdb, out, "footer_reported_by") - assert reported == ["Daniel Stenberg", "Daniel Stenberg", "daniel stenberg"] + assert reported == ["Dana Example", "Dana Example", "dana example"] def test_the_entries_keep_the_order_of_their_first_idx(monkeypatch, tmp_path):