diff --git a/blameRepo/blameRepoFiles.pl b/blameRepo/blameRepoFiles.pl index 7036ef5a..cd64e00a 100755 --- a/blameRepo/blameRepoFiles.pl +++ b/blameRepo/blameRepoFiles.pl @@ -62,9 +62,12 @@ -open(my $files, '-|', 'git', '-C', $repoDir, 'ls-files') +# -z: raw paths, each ended by a NUL. Without -z, git C-quotes a path that has a +# byte >= 0x80, '"', '\' or a control byte; the quoted name does not match the +# mask, and the file is silently not blamed. +open(my $files, '-|', 'git', '-C', $repoDir, 'ls-files', '-z') or die "unable to traverse git repo [$repoDir] $!"; -my @trackedFiles = <$files>; +my @trackedFiles = do { local $/ = "\0"; map { s/\0\z//r } <$files> }; close($files) or die "unable to traverse git repo [$repoDir]"; my $count = 0; @@ -75,7 +78,7 @@ foreach (@trackedFiles) { # next unless /^kernel/; - chomp; + # No chomp: the NUL is gone, and a newline can be part of a raw path. if ($fileRegExpr ne "") { next unless /$fileRegExpr/; @@ -124,7 +127,8 @@ $count++; print STDERR ("$count: $name\n"); - start_command($name, $blameCommand, "--blameExtension=$blameExtension", $repoDir, $name, $outputDir); + # '--' ends the options: a path that starts with '-' is not read as one. + start_command($name, $blameCommand, "--blameExtension=$blameExtension", ($overwrite ? ('--overwrite') : ()), '--', $repoDir, $name, $outputDir); } reap_command() while %running; diff --git a/blameRepo/formatBlame.pl b/blameRepo/formatBlame.pl index 7f028348..a2f15f13 100755 --- a/blameRepo/formatBlame.pl +++ b/blameRepo/formatBlame.pl @@ -22,6 +22,10 @@ use Getopt::Long; use Pod::Usage; use FindBin qw($RealBin); +use POSIX qw(_exit); +use Cwd qw(abs_path); +use Digest::SHA qw(sha1_hex); +use Fcntl qw(:flock); # build/ tempdir anchored to the script's dir, not the caller's CWD. my $buildDir = "$RealBin/build"; @@ -31,13 +35,23 @@ my $help = 0; my $man = 0; my $verbose = 0; +my $overwrite = 0; +# Wall-clock limit in seconds for blame -C100. 0 means no limit. +# blameRepoFiles.pl starts this script, so CREGIT_C100_LIMIT reaches it with no new flag. +my $c100Limit = (defined $ENV{CREGIT_C100_LIMIT} and $ENV{CREGIT_C100_LIMIT} ne '') + ? $ENV{CREGIT_C100_LIMIT} : 3600; GetOptions ("blameExtension=s" => \$blameExtension, + "c100-limit=s" => \$c100Limit, + "overwrite" => \$overwrite, "help" => \$help, # string "verbose" => \$verbose, "man" => \$man) # flag or die("Error in command line arguments\n"); +die "Error: the -C100 limit must be a whole number of seconds, not [$c100Limit]\n" + unless $c100Limit =~ /^[0-9]+$/; + if ($man) { pod2usage(-verbose=>2); exit(1) @@ -58,27 +72,110 @@ Usage( "Error [$repo] should be a git repo\n\nUsage $0 ") unless -d "$repo/.git"; Usage( "Error [$dest] should be a directory\n\nUsage $0 ") unless -d $dest and $dest ne ""; +# Two runners can blame into the same output directory. One lock per output file, +# held until exit; the second runner skips a file that the first one finished. +my $output = abs_path($dest) . "/" . $file . $blameExtension; +my $lockDir = "$buildDir/locks"; +make_path($lockDir) if not -d $lockDir; +open(my $lockFh, '>>', "$lockDir/" . sha1_hex($output) . ".lock") + or die "unable to open the lock of [$output]: $!"; +flock($lockFh, LOCK_EX) or die "unable to lock [$output]: $!"; +if (-f $output and not $overwrite) { + close $fh; + unlink($temp); + exit(0); +} + if ($verbose) { print STDERR "$0 processing repo [$repo] file [$file] [$dest]\n"; } +# Suppressed headers still owe field 2 a filename. +my %filenameOfCommit; +my $gitStderr = ''; + # -C100: credit a line moved in from another file to its author, not the mover. # --porcelain: same shas, but the header once per commit instead of once per line. -open(IN, "git -C '$repo' blame -C100 --porcelain '$file'|" ) or "unable to execute git "; -while (my $l = Read_Record()) { - print $fh $l; - print $fh "\n"; +# A huge generated file can take hours with -C100. After $c100Limit seconds, the +# script stops it and runs plain blame on the same file, with no limit. +my $started = time(); +my $status = Blame_To_Temp($c100Limit, '-C100', '--porcelain'); +if (not defined $status) { + printf STDERR "PLAIN-BLAME %s after %d s of -C100\n", $file, time() - $started; + $status = Blame_To_Temp(0, '--porcelain'); +} elsif (($status & 127) == 0 and ($status >> 8) == 128 + and $gitStderr =~ /unable to generate diff/) { + # git dies (exit 128) when -C100 must diff a candidate blob over 1 GiB (xdiff + # limit, "unable to generate diff"). Any other exit 128, and a killed git, + # fail below. + printf STDERR "PLAIN-BLAME %s after git exit 128 of -C100\n", $file; + $status = Blame_To_Temp(0, '--porcelain'); +} +close($fh) or Fail("unable to close [$temp]: $!"); +# A git that stops early (killed, out of memory) gives a short or empty stream. +# Do not keep it: no .blame file, and a non-zero exit that blameRepoFiles.pl counts. +if ($status != 0) { + unlink($temp); + unlink("$temp.err"); + print STDERR "Error: git blame of [$file] in [$repo] failed with status [$status]; no blame file written\n"; + exit(1); } -close IN; -close $fh; copy_file($temp, $dest, $file . $blameExtension); +unlink($temp); if ($verbose) { print STDERR "...completed\n"; } - -# Suppressed headers still owe field 2 a filename. -my %filenameOfCommit; +# Runs git blame with @options on $file and writes the records to the temp file. +# Returns the wait status of git, or undef if $limit (seconds, 0 = none) passed first. +# On undef, git is killed and the temp file is empty again. +sub Blame_To_Temp { + my ($limit, @options) = @_; + %filenameOfCommit = (); + seek($fh, 0, 0) and truncate($fh, 0) or die "unable to empty [$temp]: $!"; + # fork + exec, not a shell, so the pid and the exit status are those of git. + # The stderr of git goes to a file, so the script can tell the xdiff failure + # from another exit 128. It is printed again to STDERR after git ends. + my $errFile = "$temp.err"; + $gitStderr = ''; + my $pid = open(IN, '-|'); + die "unable to start git blame: $!" unless defined $pid; + if ($pid == 0) { + open(STDERR, '>', $errFile) or _exit(127); + # '--': a path that starts with '-' is a path, not an option. + exec('git', '-C', $repo, 'blame', @options, '--', $file); + print STDERR "unable to execute git: $!\n"; + _exit(127); + } + my $timedOut = 0; + eval { + local $SIG{ALRM} = sub { die "c100-limit\n" }; + alarm($limit); + while (my $l = Read_Record()) { + print $fh $l or Fail("unable to write [$temp]: $!"); + print $fh "\n" or Fail("unable to write [$temp]: $!"); + } + alarm(0); + }; + alarm(0); + if ($@) { + die $@ unless $@ eq "c100-limit\n"; + $timedOut = 1; + kill('KILL', $pid); + } + close IN; + my $waitStatus = $?; + if (open(my $efh, '<', $errFile)) { + local $/; + $gitStderr = <$efh>; + $gitStderr = '' unless defined $gitStderr; + close $efh; + print STDERR $gitStderr; + } + unlink($errFile); + return undef if $timedOut; + return $waitStatus; +} sub Read_Record { my $f ; @@ -91,17 +188,20 @@ sub Read_Record { $f = "$1;"; $sawFilename = 0; } elsif ($_ =~ /^(filename) (.+)$/) { - $filenameOfCommit{$cid} = $2 if defined $cid; + # As git prints it: C-quoted if needed, so never a raw tab or newline. + # Field 2 keeps this form; only the compare uses the raw bytes. + my $name = $2; + $filenameOfCommit{$cid} = $name if defined $cid; $sawFilename = 1; - if ($2 ne $file) { - $f .= $2 . ";"; + if (Unquote_Path($name) ne $file) { + $f .= $name . ";"; } else { $f .= ";"; } } elsif (/^ (.*)/) { #actual line if (not $sawFilename) { my $known = defined $cid ? $filenameOfCommit{$cid} : undef; - if (defined $known and $known ne $file) { + if (defined $known and Unquote_Path($known) ne $file) { $f .= $known . ";"; } else { $f .= ";"; @@ -116,6 +216,21 @@ sub Read_Record { return $f; } +# git blame --porcelain prints "filename" C-quoted (core.quotePath) when the path +# has a byte >= 0x80, '"', '\' or a control byte. Gives the raw bytes back, as +# the path is on the command line, for the compare with $file. A path that git +# did not quote never starts with '"' (git quotes every path that has one), so +# it comes back unchanged. +sub Unquote_Path { + my ($s) = @_; + return $s unless $s =~ /^"(.*)"$/s; + my $q = $1; + my %escape = ('a' => "\a", 'b' => "\b", 't' => "\t", 'n' => "\n", + 'v' => "\x0b", 'f' => "\f", 'r' => "\r", '"' => '"', '\\' => '\\'); + $q =~ s/\\([0-3][0-7][0-7]|[abtnvfr"\\])/length($1) == 3 ? chr(oct($1)) : $escape{$1}/ge; + return $q; +} + sub Usage { my ($m) = @_; print STDERR "$m\n"; @@ -123,23 +238,41 @@ sub Usage { exit(1); } +# Removes the partial output and stops with a non-zero exit. +sub Fail { + my ($m) = @_; + unlink($temp, "$temp.err"); + print STDERR "Error: $m; no blame file written\n"; + exit(1); +} + sub copy_file { my ($from, $toDir, $toName) = @_; - + my $to = "${toDir}/$toName"; $toDir = dirname($to); -# printf ("copy [$from] to [$to] [$toDir][$toName]\n"); - die "from file does not exist in copy_file [$from]" if not -f $from; if (not -d $toDir) { printf("Creating directory [$toDir]\n"); - make_path($toDir) or "die unable to create directory $to"; - } - move($from, $to) or - (unlink($to), "unable to move [$from] to [$to]"); + make_path($toDir); + Fail("unable to create directory [$toDir]") unless -d $toDir; + } + # Same folder as the output, so the rename is atomic: an existing output + # file is always complete. + my $partial = "$to.partial.$$"; + if (not copy($from, $partial)) { + my $err = $!; + unlink($partial); + Fail("unable to copy [$from] to [$partial]: $err"); + } + if (not rename($partial, $to)) { + my $err = $!; + unlink($partial); + Fail("unable to move [$partial] to [$to]: $err"); + } } __END__ @@ -155,6 +288,12 @@ =head1 SYNOPSIS Options: --blame-extension=s extension to append to output files. Default is .blame + --c100-limit=N seconds for git blame -C100. After that, the + script kills it and runs git blame without -C. + Default is $CREGIT_C100_LIMIT, else 3600. + 0 means no limit. + --overwrite replace an existing output file. Without it, the + script skips a file that already exists. --help brief help message --man full documentation @@ -177,4 +316,10 @@ =head1 DESCRIPTION with the same full path as the original file, with the .blame extension (which can be changed with the --blame option). +If git blame -C100 does not end within the limit, the script runs +git blame without -C and prints "PLAIN-BLAME after s of -C100" +to STDERR. The script does this only after the xdiff failure of git +(exit 128 with "unable to generate diff" on STDERR); any other exit 128 is a +failure. If git fails, the script writes no output file and exits 1. + =cut diff --git a/generate_dataset/DATASET.md b/generate_dataset/DATASET.md index aa115d40..5d5e1414 100644 --- a/generate_dataset/DATASET.md +++ b/generate_dataset/DATASET.md @@ -38,7 +38,7 @@ ORDER BY tokens DESC; -- Tokens authored by a specific person SELECT file_path, source_line, source_text, token_type, token_value FROM 'dataset.parquet' -WHERE person_name = 'Linus Torvalds' +WHERE person_name = 'Ada Example' ORDER BY file_path, token_index; ``` @@ -119,8 +119,36 @@ numbers or booleans. Cast at the query: `CAST(size_kb AS BIGINT)`, or ### Commit trailers (footers) Fifteen `LIST(TEXT)` columns, never NULL: absence is an empty list, so test with -`len(footer_reviewed_by) = 0`. Source is `cregit.db`'s `footers` table, grouped by -commit, keys matched case-insensitively, each list in `footers.idx` order. +`len(footer_reviewed_by) = 0`. Source is the `footers` table of the original +log DB (`-original.db`, Step 3 output), grouped by commit and joined on +`original_commit_sha`. Keys are matched case-insensitively. The trailers are +JGit's `getFooterLines` of the original commit message. + +Each of the 13 typed lists holds **one entry per person**, not the raw trailer +lines. A message can name one person twice, with the same text or with another +address (for example `Pat Example (Corp) ` and +`Pat Example `). In one list, two values are the same person +when: + +- their addresses resolve to the same `personid`. The address is the one that + `footer_personids` uses: the address inside `<…>`, else a bare address. It + is matched against `emails.emailaddr`. If the `emails` rows of one address + name two persons, the lowest `personid` is used. +- their addresses are not in `emails`, and the lowercased addresses are equal. + +A value with no address is kept as it is and is never merged, also when its +text repeats. For each person, the list keeps the value with the lowest +`footers.idx`, at the place of that idx, so the list is in the order of the +first trailer line of each person. The rule works in each list separately: one +person can be in `footer_signed_off_by` and in `footer_reviewed_by`. +`footer_personids` and `footer_person_names` do not change: they are sets. + +The source is not `cregit.db`. In the cregit repository, blobExec appends +`"\n" + "Former-commit-id: "` to a message whose last paragraph has +trailers. A git CLI message ends in `"\n"`, so the result has a blank line +before `Former-commit-id`, and JGit then finds no other trailer. A message with +no final newline (for example, from the GitHub web UI) kept its trailers. +Datasets made before this change have this defect. | Column | Trailer key | |--------|-------------| @@ -244,6 +272,7 @@ tables) maps multiple email addresses to a single `personid`. ``` token_map.commit_sha → commits.cid → commitmap.cid → commitmap.originalcid (original commit SHA) + → original DB footers.cid (trailers) → emails.emailname → persons.personid → persons.personname \_ (matched on autname + autemail) ``` @@ -315,6 +344,48 @@ whose tokenizer's lexer is not fit for real autotools input, and `.ixx`, `.inl`, `.cppm`, `.cxxm`, `.ipp`, for which srcML emits **zero tokens and exits 0**. `file_mask` on every row records the mask that decided this. +### Files blamed without `-C100` + +Step 7 blames each file with `git blame -C100`. This option credits a line that +moved in from another file to the commit that wrote it, not to the commit that +moved it. On a huge generated file (protobuf or thrift Java, a tree-sitter +`parser.c`, ICU `stubdata.cpp`), `-C100` can take 3 to 38 hours for one file. + +Thus `blameRepo/formatBlame.pl` has a wall-clock limit for `-C100`: 3600 seconds +by default. The option `--c100-limit=SECONDS` or the environment variable +`CREGIT_C100_LIMIT` sets a different limit. The value 0 removes the limit. When +the limit passes, the script stops `-C100` and blames the same file with plain +`git blame`, with no limit. The pipeline log then holds one line +`PLAIN-BLAME after s of -C100`. + +In such a file, a line that moved in from another file has the commit that moved +it. The rows of the file do not record this. To find these files, search the +pipeline log for `PLAIN-BLAME`. A file that ends within the limit gets the same +blame bytes as before. + +Git also exits 128 when `-C100` must diff a candidate blob over 1 GiB (xdiff +limit). The script then reads the stderr of git. Only when it holds +`unable to generate diff` does it run plain `git blame` and print +`PLAIN-BLAME after git exit 128 of -C100`. Any other exit 128 is a +failure, as below. + +The script writes each `.blame` file under a temporary name in the output +folder and renames it into place, so an existing `.blame` file is complete. A +failed write, close, copy or rename removes the partial file, prints an error +and exits 1. Without `--overwrite`, the script skips a file that exists; with +it, the script replaces the file. `blameRepoFiles.pl --overwrite` passes the +option on. + +Field 2 of a `.blame` record is the old file name as git prints it, with no +escape. A semicolon in the old name adds a field for a reader that splits the +record on `;`. The dataset has this form, so it stays. + +If git fails or stops early for a different reason (for example, the kernel +kills it for memory), `formatBlame.pl` writes no `.blame` file and exits 1. +`blameRepoFiles.pl` counts the error, and the blame step fails. Before this +change, the script wrote an empty `.blame` file and exited 0. The tokens of that +file were lost, and no error showed it. + ### Computed and then dropped, or never typed - `func_name` is computed in Phase 1 for `DECL` tokens and is **not** carried into @@ -341,6 +412,9 @@ source files ───────────┤ cregit.db ──────────────┤ (commits, commitmap) │ │ +-original.db ─────┤ +(footers) │ + │ persons.db ─────────────┤ (emails, persons) │ │ @@ -364,9 +438,10 @@ For each `.blame` file: ### Phase 2: DuckDB JOIN → Parquet -Join `token_map` with the three SQLite databases: +Join `token_map` with the four SQLite databases: - `commits` — commit metadata (author, committer, dates, summary) - `commitmap` — cregit → original commit SHA mapping +- `footers` of the original log DB — commit trailers, by original commit SHA - `emails` / `persons` — person identity resolution Output is written as ZSTD-compressed Parquet. @@ -378,6 +453,7 @@ uv run python generate_dataset/generate_dataset.py \ --blame-dir ../cregit-files/blame \ --source-dir ../cregit-files/jq-original \ --cregit-db ../cregit-files/jq-cregit.db \ + --original-db ../cregit-files/jq-original.db \ --persons-db ../cregit-files/jq-persons.db \ --output ../cregit-files/jq-dataset.parquet \ --repo-name jq \ @@ -391,6 +467,7 @@ uv run python generate_dataset/generate_dataset.py \ | `--blame-dir` | yes | Directory containing `.blame` files (Step 8 output). | | `--source-dir` | yes | Root of the original source tree. | | `--cregit-db` | yes | Path to `cregit.db` (Step 5 output). | +| `--original-db` | yes | Path to the git log DB of the original repository (Step 3 output). The `footer_*` columns come from its `footers` table. If the file is missing or has no `footers` table, the run stops before Phase 1. | | `--persons-db` | yes | Path to `persons.db` (Step 6 output). | | `--output` | yes | Output Parquet file path. | | `--repo-name` | no | Repository name. Default: inferred from output filename. | diff --git a/generate_dataset/generate_dataset.py b/generate_dataset/generate_dataset.py index 1712cef7..d47921ab 100644 --- a/generate_dataset/generate_dataset.py +++ b/generate_dataset/generate_dataset.py @@ -20,6 +20,7 @@ --blame-dir ../cregit-files/blame \ --source-dir ../cregit-files/jq-original \ --cregit-db ../cregit-files/jq-cregit.db \ + --original-db ../cregit-files/jq-original.db \ --persons-db ../cregit-files/jq-persons.db \ --output ../cregit-files/jq-dataset.parquet """ @@ -514,6 +515,33 @@ def project_meta_sql(project_meta) -> str: ) +# The columns that the trailer query reads from the footers table. +FOOTER_COLUMNS = ("cid", "idx", "key", "value") + + +def footers_table_problem(db_path) -> str: + """Return why db_path cannot supply the trailers, or "" when it can. + + The trailers come from the footers table that slickGitLog writes for the + original repository. Check that table before Phase 1, because Phase 1 can + take half an hour, and a bad input must stop the run before that work. + """ + try: + con = sqlite3.connect(f"file:{Path(db_path).resolve()}?mode=ro", uri=True) + try: + names = {row[1] for row in con.execute("PRAGMA table_info(footers)")} + finally: + con.close() + except sqlite3.Error as exc: + return f"it is not a readable SQLite database ({exc})" + if not names: + return "it has no footers table" + missing = [c for c in FOOTER_COLUMNS if c not in names] + if missing: + return f"its footers table has no column {', '.join(missing)}" + return "" + + def main(): parser = argparse.ArgumentParser( description="Generate unified Parquet dataset from CreGit pipeline outputs" @@ -527,6 +555,15 @@ def main(): parser.add_argument( "--cregit-db", required=True, help="Path to cregit.db (Step 5 output)" ) + parser.add_argument( + "--original-db", + required=True, + help="Path to the git log DB of the original repository (Step 3 output). " + "The footer_* columns come from its footers table, keyed by the original " + "commit SHA. The cregit DB cannot supply them: its commit messages carry " + "the Former-commit-id trailer that blobExec adds, and when a blank line " + "comes before that trailer, JGit sees no other trailer.", + ) parser.add_argument( "--persons-db", required=True, help="Path to persons.db (Step 6 output)" ) @@ -596,11 +633,22 @@ def main(): print(f"ERROR: {label} not found: {p}", file=sys.stderr) sys.exit(1) - for p, label in [(args.cregit_db, "cregit-db"), (args.persons_db, "persons-db")]: + for p, label in [(args.cregit_db, "cregit-db"), (args.original_db, "original-db"), + (args.persons_db, "persons-db")]: if not Path(p).is_file(): print(f"ERROR: {label} not found: {p}", file=sys.stderr) + if label == "original-db": + print(" Step 3 writes this file. Run the pipeline again from step 3 " + "or earlier. The trailers do not come from the cregit DB.", + file=sys.stderr) sys.exit(1) + problem = footers_table_problem(args.original_db) + if problem: + print(f"ERROR: original-db {args.original_db} cannot supply the trailers: " + f"{problem}", file=sys.stderr) + sys.exit(1) + if not args.repo_name: args.repo_name = output_path.stem.replace("-dataset", "") @@ -727,6 +775,10 @@ def main(): con.execute(f"CALL sqlite_attach('{sync_db_path}')") con.execute(f"CALL sqlite_attach('{args.cregit_db}')") con.execute(f"CALL sqlite_attach('{args.persons_db}')") + # A named catalog, not sqlite_attach: the original DB has the same table + # names as the cregit DB, and sqlite_attach puts every table in one schema. + original_db = str(args.original_db).replace(chr(39), chr(39) * 2) + con.execute(f"ATTACH '{original_db}' AS original_log (TYPE sqlite, READ_ONLY)") meta_sql = project_meta_sql(project_meta) @@ -774,8 +826,8 @@ def main(): coalesce(ftr.footer_mentored_by, []) AS footer_mentored_by, coalesce(ftr.footer_assisted_by, []) AS footer_assisted_by, coalesce(ftr.footer_thanks_to, []) AS footer_thanks_to, - coalesce(ftr.footer_personids, []) AS footer_personids, - coalesce(ftr.footer_person_names, []) AS footer_person_names + coalesce(fpr.footer_personids, []) AS footer_personids, + coalesce(fpr.footer_person_names, []) AS footer_person_names FROM token_map t JOIN commits c ON t.commit_sha = c.cid @@ -798,13 +850,65 @@ def main(): list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'helped-by') AS footer_helped_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'mentored-by') AS footer_mentored_by, list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'assisted-by') AS footer_assisted_by, - list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'thanks-to') AS footer_thanks_to, + list(f.value ORDER BY f.idx) FILTER (WHERE LOWER(f.key) = 'thanks-to') AS footer_thanks_to + -- One entry for each person in each list: the trailer line with + -- the lowest idx. A message can name one person twice, with the + -- same text or with another address of that person. + FROM ( + SELECT + f.cid, f.idx, f.key, f.value, + row_number() OVER (PARTITION BY f.cid, LOWER(f.key), f.who + ORDER BY f.idx) AS nth, + f.who + FROM ( + -- who: the personid of the address, else the lowercased + -- address, else NULL (a value with no address is kept). + -- The address is the one that footer_personids uses. + SELECT + f.cid, f.idx, f.key, f.value, + CASE WHEN fa.personid IS NOT NULL THEN 'p:' || fa.personid + WHEN f.addr IS NOT NULL THEN 'a:' || LOWER(f.addr) + END AS who + FROM ( + SELECT + f.cid, f.idx, f.key, f.value, + coalesce( + nullif(regexp_extract(f.value, '<([^>]+)>', 1), ''), + nullif(regexp_extract(f.value, '([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+[.][A-Za-z]{{2,}})', 1), '') + ) AS addr + -- The trailers come from the original repository, keyed by + -- the original commit SHA. In the cregit repository, blobExec + -- appends "\\n" + "Former-commit-id: X" to a message whose last + -- paragraph has trailers. When that message ends in "\\n" (the + -- git CLI writes it so), a blank line comes before the new + -- trailer, and JGit then sees only Former-commit-id. + FROM original_log.footers f + ) f + -- One personid for each address. min() if the emails rows of + -- one address name two persons. + LEFT JOIN ( + SELECT emailaddr, min(personid) AS personid + FROM emails + WHERE personid IS NOT NULL + GROUP BY emailaddr + ) fa ON fa.emailaddr = f.addr + ) f + ) f + WHERE f.who IS NULL OR f.nth = 1 + GROUP BY f.cid + ) ftr ON coalesce(m.originalcid, t.commit_sha) = ftr.cid + -- A separate subquery, so that the emails join cannot repeat a + -- trailer in the lists above: one address can have many emails + -- rows, one for each name that it committed with. + LEFT JOIN ( + SELECT + f.cid, list(DISTINCT fe.personid ORDER BY fe.personid) FILTER (WHERE fe.personid IS NOT NULL) AS footer_personids, list(DISTINCT coalesce(fpn.personname, fe.personid) ORDER BY coalesce(fpn.personname, fe.personid)) FILTER (WHERE coalesce(fpn.personname, fe.personid) IS NOT NULL) AS footer_person_names - FROM footers f + FROM original_log.footers f -- regexp_extract returns '' rather than NULL on no match, so -- without nullif a no-address trailer joins on '' and matches -- every `Name <>` row. Bracketed form first: correct precedence. @@ -815,7 +919,7 @@ def main(): ) LEFT JOIN persons fpn ON fe.personid = fpn.personid GROUP BY f.cid - ) ftr ON c.cid = ftr.cid + ) fpr ON coalesce(m.originalcid, t.commit_sha) = fpr.cid ORDER BY t.file_path, t.token_index ) TO '{output_path}' diff --git a/run_pipeline_process.sh b/run_pipeline_process.sh index 7ac200e4..de073b65 100755 --- a/run_pipeline_process.sh +++ b/run_pipeline_process.sh @@ -172,6 +172,7 @@ build_dataset_argv() { --blame-dir "$WORK/blame" --source-dir "$REPO_PATH_ORIGINAL" --cregit-db "$DB_PATH_CREGIT" + --original-db "$DB_PATH_ORIGINAL" --persons-db "$DB_PATH_PERSONS" --output "$DATASET_PATH" --repo-name "$REPO_NAME" @@ -1020,6 +1021,8 @@ if [ "$STEP_NUM" -ge "$FROM_STEP" ]; then DATASET_SCRIPT="$CREGIT/generate_dataset/generate_dataset.py" [ -f "$DATASET_SCRIPT" ] || die "dataset generator not found: $DATASET_SCRIPT" if [ -n "$PYTHON" ] && "$PYTHON" -c 'import duckdb' 2>/dev/null; then +# The footer_* columns come from the original log DB (see generate_dataset.py). +[ -f "$DB_PATH_ORIGINAL" ] || die "step 3 did not produce $DB_PATH_ORIGINAL, and step 10 reads the trailers from it. Resume from step 3 or earlier." build_dataset_argv "$PYTHON" "$DATASET_SCRIPT" "${DATASET_ARGV[@]}" log "dataset written: $DATASET_PATH" diff --git a/tests/t/blamePaths.t b/tests/t/blamePaths.t new file mode 100644 index 00000000..fee49aa7 --- /dev/null +++ b/tests/t/blamePaths.t @@ -0,0 +1,159 @@ +#!/usr/bin/env perl + +# Paths that git quotes (a byte >= 0x80, '"', '\', a tab, a newline) and paths +# that start with '-' must be blamed like any other path. Before the fix, +# blameRepoFiles.pl read `git ls-files` without -z (a quoted path did not match +# the mask, so the file was silently not blamed) and passed the path to +# formatBlame.pl and git without '--' (a leading '-' was read as an option). + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $root = "$FindBin::Bin/../.."; # tests/t -> repo root +my $script = "$root/blameRepo/blameRepoFiles.pl"; +my $format = "$root/blameRepo/formatBlame.pl"; +my $workdir = tempdir(CLEANUP => 1); + +$ENV{GIT_CONFIG_NOSYSTEM} = 1; +$ENV{GIT_CONFIG_GLOBAL} = '/dev/null'; +$ENV{GIT_AUTHOR_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_COMMITTER_DATE} = '2020-01-01T00:00:00 +0000'; +$ENV{GIT_AUTHOR_NAME} = 'Alice'; +$ENV{GIT_AUTHOR_EMAIL} = 'alice@example.com'; +$ENV{GIT_COMMITTER_NAME} = 'Alice'; +$ENV{GIT_COMMITTER_EMAIL} = 'alice@example.com'; + +# List form, no shell: the paths below hold quotes, tabs and newlines. +sub git { + my ($repo, @args) = @_; + system('git', '-C', $repo, @args) == 0 or die "git failed: @args"; +} + +sub git_out { + my ($repo, @args) = @_; + open(my $fh, '-|', 'git', '-C', $repo, @args) or die "git failed: @args"; + local $/; + my $out = <$fh>; + close $fh or die "git failed: @args"; + return $out; +} + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', $path) or die "$path: $!"; + print $fh $content; + close $fh; +} + +sub slurp { + my ($path) = @_; + open(my $fh, '<', $path) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; +} + +# Bytes, not characters: git and the scripts see raw bytes. +my $eAcute = "\xc3\xa9"; +my $uUml = "\xc3\xbc"; +my @hard = ( + "caf${eAcute}.c", # non-ASCII byte + "dir ${uUml}/x.h", # non-ASCII byte in a folder name + "q\"uote.c", # double quote + "back\\slash.c", # backslash + "tab\there.c", # tab + "new\nline.c", # newline + "it's.c", # single quote (no quoting by git) + "-dash.c", # leading '-' + "---- Books & Notes ----/notes.c", # leading '-' folder + "a/-inner.c", # '-' in a later segment +); +my @plain = ("normal.c"); + +my $repo = "$workdir/repo"; +mkdir $repo or die $!; +git($repo, 'init', '-q', '-b', 'main'); +my $n = 0; +foreach my $p (@hard, @plain) { + my $full = "$repo/$p"; + (my $dir = $full) =~ s{/[^/]*$}{}; + system('mkdir', '-p', '--', $dir) == 0 or die "mkdir $dir"; + $n++; + write_file($full, "int v$n;\nint w$n;\n"); +} +write_file("$repo/notes.txt", "not source\n"); +git($repo, 'add', '-A'); +git($repo, 'commit', '-q', '-m', 'first'); + +# Sanity: git really quotes the hard paths in plain ls-files output. +my $listed = git_out($repo, 'ls-files'); +like($listed, qr/^"caf\\303\\251\.c"$/m, "git ls-files quotes a non-ASCII path (the defect input)"); +like($listed, qr/^"tab\\there\.c"$/m, "git ls-files quotes a tab"); + +my $out = "$workdir/blame-out"; +mkdir $out or die $!; +my $stdout = `perl '$script' '$repo' '$out' '\\.(c|h)\$' 2>'$workdir/stderr'`; +my $status = $?; +my $stderr = slurp("$workdir/stderr"); +is($status, 0, "blameRepoFiles.pl succeeds on hard paths") or diag($stderr); +my $total = scalar(@hard) + scalar(@plain); +like($stdout, qr/Newly processed \[$total\] Already done \[0\] files Error \[0\]/, + "every mask-matched file is processed, none fails"); + +foreach my $p (@hard, @plain) { + (my $shown = $p) =~ s/([^\x20-\x7e])/sprintf('\\x%02x', ord($1))/ge; + my $b = slurp("$out/$p.blame"); + ok(defined $b, "blame file exists for [$shown]"); + my @lines = defined $b ? split(/\n/, $b) : (); + is(scalar(@lines), 2, "two blame lines for [$shown]"); + # Field 2 is empty: the lines come from this file, not from another one. + like($lines[0] // '', qr/^[0-9a-f]{40};;\tint v[0-9]+;$/, "record format for [$shown]"); +} +ok(!-e "$out/notes.txt.blame", "the mask still filters out notes.txt"); + +# A second run finds every hard path as done (the -f check uses the raw path). +my $again = `perl '$script' '$repo' '$out' '\\.(c|h)\$' 2>/dev/null`; +like($again, qr/Newly processed \[0\] Already done \[$total\] files Error \[0\]/, + "a second run skips every hard path as done"); + +# formatBlame.pl alone, with a path that starts with '-'. +{ + my $o2 = "$workdir/fb-out"; + mkdir $o2 or die $!; + system('perl', $format, '--', $repo, '-dash.c', $o2) == 0 + or fail("formatBlame.pl -- -dash.c fails"); + ok(-s "$o2/-dash.c.blame", "formatBlame.pl blames a path that starts with '-'"); +} + +# git blame --porcelain quotes "filename" too. The compare with the blamed path +# uses the raw bytes (above: a hard path never names itself in field 2). Field 2 +# keeps git's quoted form, as before the fix: it never holds a raw newline or tab, +# so a record stays on one line. +foreach my $case (["old${eAcute}.c", "\"old\\303\\251.c\""], + ["old\nname.c", "\"old\\nname.c\""]) { + my ($old, $quoted) = @$case; + (my $shown = $old) =~ s/([^\x20-\x7e])/sprintf('\\x%02x', ord($1))/ge; + my $r = "$workdir/renamed-" . length($old) . "-" . ord(substr($old, 3, 1)); + mkdir $r or die $!; + git($r, 'init', '-q', '-b', 'main'); + write_file("$r/$old", join('', map { "int line$_;\n" } 1 .. 6)); + git($r, 'add', '-A'); + git($r, 'commit', '-q', '-m', 'add'); + git($r, 'mv', '--', $old, "new${uUml}.c"); + git($r, 'commit', '-q', '-m', 'rename'); + my $o3 = "$r-out"; + mkdir $o3 or die $!; + my $s = system('perl', $script, $r, $o3, '\.c$'); + is($s, 0, "blameRepoFiles.pl succeeds on the repo renamed from [$shown]"); + my $b = slurp("$o3/new${uUml}.c.blame") // ''; + my @lines = split(/\n/, $b); + is(scalar(@lines), 6, "six blame lines, one per record, after a rename from [$shown]"); + is(scalar(grep { /^[0-9a-f]{40};\Q$quoted\E;\tint line[0-9];$/ } @lines), 6, + "field 2 is git's quoted form of [$shown]"); +} + +done_testing(); diff --git a/tests/t/formatBlame.t b/tests/t/formatBlame.t index 5c13a737..ceb00bdd 100644 --- a/tests/t/formatBlame.t +++ b/tests/t/formatBlame.t @@ -2,9 +2,14 @@ use strict; use warnings; -use Test::More tests => 20; +use Test::More tests => 70; use FindBin; use File::Temp qw(tempdir); +use File::Basename qw(dirname); +use File::Spec; +use Cwd qw(abs_path); +use Digest::SHA qw(sha1_hex); +use POSIX (); my $root = "$FindBin::Bin/../.."; # tests/t -> repo root @@ -50,6 +55,17 @@ sub slurp_lines { return @lines; } +# Find a program on PATH in Perl. Do not run `command -v`: Perl runs a command +# with no shell characters directly, and `command` is often only a shell builtin. +sub find_on_path { + my ($name) = @_; + for my $dir (File::Spec->path) { + my $file = File::Spec->catfile($dir, $name); + return $file if -f $file && -x _; + } + die "$name is not on PATH"; +} + my $repo = "$workdir/repo"; mkdir $repo or die $!; git($repo, "init -q -b main"); @@ -132,6 +148,185 @@ my $cid2 = commit_as($repo, "Bob", "second"); "all six moved lines are credited to the author, naming origin.c"); } +# The -C100 time limit and the git exit status, on the same moved.c repository. +# A fake git on PATH is slow or fails for "blame"; all other commands go to the real git. +{ + my $repo2 = "$workdir/moved"; + my $mover = git($repo2, "rev-parse HEAD"); + my $realGit = find_on_path('git'); + my $fakeBin = "$workdir/fakebin"; + mkdir $fakeBin or die $!; + write_file("$fakeBin/git", <<"FAKE"); +#!/bin/sh +blame=0; c100=0 +for a in "\$@"; do + [ "\$a" = blame ] && blame=1 + [ "\$a" = -C100 ] && c100=1 +done +if [ \$blame = 1 ]; then + case "\$FAKE_GIT_MODE" in + slow-c100) [ \$c100 = 1 ] && exec sleep 60 ;; + slow-c100-plain-fails) [ \$c100 = 1 ] && exec sleep 60; exit 3 ;; + fail) echo "fatal: fake failure" >&2; exit 3 ;; + c100-exit128) [ \$c100 = 1 ] && { echo "fatal: unable to generate diff (fake)" >&2; exit 128; } ;; + c100-exit128-other) [ \$c100 = 1 ] && { echo "fatal: bad object (fake)" >&2; exit 128; } ;; + exit128) echo "fatal: unable to generate diff (fake)" >&2; exit 128 ;; + killed) printf '%040d 1 1 1\\n' 0; kill -9 \$\$ ;; + esac +fi +exec '$realGit' "\$@" +FAKE + chmod 0755, "$fakeBin/git" or die $!; + + my $buildDir = dirname($script) . "/build"; + my @tempsBefore = glob("$buildDir/tmpfile-*"); + + my $slurp = sub { + my ($f) = @_; + open(my $fh, '<', $f) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; + }; + my $run = sub { + my ($mode, $opts, %env) = @_; + my $dest = tempdir(CLEANUP => 1); + local $ENV{PATH} = "$fakeBin:$ENV{PATH}"; + local $ENV{FAKE_GIT_MODE} = $mode; + local $ENV{CREGIT_C100_LIMIT} = $env{limit} if defined $env{limit}; + my $t0 = time(); + my $status = system("perl '$script' $opts '$repo2' moved.c '$dest' 2>'$workdir/stderr'"); + return ($status, $dest, time() - $t0, $slurp->("$workdir/stderr")); + }; + + # (a) A blame that ends within the limit: the same bytes as with no limit. + my ($s1, $d1) = $run->('none', ''); + my ($s2, $d2) = $run->('none', '--c100-limit=0'); + my ($s3, $d3) = $run->('none', '--c100-limit=3600'); + is($s1 + $s2 + $s3, 0, "the default, no limit and 3600 s runs all succeed"); + my $bytes = $slurp->("$d1/moved.c.blame"); + is($slurp->("$d2/moved.c.blame"), $bytes, "no limit gives the same bytes as the default limit"); + is($slurp->("$d3/moved.c.blame"), $bytes, "a 3600 s limit gives the same bytes as the default limit"); + + # (b) -C100 passes the limit: plain blame output and the PLAIN-BLAME line. + my ($s4, $d4, $sec4, $err4) = $run->('slow-c100', '--c100-limit=1'); + is($s4, 0, "a -C100 run that passes the limit still succeeds"); + cmp_ok($sec4, '<', 30, "the script kills the slow -C100 git and does not wait for it"); + like($err4, qr/^PLAIN-BLAME moved\.c after [0-9]+ s of -C100$/m, + "the script prints the PLAIN-BLAME line to STDERR"); + my @plain = split /\n/, ($slurp->("$d4/moved.c.blame") // ''); + is(scalar(@plain), 10, "the plain blame file has ten lines"); + my @byMover = grep { /^\Q$mover\E;;/ } @plain[2 .. 7]; + is(scalar(@byMover), 6, "plain blame credits the moved lines to the mover, so no -C100 ran"); + my $plainRef = `git -C '$repo2' blame --porcelain moved.c | grep -c '^ '`; + chomp $plainRef; + is($plainRef, 10, "git blame without -C gives the same line count"); + + # The environment variable sets the limit when there is no flag. + my ($s5, $d5, $sec5, $err5) = $run->('slow-c100', '', limit => 1); + is($s5, 0, "CREGIT_C100_LIMIT=1 run succeeds"); + like($err5, qr/^PLAIN-BLAME moved\.c after [0-9]+ s of -C100$/m, + "CREGIT_C100_LIMIT sets the -C100 limit"); + is($slurp->("$d5/moved.c.blame"), $slurp->("$d4/moved.c.blame"), + "the flag and the environment variable give the same bytes"); + + # (c) git fails: a non-zero exit and no blame file. + my ($s6, $d6, undef, $err6) = $run->('fail', ''); + isnt($s6, 0, "a failing git blame gives a non-zero exit"); + ok(!-e "$d6/moved.c.blame", "a failing git blame writes no blame file"); + like($err6, qr/^Error: git blame of \[moved\.c\]/m, "a failing git blame prints an error"); + + # git killed after part of its output: the defect that wrote an empty .blame. + my ($s7, $d7) = $run->('killed', ''); + isnt($s7, 0, "a killed git blame gives a non-zero exit"); + ok(!-e "$d7/moved.c.blame", "a killed git blame writes no blame file"); + + # -C100 passes the limit and plain blame then fails. + my ($s8, $d8, undef, $err8) = $run->('slow-c100-plain-fails', '--c100-limit=1'); + isnt($s8, 0, "a failing plain blame after the limit gives a non-zero exit"); + ok(!-e "$d8/moved.c.blame", "a failing plain blame writes no blame file"); + + # (d) -C100 ends with git exit 128 (xdiff refuses a candidate blob over 1 GiB): plain blame. + my ($s9, $d9, undef, $err9) = $run->('c100-exit128', ''); + is($s9, 0, "a -C100 git exit 128 falls back to plain blame and succeeds"); + like($err9, qr/^PLAIN-BLAME moved\.c after git exit 128 of -C100$/m, + "the script prints the PLAIN-BLAME line for the git exit 128"); + is($slurp->("$d9/moved.c.blame"), $slurp->("$d4/moved.c.blame"), + "the exit 128 fallback gives the same bytes as the time-limit fallback"); + + # Plain blame also ends with exit 128: a non-zero exit and no blame file. + my ($s10, $d10) = $run->('exit128', ''); + isnt($s10, 0, "a plain blame exit 128 after the fallback gives a non-zero exit"); + ok(!-e "$d10/moved.c.blame", "a failing fallback writes no blame file"); + + # An exit 128 of -C100 that is not the xdiff failure is an error, not a fallback. + my ($s11, $d11, undef, $err11) = $run->('c100-exit128-other', ''); + isnt($s11, 0, "a -C100 exit 128 with another message gives a non-zero exit"); + ok(!-e "$d11/moved.c.blame", "a -C100 exit 128 with another message writes no blame file"); + unlike($err11, qr/PLAIN-BLAME/, "a -C100 exit 128 with another message prints no PLAIN-BLAME line"); + + my @tempsAfter = glob("$buildDir/tmpfile-*"); + is(scalar(@tempsAfter), scalar(@tempsBefore), "the failed runs leave no temp file in build/"); + + # A limit that is not a whole number of seconds is rejected. + my ($s9) = $run->('none', '--c100-limit=abc'); + isnt($s9, 0, "a limit that is not a number is rejected"); +} + +# An existing output file: kept without --overwrite, replaced with it. A failed +# write leaves no partial file and a non-zero exit. +{ + my $repo2 = "$workdir/moved"; + my $dest = tempdir(CLEANUP => 1); + write_file("$dest/moved.c.blame", "stale\n"); + is(system("perl '$script' '$repo2' moved.c '$dest' 2>/dev/null"), 0, + "a run with an existing output file succeeds"); + is(join('', slurp_lines("$dest/moved.c.blame")), "stale", + "without --overwrite an existing output file is kept"); + is(system("perl '$script' --overwrite '$repo2' moved.c '$dest' 2>/dev/null"), 0, + "a run with --overwrite succeeds"); + my @fresh = slurp_lines("$dest/moved.c.blame"); + is(scalar(@fresh), 10, "--overwrite replaces the existing output file"); + my @partials = glob("$dest/*.partial.*"); + is(scalar(@partials), 0, "no partial file stays next to the output"); + + SKIP: { + skip "root can write to a read-only folder", 3 if $> == 0; + my $ro = tempdir(CLEANUP => 1); + chmod 0555, $ro or die $!; + my $out = `perl '$script' '$repo2' moved.c '$ro' 2>&1`; + isnt($? , 0, "an output folder that cannot be written gives a non-zero exit"); + ok(!-e "$ro/moved.c.blame", "no blame file is written to it"); + like($out, qr/^Error: unable to copy/m, "the script says that the copy failed"); + chmod 0755, $ro; + } +} + +# Field 2 is the old file name as git prints it. It is not escaped, so a +# semicolon in the old name adds a field when a reader splits on ";". This +# test records the behavior; the dataset was built with it. +{ + my $repo4 = "$workdir/semicolon"; + mkdir $repo4 or die $!; + git($repo4, "init -q -b main"); + write_file("$repo4/old;name.c", "int one;\n"); + git($repo4, "add ."); + my $first = commit_as($repo4, "Alice", "write the file"); + git($repo4, "mv 'old;name.c' new.c"); + commit_as($repo4, "Carol", "rename the file"); + + my $dest = tempdir(CLEANUP => 1); + is(system("perl '$script' '$repo4' new.c '$dest' 2>/dev/null"), 0, + "formatBlame.pl succeeds when the old name has a semicolon"); + my @lines = slurp_lines("$dest/new.c.blame"); + is($lines[0], "$first;old;name.c;\tint one;", + "field 2 keeps the semicolon of the old name, unescaped"); + my ($head) = split /\t/, $lines[0], 2; + is(scalar(split /;/, $head, -1), 4, + "a split on a semicolon gives one more field than a name without one"); +} + # Two non-contiguous groups from one commit, in a renamed file: --porcelain # suppresses the second group's header, and field 2 must survive that. { @@ -164,3 +359,74 @@ my $cid2 = commit_as($repo, "Bob", "second"); is($lines[2], "$first;old.c;\tint three;", "the second group of the same commit still names the old filename"); } + +# Two runners on one output directory: a lock per output file, and a skip when +# the file is already there. A fake git counts the blames and makes them slow. +{ + my $realGit = find_on_path('git'); + my $lockBin = "$workdir/lockbin"; + mkdir $lockBin or die $!; + my $counter = "$workdir/blame-count"; + write_file("$lockBin/git", <<"FAKE"); +#!/bin/sh +for a in "\$@"; do + if [ "\$a" = blame ]; then echo x >> '$counter'; sleep 2; fi +done +exec '$realGit' "\$@" +FAKE + chmod 0755, "$lockBin/git" or die $!; + my $count = sub { + return 0 unless -f $counter; + my @l = slurp_lines($counter); + unlink $counter; + return scalar(@l); + }; + my $slurp = sub { + open(my $fh, '<', $_[0]) or return undef; + local $/; + my $c = <$fh>; + close $fh; + return $c; + }; + local $ENV{PATH} = "$lockBin:$ENV{PATH}"; + + my $ref = tempdir(CLEANUP => 1); + system("perl '$script' '$repo' g.c '$ref' 2>/dev/null") == 0 or die "reference run failed"; + $count->(); + my $refBytes = $slurp->("$ref/g.c.blame"); + + # (a) The output file exists: no blame, and the file keeps its bytes. + my $done = tempdir(CLEANUP => 1); + write_file("$done/g.c.blame", "keep\n"); + my $s0 = system("perl '$script' '$repo' g.c '$done' 2>/dev/null"); + is($s0, 0, "a file that is already blamed gives exit 0"); + is($slurp->("$done/g.c.blame"), "keep\n", "an existing blame file keeps its bytes"); + is($count->(), 0, "an existing blame file starts no git blame"); + + # (b) Two processes on the same file at the same time: one blames, one skips. + my $both = tempdir(CLEANUP => 1); + my @pids; + for my $i (1, 2) { + my $pid = fork(); + die "fork: $!" unless defined $pid; + if ($pid == 0) { + exec("perl '$script' '$repo' g.c '$both' 2>/dev/null") or POSIX::_exit(127); + } + push @pids, $pid; + select(undef, undef, undef, 0.3); + } + my @st = map { waitpid($_, 0); $? } @pids; + is($st[0], 0, "the first of two runners gives exit 0"); + is($st[1], 0, "the second of two runners gives exit 0"); + is($count->(), 1, "two runners on one file start one git blame"); + is($slurp->("$both/g.c.blame"), $refBytes, "the blame file of two runners equals a single run"); + ok(-s "$both/g.c.blame", "the blame file of two runners is not empty"); + + # (c) The locks stay in build/, out of the output directory. + my $lock = dirname($script) . "/build/locks/" . sha1_hex(abs_path($both) . "/g.c.blame") . ".lock"; + ok(-f $lock, "the lock file is in build/locks/"); + opendir(my $dh, $both) or die $!; + my @entries = sort grep { !/^\.\.?$/ } readdir($dh); + closedir $dh; + is_deeply(\@entries, ["g.c.blame"], "the output directory holds only the blame file"); +} diff --git a/tests/test_generate_dataset.py b/tests/test_generate_dataset.py index e7addb25..fba697aa 100644 --- a/tests/test_generate_dataset.py +++ b/tests/test_generate_dataset.py @@ -100,6 +100,16 @@ def test_is_ws_treats_none_as_not_whitespace(): # there: filling a tmpfs also exhausts system memory. # --------------------------------------------------------------------------- # +def make_original_db(path): + """The git log DB of the original repository, as slickGitLog writes it in + step 3. Only its footers table is read.""" + con = sqlite3.connect(path) + con.execute("CREATE TABLE footers (cid CHAR(40), idx INTEGER, key TEXT, value TEXT)") + con.commit() + con.close() + return path + + def run_main(monkeypatch, tmp_path, argv_extra=()): """Drive main() far enough to reach the scratch allocation, then stop. @@ -116,6 +126,7 @@ def run_main(monkeypatch, tmp_path, argv_extra=()): (src / "a.c").write_text("int x;\n") for name in ("cregit.db", "persons.db"): (tmp_path / name).write_text("") + make_original_db(tmp_path / "original.db") out = tmp_path / "out" / "proj-dataset.parquet" seen = {} @@ -130,6 +141,7 @@ def fake_mkstemp(*a, **kw): "--blame-dir", str(blame), "--source-dir", str(tmp_path / "src"), "--cregit-db", str(tmp_path / "cregit.db"), + "--original-db", str(tmp_path / "original.db"), "--persons-db", str(tmp_path / "persons.db"), "--output", str(out), *argv_extra, @@ -308,7 +320,8 @@ def test_a_key_absent_from_the_sidecar_fails_loudly(tmp_path): # --------------------------------------------------------------------------- # TOTAL_COLUMNS = 67 # 38 token/commit columns + 29 metadata -SHA = "a" * 40 +SHA = "a" * 40 # the commit in the cregit repository +ORIGINAL_SHA = "b" * 40 # the same commit in the original repository def build_tiny_project(tmp_path): @@ -333,9 +346,12 @@ def build_tiny_project(tmp_path): con.execute("INSERT INTO commits VALUES (?,?,?,?,?,?,?,?,0)", (SHA, "A Dev", "a@example.com", "2020-01-01", "A Dev", "a@example.com", "2020-01-01", "first")) + con.execute("INSERT INTO commitmap VALUES (?,?,'')", (SHA, ORIGINAL_SHA)) con.commit() con.close() + original_db = make_original_db(tmp_path / "original.db") + persons_db = tmp_path / "persons.db" con = sqlite3.connect(persons_db) con.executescript(""" @@ -347,26 +363,27 @@ def build_tiny_project(tmp_path): """) con.commit() con.close() - return blame, src, cregit_db, persons_db + return blame, src, cregit_db, original_db, persons_db def generate(monkeypatch, tmp_path, repo_name="proj", argv_extra=(), seed=None): """Run main() for real and return the Parquet path. - `seed(cregit_db, persons_db)` runs between the fixture and main(), for the - tests that need a trailer or a person in the input. + `seed(cregit_db, original_db, persons_db)` runs between the fixture and + main(), for the tests that need a trailer or a person in the input. """ import generate_dataset as gd - blame, src, cregit_db, persons_db = build_tiny_project(tmp_path) + blame, src, cregit_db, original_db, persons_db = build_tiny_project(tmp_path) if seed is not None: - seed(cregit_db, persons_db) + seed(cregit_db, original_db, persons_db) out = tmp_path / "out" / "proj-dataset.parquet" monkeypatch.setattr(gd.sys, "argv", [ "generate_dataset.py", "--blame-dir", str(blame), "--source-dir", str(src), "--cregit-db", str(cregit_db), + "--original-db", str(original_db), "--persons-db", str(persons_db), "--output", str(out), "--repo-name", repo_name, @@ -455,16 +472,21 @@ def test_an_unknown_project_key_stops_the_run_before_phase_1(monkeypatch, tmp_pa # --------------------------------------------------------------------------- # -def seed_footers(*trailers, people=()): - """Return a `seed` for generate(): trailers into footers, people into - emails/persons. `people` is (personid, emailaddr, personname).""" - def seed(cregit_db, persons_db): - con = sqlite3.connect(cregit_db) - for idx, (key, value) in enumerate(trailers): - con.execute("INSERT INTO footers VALUES (?,?,?,?)", - (SHA, idx, key, value)) - con.commit() - con.close() +def seed_footers(*trailers, people=(), cregit_trailers=()): + """Return a `seed` for generate(): trailers into the footers table of the + original DB, keyed by the original commit SHA, and people into + emails/persons. `people` is (personid, emailaddr, personname). + `cregit_trailers` go into the footers table of the cregit DB, keyed by the + cregit commit SHA, as slickGitLog writes them in step 4.""" + def seed(cregit_db, original_db, persons_db): + for db, sha, rows in ((original_db, ORIGINAL_SHA, trailers), + (cregit_db, SHA, cregit_trailers)): + con = sqlite3.connect(db) + for idx, (key, value) in enumerate(rows): + con.execute("INSERT INTO footers VALUES (?,?,?,?)", + (sha, idx, key, value)) + con.commit() + con.close() con = sqlite3.connect(persons_db) for i, (pid, addr, pname) in enumerate(people, start=1): con.execute("INSERT INTO emails (recordid, personid, fullemail, " @@ -518,6 +540,287 @@ def test_a_trailer_with_no_address_attributes_nobody(monkeypatch, tmp_path): assert list(names) == [] +# --------------------------------------------------------------------------- # +# The trailers come from the original repository. In the cregit repository, +# blobExec appends "\n" + "Former-commit-id: X" to a message whose last +# paragraph has trailers. A git CLI message ends in "\n", so a blank line then +# comes before Former-commit-id, and JGit (step 4) sees no other trailer. +# --------------------------------------------------------------------------- # + +CLI_MESSAGE = ("Fix the parser\n" + "\n" + "The parser stopped at the first comment.\n" + "\n" + "Signed-off-by: A Dev \n" + "Reviewed-by: Bob B \n") + + +def rewritten_by_blobexec(message: str, original_sha: str) -> str: + """FormerCommitFooter.append for a last paragraph that has trailers.""" + return message + "\n" + "Former-commit-id: " + original_sha + + +def git_trailer_keys(message: str) -> list[str]: + import subprocess + out = subprocess.run(["git", "interpret-trailers", "--parse"], input=message, + capture_output=True, text=True, check=True).stdout + return [line.split(":", 1)[0] for line in out.splitlines()] + + +def test_the_rewritten_cli_message_hides_its_trailers_from_a_trailer_parser(): + """The defect itself, with git as the parser: the original message has two + trailers, and the rewritten message has only Former-commit-id.""" + import shutil + if shutil.which("git") is None: + pytest.skip("git is not on PATH") + rewritten = rewritten_by_blobexec(CLI_MESSAGE, ORIGINAL_SHA) + assert "\n\nFormer-commit-id: " in rewritten + assert git_trailer_keys(CLI_MESSAGE) == ["Signed-off-by", "Reviewed-by"] + assert git_trailer_keys(rewritten) == ["Former-commit-id"] + + +def test_a_cli_message_keeps_its_signed_off_by_and_reviewed_by(monkeypatch, tmp_path): + """The two DBs hold what slickGitLog finds for CLI_MESSAGE: the original DB + has both trailers, and the cregit DB has only Former-commit-id.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Reviewed-by", "Bob B "), + cregit_trailers=[("Former-commit-id", ORIGINAL_SHA)], + people=[("adev", "a@example.com", "A Dev"), + ("bob", "bob@example.com", "Bob B")])) + + row = duckdb.sql("select original_commit_sha, footer_signed_off_by, " + "footer_reviewed_by, footer_personids, footer_person_names " + f"from read_parquet('{out}')").fetchone() + sha, signed, reviewed, ids, names = row + assert sha == ORIGINAL_SHA + assert list(signed) == ["A Dev "] + assert list(reviewed) == ["Bob B "] + assert list(ids) == ["adev", "bob"] + assert list(names) == ["A Dev", "Bob B"] + + +def test_an_address_with_two_names_does_not_repeat_the_trailer(monkeypatch, tmp_path): + """The persons DB has one emails row for each name that an address + committed with. The join to emails must not repeat the trailer.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reviewed-by", "Bob B "), + people=[("bob", "bob@example.com", "Bob B"), + ("bob", "bob@example.com", "bob-laptop")])) + + ids, names, reviewed = footers_of(duckdb, out) + assert list(reviewed) == ["Bob B "] + assert list(ids) == ["bob"] + assert list(names) == ["Bob B"] + + +# --------------------------------------------------------------------------- # +# One entry for each person in each typed list. Two values are the same person +# when their addresses resolve to one personid; an address that is not in +# emails merges on its lowercased text; a value with no address never merges. +# The entry kept is the value with the lowest idx, at the place of that idx. +# --------------------------------------------------------------------------- # + +def typed_lists_of(duckdb, path, *cols): + row = duckdb.sql(f"select {', '.join(cols)} from read_parquet('{path}')").fetchone() + return [list(v) for v in row] + + +def test_an_identical_repeated_trailer_gives_one_entry(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["A Dev "] + + +def test_two_addresses_of_one_person_give_the_first_value(monkeypatch, tmp_path): + """The Linux case: "Pat Example (Corp) " and + "Pat Example " are one personid.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "Pat Example (Corp) "), + ("Signed-off-by", "Pat Example "), + people=[("pat", "pat@corp.example.com", "Pat Example"), + ("pat", "pat@home.example.com", "Pat Example")])) + + signed, ids = typed_lists_of(duckdb, out, "footer_signed_off_by", "footer_personids") + assert signed == ["Pat Example (Corp) "] + assert ids == ["pat"] + + +def test_an_address_with_two_personids_uses_the_lowest(monkeypatch, tmp_path): + """Two emails rows name one address for two persons. The lowest personid is + the key, so the other address of that lowest person is the same person. + The higher personid is listed first in emails, so a rule that took the + first row, or the highest one, would give two entries.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "Sam One "), + ("Signed-off-by", "Sam Two "), + people=[("zed", "sam@example.com", "Sam One"), + ("abe", "sam@example.com", "Sam One"), + ("abe", "sam@other.example.com", "Sam Two")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["Sam One "] + + +def test_two_addresses_of_two_persons_stay_two_entries(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev"), + ("adev2", "a@other.example.com", "A Dev")])) + + [signed] = typed_lists_of(duckdb, out, "footer_signed_off_by") + assert signed == ["A Dev ", "A Dev "] + + +def test_an_unresolved_address_merges_across_letter_case(monkeypatch, tmp_path): + """No emails row for the address: the lowercased address is the key.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "Carol "), + ("Reported-by", "carol "))) + + reported, ids = typed_lists_of(duckdb, out, "footer_reported_by", "footer_personids") + assert reported == ["Carol "] + assert ids == [] + + +def test_a_bare_address_and_a_bracketed_one_are_one_person(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "carol@example.com"), + ("Reported-by", "Carol "))) + + [reported] = typed_lists_of(duckdb, out, "footer_reported_by") + assert reported == ["carol@example.com"] + + +def test_a_value_with_no_address_is_never_merged(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Reported-by", "Dana Example"), + ("Reported-by", "Dana Example"), + ("Reported-by", "dana example"))) + + [reported] = typed_lists_of(duckdb, out, "footer_reported_by") + assert reported == ["Dana Example", "Dana Example", "dana example"] + + +def test_the_entries_keep_the_order_of_their_first_idx(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), # idx 0: a, kept + ("Signed-off-by", "Bob B "), # idx 1: bob, kept + ("Signed-off-by", "No Address"), # idx 2: kept + ("Signed-off-by", "A Dev (work) "), # idx 3: a again, dropped + ("Signed-off-by", "Carol "), # idx 4: unresolved, kept + ("Signed-off-by", "Bob B "), # idx 5: bob again, dropped + people=[("adev", "a@example.com", "A Dev"), + ("adev", "a@work.example", "A Dev"), + ("bob", "bob@example.com", "Bob B")])) + + signed, ids, names = typed_lists_of(duckdb, out, "footer_signed_off_by", + "footer_personids", "footer_person_names") + assert signed == ["A Dev ", "Bob B ", "No Address", + "Carol "] + assert ids == ["adev", "bob"] + assert names == ["A Dev", "Bob B"] + + +def test_one_person_in_two_lists_stays_in_both(monkeypatch, tmp_path): + """The rule works inside each typed list, not across the lists.""" + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + ("Signed-off-by", "A Dev "), + ("Reviewed-by", "A Dev "), + ("signed-off-by", "A Dev "), + people=[("adev", "a@example.com", "A Dev")])) + + signed, reviewed = typed_lists_of(duckdb, out, "footer_signed_off_by", + "footer_reviewed_by") + assert signed == ["A Dev "] + assert reviewed == ["A Dev "] + + +def test_the_trailers_of_the_cregit_db_are_not_read(monkeypatch, tmp_path): + duckdb = pytest.importorskip("duckdb") + out = generate(monkeypatch, tmp_path, seed=seed_footers( + cregit_trailers=[("Signed-off-by", "A Dev ")])) + + (signed,) = duckdb.sql("select footer_signed_off_by " + f"from read_parquet('{out}')").fetchone() + assert list(signed) == [] + + +def test_the_trailer_key_falls_back_to_the_cregit_sha_with_no_map_row(monkeypatch, tmp_path): + """The same fallback as original_commit_sha: no commitmap row, so the + original commit SHA is the cregit SHA.""" + duckdb = pytest.importorskip("duckdb") + + def seed(cregit_db, original_db, _persons_db): + con = sqlite3.connect(cregit_db) + con.execute("DELETE FROM commitmap") + con.commit() + con.close() + con = sqlite3.connect(original_db) + con.execute("INSERT INTO footers VALUES (?,0,'Signed-off-by','A Dev ')", + (SHA,)) + con.commit() + con.close() + + out = generate(monkeypatch, tmp_path, seed=seed) + sha, signed = duckdb.sql("select original_commit_sha, footer_signed_off_by " + f"from read_parquet('{out}')").fetchone() + assert sha == SHA + assert list(signed) == ["A Dev "] + + +def test_a_missing_original_db_stops_the_run_before_phase_1(monkeypatch, tmp_path, capsys): + """An old resume can lack the step 3 DB. The run must stop and say so, not + read the trailers from the cregit DB.""" + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, + argv_extra=("--original-db", str(tmp_path / "absent.db"))) + err = capsys.readouterr().err + assert "original-db not found" in err + assert "step 3" in err + + +@pytest.mark.parametrize("content, reason", [ + (b"", "no footers table"), + (b"not a database, only text" * 40, "not a readable SQLite database"), +], ids=["empty-file", "not-sqlite"]) +def test_an_original_db_with_no_footers_table_stops_the_run_before_phase_1( + monkeypatch, tmp_path, capsys, content, reason): + bad = tmp_path / "bad.db" + bad.write_bytes(content) + with pytest.raises(SystemExit): + run_main(monkeypatch, tmp_path, argv_extra=("--original-db", str(bad))) + assert reason in capsys.readouterr().err + + +def test_a_call_without_the_original_db_is_refused(monkeypatch, tmp_path): + """An old caller that does not pass --original-db must fail, not fall back.""" + import generate_dataset as gd + blame, src, cregit_db, _original_db, persons_db = build_tiny_project(tmp_path) + monkeypatch.setattr(gd.sys, "argv", [ + "generate_dataset.py", "--blame-dir", str(blame), "--source-dir", str(src), + "--cregit-db", str(cregit_db), "--persons-db", str(persons_db), + "--output", str(tmp_path / "out" / "proj-dataset.parquet")]) + with pytest.raises(SystemExit): + gd.main() + + # --------------------------------------------------------------------------- # # Token-line repairs before the source walk: srcML mojibake, the Rust position # prefix and the BOM. Each test checks text and position after the defect.